From 129f5eaf1e5bcc4e7126325d6f094ae486081887 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:06:58 -0400 Subject: [PATCH 01/22] drm: expose GEM memory accounting through Rust fdinfo Wire Rust DRM file operations and drivers into the standard fdinfo memory helper. Observe shmem residency without populating pages or taking the reservation lock, including cached pages after device unpinning. Add residency controls and a client-memory reporter which deduplicates shared client FDs and keeps shared/imported memory distinct from private handle accounting. --- drivers/gpu/drm/asahi/driver.rs | 4 + drivers/gpu/drm/drm_gem_shmem_helper.c | 36 +++++++ drivers/gpu/drm/tests/drm_gem_shmem_test.c | 15 +++ include/drm/drm_gem_shmem_helper.h | 4 + rust/kernel/drm/device.rs | 3 +- rust/kernel/drm/driver.rs | 10 ++ rust/kernel/drm/gem/mod.rs | 4 + rust/kernel/drm/gem/shmem.rs | 4 +- tools/asahi/gpu-fdinfo-tests.py | 93 ++++++++++++++++++ tools/asahi/gpu-memory-report-tests.py | 48 ++++++++++ tools/asahi/gpu-memory-report.py | 104 +++++++++++++++++++++ 11 files changed, 321 insertions(+), 4 deletions(-) create mode 100644 tools/asahi/gpu-fdinfo-tests.py create mode 100644 tools/asahi/gpu-memory-report-tests.py create mode 100644 tools/asahi/gpu-memory-report.py diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 510a70ba4f0bd9..df45f4a8ca91d1 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -5,6 +5,7 @@ use crate::module_parameters; use kernel::bindings; use kernel::{ + c_str, device::{ Core, DeviceContext, // @@ -100,6 +101,9 @@ impl drm::driver::Driver for AsahiDrmDriver { const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; + const SHOW_FDINFO: Option< + unsafe extern "C" fn(*mut bindings::drm_printer, *mut bindings::drm_file), + > = Some(bindings::drm_show_memory_stats); kernel::declare_drm_ioctls! { (ASAHI_GET_PARAMS, drm_asahi_get_params, diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c00da634972e50..c1b902ea067662 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,41 @@ MODULE_IMPORT_NS("DMA_BUF"); +/** + * drm_gem_shmem_object_rss - Observe resident shmem backing without populating it + * @obj: shmem GEM object whose handle keeps it alive + * + * Count page-cache pages even after the device has released its page array. + * The object's backing file remains alive throughout the fdinfo walk. An + * imported object retains its DMA mapping instead of a private shmem file. + */ +size_t drm_gem_shmem_object_rss(struct drm_gem_object *obj) +{ + if (obj->import_attach) + return obj->size; + if (!obj->filp) + return 0; + + return min_t(size_t, obj->size, + READ_ONCE(obj->filp->f_mapping->nrpages) << PAGE_SHIFT); +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_object_rss); + +/** + * drm_gem_shmem_object_status - Report resident backing for fdinfo + * @obj: shmem GEM object whose handle keeps it alive + * + * The fdinfo walk holds the client's handle-table spinlock. Observe residency + * without taking the reservation lock or populating pages. Cached pages are + * still resident after unpinning. Reclaim may race this observational snapshot, + * as permitted by drm_gem_object_funcs.status. + */ +enum drm_gem_object_status drm_gem_shmem_object_status(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_rss(obj) ? DRM_GEM_OBJECT_RESIDENT : 0; +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_object_status); + /** * DOC: overview * diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c index 5e69ff1d1ad673..dc4e8f3284ae5e 100644 --- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c +++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -53,6 +54,8 @@ static void drm_gem_shmem_test_obj_create(struct kunit *test) shmem = drm_gem_shmem_create(drm_dev, TEST_SIZE); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, shmem); KUNIT_EXPECT_EQ(test, shmem->base.size, TEST_SIZE); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_status(&shmem->base), 0); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), 0); KUNIT_EXPECT_NOT_NULL(test, shmem->base.filp); KUNIT_EXPECT_NOT_NULL(test, shmem->base.funcs); @@ -140,9 +143,14 @@ static void drm_gem_shmem_test_pin_pages(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); + /* Keep the cached backing resident while checking the unpinned state. */ + mapping_set_unevictable(shmem->base.filp->f_mapping); ret = drm_gem_shmem_pin(shmem); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_ASSERT_NOT_NULL(test, shmem->pages); + KUNIT_EXPECT_TRUE(test, drm_gem_shmem_object_status(&shmem->base) & + DRM_GEM_OBJECT_RESIDENT); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), TEST_SIZE); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->pages_use_count), 1); for (i = 0; i < (shmem->base.size >> PAGE_SHIFT); i++) @@ -150,7 +158,12 @@ static void drm_gem_shmem_test_pin_pages(struct kunit *test) drm_gem_shmem_unpin(shmem); KUNIT_EXPECT_NULL(test, shmem->pages); + /* Releasing a device pin leaves the backing file's cached pages resident. */ + KUNIT_EXPECT_TRUE(test, drm_gem_shmem_object_status(&shmem->base) & + DRM_GEM_OBJECT_RESIDENT); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), TEST_SIZE); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->pages_use_count), 0); + mapping_clear_unevictable(shmem->base.filp->f_mapping); } /* @@ -342,6 +355,8 @@ static void drm_gem_shmem_test_purge(struct kunit *test) KUNIT_EXPECT_NULL(test, shmem->pages); KUNIT_EXPECT_NULL(test, shmem->sgt); KUNIT_EXPECT_EQ(test, shmem->madv, -1); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), 0); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_status(&shmem->base), 0); } static int drm_gem_shmem_test_init(struct kunit *test) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index d25bca5352aa68..7db2bd21d85b21 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -298,6 +298,10 @@ static inline int drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct v * Driver ops */ +enum drm_gem_object_status +drm_gem_shmem_object_status(struct drm_gem_object *obj); +size_t drm_gem_shmem_object_rss(struct drm_gem_object *obj); + struct drm_gem_object * drm_gem_shmem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 4d6674f2450492..a73107c7b143aa 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -191,8 +191,7 @@ impl UnregisteredDevice { gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - - show_fdinfo: None, + show_fdinfo: T::SHOW_FDINFO, fbdev_probe: None, major: T::INFO.major, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 4971f3496b5813..eca6df3005a7fa 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -138,6 +138,16 @@ pub trait Driver { /// their `THIS_MODULE` so open files and exported DMA buffers retain the code. const MODULE: Option<&'static crate::ThisModule> = None; + /// Optional statistics for the standard DRM fdinfo interface. + /// + /// GEM drivers can opt into `drm_show_memory_stats`. Drivers may instead + /// provide engine counters or memory not represented by GEM handles. + /// The default reports only the generic DRM client identity. + const SHOW_FDINFO: Option< + unsafe extern "C" fn(*mut bindings::drm_printer, *mut bindings::drm_file), + > = None; + + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 7eabfbfdb23db0..ee4e1ec08b8018 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -526,6 +526,10 @@ pub(super) const fn create_fops( fops.read = Some(bindings::drm_read); fops.llseek = Some(bindings::noop_llseek); fops.mmap = Some(bindings::drm_gem_mmap); + #[cfg(CONFIG_PROC_FS)] + { + fops.show_fdinfo = Some(bindings::drm_show_fdinfo); + } fops.fop_flags = bindings::FOP_UNSIGNED_OFFSET; fops diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fb7e5c38e5483e..134bd0742bc336 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -157,8 +157,8 @@ impl Object { vmap: Some(bindings::drm_gem_shmem_object_vmap), vunmap: Some(bindings::drm_gem_shmem_object_vunmap), mmap: Some(bindings::drm_gem_shmem_object_mmap), - status: None, - rss: None, + status: Some(bindings::drm_gem_shmem_object_status), + rss: Some(bindings::drm_gem_shmem_object_rss), #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, diff --git a/tools/asahi/gpu-fdinfo-tests.py b/tools/asahi/gpu-fdinfo-tests.py new file mode 100644 index 00000000000000..1df2da6ea2a7e6 --- /dev/null +++ b/tools/asahi/gpu-fdinfo-tests.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production fdinfo residency helpers with cached and imported backing.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "drivers/gpu/drm/drm_gem_shmem_helper.c" + + +def function(text, signature): + start = text.index(signature) + opening = text.index("{", start) + depth = 1 + end = opening + 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + original = SOURCE.read_text() + functions = "\n".join(function(original, signature) for signature in [ + "size_t drm_gem_shmem_object_rss(", + "enum drm_gem_object_status drm_gem_shmem_object_status(", + ]) + fixture = r''' +#include +#include +#include +#include +#define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x)) +#define min_t(type,a,b) ((type)(a) < (type)(b) ? (type)(a) : (type)(b)) +#define PAGE_SIZE (1UL << PAGE_SHIFT) +struct address_space { unsigned long nrpages; }; +struct file { struct address_space *f_mapping; }; +struct drm_gem_object { size_t size; void *import_attach; struct file *filp; }; +enum drm_gem_object_status { DRM_GEM_OBJECT_RESIDENT = 1 }; +FUNCTIONS +int main(void) { + struct address_space mapping = {0}; + struct file backing = { .f_mapping = &mapping }; + struct drm_gem_object obj = { .size = 5 * PAGE_SIZE }; + assert(drm_gem_shmem_object_rss(&obj) == 0); + assert(drm_gem_shmem_object_status(&obj) == 0); + obj.filp = &backing; + assert(drm_gem_shmem_object_rss(&obj) == 0); + mapping.nrpages = 2; + assert(drm_gem_shmem_object_rss(&obj) == 2 * PAGE_SIZE); + assert(drm_gem_shmem_object_status(&obj) == DRM_GEM_OBJECT_RESIDENT); + /* A cached backing remains resident independently of a device page-array pin. */ + mapping.nrpages = 5; + assert(drm_gem_shmem_object_rss(&obj) == obj.size); + mapping.nrpages = 9; + assert(drm_gem_shmem_object_rss(&obj) == obj.size); + mapping.nrpages = 0; + assert(drm_gem_shmem_object_rss(&obj) == 0); + assert(drm_gem_shmem_object_status(&obj) == 0); + obj.filp = NULL; + obj.import_attach = &mapping; + assert(drm_gem_shmem_object_rss(&obj) == obj.size); + assert(drm_gem_shmem_object_status(&obj) == DRM_GEM_OBJECT_RESIDENT); + assert(obj.size == 5 * PAGE_SIZE && mapping.nrpages == 0); + puts("PASS unpopulated, cached, partial, bounded, evicted and imported backing"); +} +'''.replace("FUNCTIONS", functions) + source = args.output / "fdinfo.c" + source.write_text(fixture) + results = [] + for shift in [12, 14]: + binary = args.output / f"fdinfo-{shift}" + subprocess.run(["cc", "-std=gnu11", "-Wall", "-Werror", "-g", + "-fsanitize=address,undefined", "-fno-pie", "-no-pie", + f"-DPAGE_SHIFT={shift}", str(source), "-o", str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, check=True) + results.append({"page_shift": shift, "stdout": result.stdout, "stderr": result.stderr}) + record = {"production_sha256": hashlib.sha256(functions.encode()).hexdigest(), + "results": results, + "scope": "Actual C helper bodies with typed file/cache boundary stubs; not kernel ABI or hardware execution."} + (args.output / "result.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/gpu-memory-report-tests.py b/tools/asahi/gpu-memory-report-tests.py new file mode 100644 index 00000000000000..6b8f975dd30adb --- /dev/null +++ b/tools/asahi/gpu-memory-report-tests.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("gpu_memory_report", + Path(__file__).with_name("gpu-memory-report.py")) +reporter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(reporter) + + +class MemoryReportTests(unittest.TestCase): + def test_units_and_unavailable_residency(self): + self.assertEqual(reporter.size_bytes("3 MiB"), 3 * 1024**2) + self.assertEqual(reporter.size_bytes("17"), 17) + self.assertEqual(reporter.size_bytes("64 KiB"), 65536) + with self.assertRaises(ValueError): + reporter.size_bytes("3 bananas") + self.assertIsNone(reporter.parse_fdinfo("pos:\t0\n")) + + def test_shared_descriptors_are_not_recounted_as_allocations(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + node = root / "renderD128" + node.touch() + for pid, fds in [(100, [5, 6]), (200, [8])]: + process = root / str(pid) + (process / "fdinfo").mkdir(parents=True) + (process / "fd").mkdir() + (process / "comm").write_text(f"process-{pid}\n") + for fd in fds: + (process / "fd" / str(fd)).symlink_to(node) + (process / "fdinfo" / str(fd)).write_text( + "drm-driver:\tasahi\ndrm-client-id:\t7\n" + "drm-total-memory:\t10 MiB\ndrm-shared-memory:\t2 MiB\n") + result = reporter.report(root, root / "debug") + self.assertEqual(len(result["clients"]), 1) + client = result["clients"][0] + self.assertEqual(client["fd_count"], 3) + self.assertEqual(len(client["processes"]), 2) + self.assertEqual(result["private_handle_bytes"], 8 * 1024**2) + self.assertIsNone(client["memory_bytes"]["resident"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/asahi/gpu-memory-report.py b/tools/asahi/gpu-memory-report.py new file mode 100644 index 00000000000000..aec96590e527d2 --- /dev/null +++ b/tools/asahi/gpu-memory-report.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Attribute Asahi GEM handles without counting duplicated DRM file descriptors twice.""" +import argparse +import json +from pathlib import Path +import re + + +def size_bytes(value): + match = re.fullmatch(r"(\d+)(?:\s+(B|KiB|MiB|GiB))?", value.strip()) + if not match: + raise ValueError(f"unsupported DRM size: {value!r}") + return int(match[1]) * {None: 1, "B": 1, "KiB": 1024, + "MiB": 1024**2, "GiB": 1024**3}[match[2]] + + +def parse_fdinfo(text): + fields = dict(line.split(":", 1) for line in text.splitlines() if ":" in line) + fields = {key: value.strip() for key, value in fields.items()} + if "drm-driver" not in fields or "drm-client-id" not in fields: + return None + return fields + + +def collect(proc_root, driver="asahi"): + clients = {} + unreadable = 0 + for process in sorted(proc_root.iterdir()): + if not process.name.isdigit(): + continue + try: + name = (process / "comm").read_text().strip() + infos = list((process / "fdinfo").iterdir()) + except (OSError, ValueError): + unreadable += 1 + continue + for info in infos: + try: + fields = parse_fdinfo(info.read_text()) + if not fields or fields["drm-driver"] != driver: + continue + node = (process / "fd" / info.name).resolve(strict=True) + identity = fields.get("drm-pdev", str(node)) + key = (driver, identity, int(fields["drm-client-id"])) + if key not in clients: + memory = { + kind: size_bytes(fields[f"drm-{kind}-memory"]) + if f"drm-{kind}-memory" in fields else None + for kind in ["total", "shared", "resident", "active", "purgeable"] + } + clients[key] = { + "driver": driver, "device": identity, "client_id": key[2], + "name": fields.get("drm-client-name"), "memory_bytes": memory, + "processes": {}, "fd_count": 0, + } + clients[key]["processes"][int(process.name)] = name + clients[key]["fd_count"] += 1 + except (OSError, ValueError): + unreadable += 1 + rows = [] + for client in clients.values(): + client["processes"] = [{"pid": pid, "name": name} + for pid, name in sorted(client["processes"].items())] + rows.append(client) + return rows, unreadable + + +def report(proc_root, debug_root, driver="asahi"): + clients, unreadable = collect(proc_root, driver) + allocation_extents = {} + for generation in ["g17", "m3", "g16"]: + path = debug_root / f"asahi-{generation}" / "memory" + try: + text = path.read_text() + except OSError: + continue + allocation_extents[str(path)] = text + known = [row["memory_bytes"] for row in clients + if row["memory_bytes"]["total"] is not None + and row["memory_bytes"]["shared"] is not None] + return { + "clients": clients, + "clients_with_memory_reporting": len(known), + "private_handle_bytes": sum(row["total"] - row["shared"] for row in known), + "driver_allocation_extents": allocation_extents, + "unreadable_entries": unreadable, + "notes": [ + "Each DRM client is counted once across duplicated and inherited descriptors.", + "Resident/shared values from different clients must not be added as unique physical memory.", + "Private handle bytes exclude kernel allocations and objects retained without client handles.", + "Driver allocation extents are logical sizes, not a resident-page measurement.", + "Missing resident counters mean unavailable, not zero. Root access gives fuller process coverage.", + ], + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proc-root", type=Path, default=Path("/proc")) + parser.add_argument("--debug-root", type=Path, default=Path("/sys/kernel/debug")) + parser.add_argument("--driver", default="asahi") + args = parser.parse_args() + print(json.dumps(report(args.proc_root, args.debug_root, args.driver), indent=2)) From 8bea0247ca6c7d5d85a6c8fb865efbf1e7282e49 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:07:18 -0400 Subject: [PATCH 02/22] drm/asahi: account G17 allocations and compact render state Track GEM allocations across GPU generations and classify kernel backing without counting client aliases again. Expose current and peak extents through a diagnostic memory view; keep imported wrapper accounting explicit. Remove two unused pages from each render generation and the unused register shadow and alias from each render pair. Preserve generation depth, work capacity and embedded descriptor register arrays. The cold Neo comparison saves 528 KiB per pair, or 7.22 MiB with the same fourteen installed pairs. --- drivers/gpu/drm/asahi/agx_memory_stats.rs | 253 ++++++++++++++++++ drivers/gpu/drm/asahi/asahi_core.rs | 1 + drivers/gpu/drm/asahi/g17/buffer.rs | 2 + drivers/gpu/drm/asahi/g17/buffer/target.rs | 1 + drivers/gpu/drm/asahi/g17/context.rs | 1 + drivers/gpu/drm/asahi/g17/freelist.rs | 1 + .../gpu/drm/asahi/g17/fw/render/storage.rs | 15 +- drivers/gpu/drm/asahi/g17/initdata.rs | 12 +- drivers/gpu/drm/asahi/g17/mod.rs | 7 + drivers/gpu/drm/asahi/g17/object.rs | 10 + .../gpu/drm/asahi/g17/queue/render/memory.rs | 54 +--- drivers/gpu/drm/asahi/gem.rs | 19 ++ drivers/gpu/drm/asahi/m3/agx_memory_stats.rs | 73 ----- drivers/gpu/drm/asahi/m3/mod.rs | 1 - tools/asahi/gpu-memory-tests.rs | 27 ++ tools/asahi/neo-auxiliary-tests.py | 3 + 16 files changed, 356 insertions(+), 124 deletions(-) create mode 100644 drivers/gpu/drm/asahi/agx_memory_stats.rs delete mode 100644 drivers/gpu/drm/asahi/m3/agx_memory_stats.rs create mode 100644 tools/asahi/gpu-memory-tests.rs diff --git a/drivers/gpu/drm/asahi/agx_memory_stats.rs b/drivers/gpu/drm/asahi/agx_memory_stats.rs new file mode 100644 index 00000000000000..568157432298cd --- /dev/null +++ b/drivers/gpu/drm/asahi/agx_memory_stats.rs @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded host accounting for driver-owned GEM extents and M3 coherent backing. +//! +//! These are allocation extents, not resident/pinned pages or unique imported +//! physical memory. GEM construction/unwind is included. Counters are per +//! module instance and each read is observational, not a transactional snapshot. +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +struct Counters { + objects: AtomicU64, + bytes: AtomicU64, + peak_objects: AtomicU64, + peak_bytes: AtomicU64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backing_owner_drop_releases_extent_once_and_keeps_peak() { + static COUNTERS: Counters = Counters::new(); + let first = Allocation::new(&COUNTERS, 0x4000).unwrap(); + let second = Allocation::new(&COUNTERS, 0x8000).unwrap(); + assert_eq!(COUNTERS.snapshot(), [2, 0xc000, 2, 0xc000]); + let shared_owner = std::sync::Arc::new(first); + let retained = shared_owner.clone(); + drop(shared_owner); + drop(second); + assert_eq!(COUNTERS.snapshot(), [1, 0x4000, 2, 0xc000]); + drop(retained); + assert_eq!(COUNTERS.snapshot(), [0, 0, 2, 0xc000]); + } + + #[test] + fn refused_extent_does_not_change_live_object_count() { + static COUNTERS: Counters = Counters::new(); + assert!(matches!(Allocation::new(&COUNTERS, 0), Err(EINVAL))); + assert_eq!(COUNTERS.snapshot(), [0; 4]); + COUNTERS.bytes.store(u64::MAX, Ordering::Relaxed); + assert!(matches!(Allocation::new(&COUNTERS, 1), Err(EOVERFLOW))); + assert_eq!(COUNTERS.objects.load(Ordering::Relaxed), 0); + assert_eq!(COUNTERS.bytes.load(Ordering::Relaxed), u64::MAX); + } + + #[test] + fn concurrent_aliases_classify_one_backing_and_release_it_once() { + let allocation = std::sync::Arc::new(Allocation::gem(0x4000, true).unwrap()); + std::thread::scope(|scope| { + for category in [ + KernelCategory::Firmware, + KernelCategory::Descriptor, + KernelCategory::QueueConfig, + KernelCategory::FreeList, + ] { + let alias = allocation.clone(); + scope.spawn(move || alias.classify_kernel(category)); + } + }); + let class_bytes: u64 = KERNEL_CATEGORIES.iter().map(|c| c.snapshot()[1]).sum(); + let class_objects: u64 = KERNEL_CATEGORIES.iter().map(|c| c.snapshot()[0]).sum(); + assert_eq!((class_objects, class_bytes), (1, 0x4000)); + assert_eq!(KERNEL_CATEGORIES[0].snapshot()[0], 0); + let retained = allocation.clone(); + drop(allocation); + assert_eq!(KERNEL_GEM.snapshot()[1], 0x4000); + drop(retained); + assert_eq!(KERNEL_GEM.snapshot()[1], 0); + assert!(KERNEL_CATEGORIES + .iter() + .all(|c| c.snapshot()[0] == 0 && c.snapshot()[1] == 0)); + } +} +impl Counters { + const fn new() -> Self { + Self { + objects: AtomicU64::new(0), + bytes: AtomicU64::new(0), + peak_objects: AtomicU64::new(0), + peak_bytes: AtomicU64::new(0), + } + } + fn snapshot(&self) -> [u64; 4] { + let objects = self.objects.load(Ordering::Relaxed); + let bytes = self.bytes.load(Ordering::Relaxed); + [ + objects, + bytes, + self.peak_objects.load(Ordering::Relaxed).max(objects), + self.peak_bytes.load(Ordering::Relaxed).max(bytes), + ] + } +} +static USER_GEM: Counters = Counters::new(); +static KERNEL_GEM: Counters = Counters::new(); +// Core-created objects may be local dumb buffers or wrappers around imported backing. +static CORE_GEM: Counters = Counters::new(); +static COHERENT: Counters = Counters::new(); + +/// A breakdown of kernel GEM extents, not an additional memory total. +#[derive(Copy, Clone)] +#[repr(u32)] +pub(crate) enum KernelCategory { + Other = 0, + Firmware = 1, + Descriptor = 2, + WorkState = 3, + QueueState = 4, + QueueConfig = 5, + FreeList = 6, + RenderGeneration = 7, + RenderTarget = 8, + TileBuffer = 9, + QueueSupport = 10, + Scratch = 11, + Scheduler = 12, +} +const CATEGORY_NAMES: [&str; 13] = [ + "other", + "firmware", + "descriptor", + "work_state", + "queue_state", + "queue_config", + "free_list", + "render_generation", + "render_target", + "tile_buffer", + "queue_support", + "scratch", + "scheduler", +]; +static KERNEL_CATEGORIES: [Counters; 13] = [const { Counters::new() }; 13]; +const NO_CATEGORY: u32 = u32::MAX; + +/// Drop with the actual backing owner, never with an individual GEM handle or +/// its mapping. Retained fault owners therefore remain included after exit. +pub(crate) struct Allocation { + counters: &'static Counters, + bytes: u64, + kernel_category: AtomicU32, +} +impl Allocation { + fn new(counters: &'static Counters, size: usize) -> Result { + let bytes = size as u64; + if bytes == 0 { + return Err(EINVAL); + } + let previous = counters + .bytes + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(bytes) + }) + .map_err(|_| EOVERFLOW)?; + let objects = counters.objects.fetch_add(1, Ordering::Relaxed) + 1; + counters + .peak_bytes + .fetch_max(previous + bytes, Ordering::Relaxed); + counters.peak_objects.fetch_max(objects, Ordering::Relaxed); + Ok(Self { + counters, + bytes, + kernel_category: AtomicU32::new(NO_CATEGORY), + }) + } + pub(crate) fn gem(size: usize, kernel: bool) -> Result { + let allocation = Self::new(if kernel { &KERNEL_GEM } else { &USER_GEM }, size)?; + if kernel { + Self::add_category(0, allocation.bytes); + allocation.kernel_category.store(0, Ordering::Relaxed); + } + Ok(allocation) + } + pub(crate) fn coherent(size: usize) -> Result { + Self::new(&COHERENT, size) + } + pub(crate) fn core(size: usize) -> Result { + Self::new(&CORE_GEM, size) + } + + fn add_category(index: usize, bytes: u64) { + let counters = &KERNEL_CATEGORIES[index]; + let previous = counters.bytes.fetch_add(bytes, Ordering::Relaxed); + let objects = counters.objects.fetch_add(1, Ordering::Relaxed) + 1; + counters + .peak_bytes + .fetch_max(previous + bytes, Ordering::Relaxed); + counters.peak_objects.fetch_max(objects, Ordering::Relaxed); + } + + /// Refine an unclassified backing once. Aliases do not create another allocation. + pub(crate) fn classify_kernel(&self, category: KernelCategory) { + let index = category as u32; + if index != 0 + && self + .kernel_category + .compare_exchange(0, index, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + Self::add_category(index as usize, self.bytes); + KERNEL_CATEGORIES[0] + .bytes + .fetch_sub(self.bytes, Ordering::Relaxed); + KERNEL_CATEGORIES[0].objects.fetch_sub(1, Ordering::Relaxed); + } + } +} +impl Drop for Allocation { + fn drop(&mut self) { + let category = self.kernel_category.load(Ordering::Relaxed); + if category != NO_CATEGORY { + KERNEL_CATEGORIES[category as usize] + .bytes + .fetch_sub(self.bytes, Ordering::Relaxed); + KERNEL_CATEGORIES[category as usize] + .objects + .fetch_sub(1, Ordering::Relaxed); + } + self.counters.bytes.fetch_sub(self.bytes, Ordering::Relaxed); + self.counters.objects.fetch_sub(1, Ordering::Relaxed); + } +} + +pub(crate) struct View; +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 unit=allocation_extent_bytes")?; + for (name, counters) in [ + ("user_gem", &USER_GEM), + ("kernel_gem", &KERNEL_GEM), + ("core_gem_may_alias_imports", &CORE_GEM), + ("m3_coherent", &COHERENT), + ] { + let [objects, bytes, peak_objects, peak_bytes] = counters.snapshot(); + writeln!( + f, + "{} objects={} bytes={} peak_objects={} peak_bytes={}", + name, objects, bytes, peak_objects, peak_bytes + )?; + } + writeln!(f, "kernel_categories=subset_of_kernel_gem")?; + for (name, counters) in CATEGORY_NAMES.iter().zip(&KERNEL_CATEGORIES) { + let [objects, bytes, peak_objects, peak_bytes] = counters.snapshot(); + writeln!( + f, + "kernel_{} objects={} bytes={} peak_objects={} peak_bytes={}", + name, objects, bytes, peak_objects, peak_bytes + )?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 4dca72fb85dc07..8d32b97da1d24b 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -3,6 +3,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. +mod agx_memory_stats; mod alloc; mod buffer; mod channel; diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea02..c23d4cebb70968 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -379,6 +379,7 @@ impl Tvb { return Err(ERANGE); } let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::TileBuffer); let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; for index in 0..blocks { @@ -430,6 +431,7 @@ impl Tvb { self.extensions.reserve(1, GFP_KERNEL)?; self.addresses.reserve(added, GFP_KERNEL)?; let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::TileBuffer); let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; for relative in 0..added { diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c935..99142c3dd6bd83 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -114,6 +114,7 @@ struct Object { impl Object { fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize) -> Result { let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::RenderTarget); let prot = mmu::PROT_GPU_FW_SHARED_RW; let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; Ok(Self { diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44beef..d4e9eaa16a2d1d 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -142,6 +142,7 @@ impl Context { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + scheduler.classify_memory(crate::agx_memory_stats::KernelCategory::Scheduler); scheduler.initialize::(0, |value| { value.init(); Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc0..3ec0d50033e793 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -82,6 +82,7 @@ impl Pages { mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs index e1e6a10949fb44..a53a7d993f822f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -9,11 +9,18 @@ use kernel::prelude::*; pub(crate) const SUPPORT_SIZE: usize = 0x40000; pub(crate) const EVENT_SLOTS: usize = 0x4000; pub(crate) const EVENT_COUNT: usize = 35; -pub(crate) const STATE_SIZE: usize = 0x18000; -pub(crate) const DEFLAKE: usize = 0x4000; -pub(crate) const TA_STATUS: usize = 0x8000; -pub(crate) const AUXILIARY: usize = 0x10000; +// Host-private packing: consumers receive separate GPU pointers to these ranges. +// Keep every accessed range and all generation slots, without unused backing pages. +pub(crate) const STATE_SIZE: usize = 0x10000; +pub(crate) const DEFLAKE: usize = 0; +pub(crate) const TA_STATUS: usize = 0x4000; +pub(crate) const AUXILIARY: usize = 0x8000; pub(crate) const AUXILIARY_SIZE: usize = 0x8000; +const _: () = { + assert!(DEFLAKE + 0x4000 <= TA_STATUS); + assert!(TA_STATUS + 0x4000 <= AUXILIARY); + assert!(AUXILIARY + AUXILIARY_SIZE == STATE_SIZE); +}; #[repr(C)] pub(crate) struct Event { diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09eb..f53ef73e10bd73 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -64,12 +64,14 @@ impl ComputeShared { KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; let runs = KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; + pages.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); + runs.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_MAPPED, GFP_KERNEL)?; for _ in 0..COMPUTE_BLOCKS_MAPPED { - blocks.push( - KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, - GFP_KERNEL, - )?; + let block = KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, + CpuMap::WriteCombined)?; + block.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); + blocks.push(block, GFP_KERNEL)?; } let id = u64::from(super::ids::OBJECTS.object()?); Ok(Self { @@ -95,6 +97,7 @@ impl ComputeScheduler { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + object.classify_memory(crate::agx_memory_stats::KernelCategory::Scheduler); object.initialize::(0, |state| { state.init(); Ok(()) @@ -675,6 +678,7 @@ impl InitData { } fn add(&mut self, object: KernelObject) -> Result { + object.classify_memory(crate::agx_memory_stats::KernelCategory::Firmware); let index = self.objects.len(); self.objects.push(object, GFP_KERNEL)?; Ok(index) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6ca0398abd2e27..937eb4fa5a5959 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -824,6 +824,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _memory: Pin>>, } impl Gpu { @@ -882,6 +883,11 @@ impl Gpu { GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let memory = KBox::pin_init( + directory.read_only_file(c"memory", crate::agx_memory_stats::View), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -890,6 +896,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + _memory: memory, }, GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae4..664e513ac4c2cb 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -66,6 +66,9 @@ unsafe impl Send for KernelObject {} unsafe impl Sync for KernelObject {} impl KernelObject { + pub(crate) fn classify_memory(&self, category: crate::agx_memory_stats::KernelCategory) { + self.obj.gem.classify_memory(category); + } /// Allocates a zeroed object of `size` bytes and maps it into `vm`. pub(crate) fn new( dev: &AsahiDrmDevice, @@ -478,6 +481,13 @@ impl Pool { }); } let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; + use crate::agx_memory_stats::KernelCategory; + object.classify_memory(match family { + Family::Descriptor => KernelCategory::Descriptor, + Family::WorkState => KernelCategory::WorkState, + Family::QueueState => KernelCategory::QueueState, + Family::QueueConfig => KernelCategory::QueueConfig, + }); let reserved = { let mut inner = self.inner.lock(); if inner.stopped { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 85319585b3d135..ba3d4b273cc830 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -13,6 +13,7 @@ use crate::g17::{ }, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; +use crate::agx_memory_stats::KernelCategory; use crate::{ hw::t8140::{dynamic, kernel_window, low, queues, resources as cfg}, mmu, @@ -140,6 +141,7 @@ impl Graph { prot, CpuMap::WriteCombined, )?; + support.classify_memory(KernelCategory::QueueSupport); let address = queues.gpu_va(); for stage in 0..2 { queues.write( @@ -241,6 +243,7 @@ struct Compact { impl Compact { fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + backing.classify_memory(KernelCategory::Scratch); let mapping = buffer::compact_alias( &backing, vm, @@ -266,14 +269,16 @@ pub(super) struct Generation { } impl Generation { fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { - KernelObject::new( + let state = KernelObject::new( alloc.dev, vm, crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, cfg::TVB_PAGE), storage::STATE_SIZE, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, - ) + )?; + state.classify_memory(KernelCategory::RenderGeneration); + Ok(state) } fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { let deflake = buffer::compact_alias( @@ -290,6 +295,7 @@ impl Generation { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + fragment.classify_memory(KernelCategory::RenderGeneration); let fragment_client = buffer::compact_alias( &fragment, vm, @@ -347,7 +353,6 @@ impl Generation { } pub(super) struct Memory { - _fragment_view: mmu::KernelMapping, pub(super) graph: Graph, scratch: Compact, discard: Compact, @@ -357,9 +362,6 @@ pub(super) struct Memory { pub(super) timestamps: KernelObject, pub(super) manager: buffer::Manager, pub(super) lifecycle: [fw::ObjectIds; 2], - // The extra client views retain their allocation positions even though - // kick entries consume the register arrays embedded in the descriptors. - registers: KernelObject, pub(super) context: Arc, pub(super) installed_pool: Option, } @@ -430,22 +432,6 @@ impl Memory { word, ) })?; - let registers = KernelObject::new( - alloc.dev, - vm, - crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), - mmu::UAT_PGSZ, - mmu::PROT_GPU_SHARED_RO, - CpuMap::WriteCombined, - )?; - let fragment_view = graph.descriptors.map_range( - vm, - TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, - dynamic::CLIENT_LOWER, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - )?; Ok(( storage.write_init(kernel::try_init!(Self { graph, @@ -457,8 +443,6 @@ impl Memory { timestamps, manager, lifecycle: [tiling, fragment], - _fragment_view: fragment_view, - registers: registers, context, installed_pool: None, }))?, @@ -542,21 +526,9 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick records use the arrays embedded in this descriptor directly. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, @@ -609,15 +581,13 @@ impl Memory { ) -> Result> { if self.scratch.mapping.is_none() || self.discard.mapping.is_none() - || self.registers.gpu_va() == 0 { return Err(EIO); } let mut aliases = KVec::new(); - aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.reserve(tvb.len().checked_add(2).ok_or(EOVERFLOW)?, GFP_KERNEL)?; aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; for mapping in core::mem::take(tvb) { aliases.push(mapping, GFP_KERNEL)?; } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 2266d18e9c85f2..911246738abc80 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -58,6 +58,7 @@ pub(crate) struct AsahiObject { exportable: bool, /// Whether this is a kernel-created object. kernel: bool, + _allocation: Option, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, } @@ -65,6 +66,14 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +impl AsahiObject { + pub(crate) fn classify_memory(&self, category: crate::agx_memory_stats::KernelCategory) { + if let Some(allocation) = &self._allocation { + allocation.classify_kernel(category); + } + } +} + /// Marks whether the pages of `gem` are held for the device. /// /// Populated asahi objects keep their pages referenced (device SG table, kernel vmap or a PFN @@ -211,6 +220,8 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + /// Distinguishes direct driver allocation from DRM-core imported or dumb objects. + driver_created: bool, } /// Create a new kernel-owned GEM object. @@ -250,6 +261,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, + driver_created: true, }, )?; @@ -283,6 +295,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + driver_created: true, }, )?; @@ -303,6 +316,11 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, + _allocation: if args.driver_created { + Some(crate::agx_memory_stats::Allocation::gem(size, args.kernel)?) + } else if size != 0 { + Some(crate::agx_memory_stats::Allocation::core(size)?) + } else { None }, size, flags: args.flags, exportable: args.exportable, @@ -318,6 +336,7 @@ impl DriverObject for AsahiObject { flags: 0, exportable: true, kernel: false, + driver_created: false, }) } diff --git a/drivers/gpu/drm/asahi/m3/agx_memory_stats.rs b/drivers/gpu/drm/asahi/m3/agx_memory_stats.rs deleted file mode 100644 index 1df3eb309dcede..00000000000000 --- a/drivers/gpu/drm/asahi/m3/agx_memory_stats.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Bounded host accounting for GEM extents and M3 coherent firmware backing. -//! -//! These are allocation extents, not resident/pinned pages or unique imported -//! physical memory. GEM construction/unwind is included. Counters are per -//! module instance and each read is observational, not a transactional snapshot. -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::prelude::*; - -struct Counters { - objects: AtomicU64, - bytes: AtomicU64, - peak_objects: AtomicU64, - peak_bytes: AtomicU64, -} -impl Counters { - const fn new() -> Self { - Self { objects: AtomicU64::new(0), bytes: AtomicU64::new(0), - peak_objects: AtomicU64::new(0), peak_bytes: AtomicU64::new(0) } - } - fn snapshot(&self) -> [u64; 4] { - let objects = self.objects.load(Ordering::Relaxed); - let bytes = self.bytes.load(Ordering::Relaxed); - [objects, bytes, self.peak_objects.load(Ordering::Relaxed).max(objects), - self.peak_bytes.load(Ordering::Relaxed).max(bytes)] - } -} -static USER_GEM: Counters = Counters::new(); -static KERNEL_GEM: Counters = Counters::new(); -static COHERENT: Counters = Counters::new(); - -/// Drop with the actual backing owner, never with an individual GEM handle or -/// its mapping. Retained fault owners therefore remain included after exit. -pub(crate) struct Allocation { - counters: &'static Counters, - bytes: u64, -} -impl Allocation { - fn new(counters: &'static Counters, size: usize) -> Result { - let bytes = size as u64; - if bytes == 0 { return Err(EINVAL); } - let previous = counters.bytes.fetch_update(Ordering::Relaxed, Ordering::Relaxed, - |current| current.checked_add(bytes)).map_err(|_| EOVERFLOW)?; - let objects = counters.objects.fetch_add(1, Ordering::Relaxed) + 1; - counters.peak_bytes.fetch_max(previous + bytes, Ordering::Relaxed); - counters.peak_objects.fetch_max(objects, Ordering::Relaxed); - Ok(Self { counters, bytes }) - } - pub(crate) fn gem(size: usize, kernel: bool) -> Result { - Self::new(if kernel { &KERNEL_GEM } else { &USER_GEM }, size) - } - pub(crate) fn coherent(size: usize) -> Result { Self::new(&COHERENT, size) } -} -impl Drop for Allocation { - fn drop(&mut self) { - self.counters.bytes.fetch_sub(self.bytes, Ordering::Relaxed); - self.counters.objects.fetch_sub(1, Ordering::Relaxed); - } -} - -pub(crate) struct View; -impl kernel::debugfs::Writer for View { - fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "version=1 unit=allocation_extent_bytes")?; - for (name, counters) in [("user_gem", &USER_GEM), ("kernel_gem", &KERNEL_GEM), - ("m3_coherent", &COHERENT)] { - let [objects, bytes, peak_objects, peak_bytes] = counters.snapshot(); - writeln!(f, "{} objects={} bytes={} peak_objects={} peak_bytes={}", - name, objects, bytes, peak_objects, peak_bytes)?; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 3d900a80fef382..4c56318d504f4c 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -77,7 +77,6 @@ mod m3_thermal_policy; mod agx_host_progress; -mod agx_memory_stats; mod agx_timing_stats; diff --git a/tools/asahi/gpu-memory-tests.rs b/tools/asahi/gpu-memory-tests.rs new file mode 100644 index 00000000000000..7aeffeca1166b4 --- /dev/null +++ b/tools/asahi/gpu-memory-tests.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Compile and exercise the production allocation lifetime counters on the host. + +extern crate self as kernel; + +pub mod prelude { + #[derive(Debug, PartialEq, Eq)] + pub enum Error { + Invalid, + Overflow, + } + pub const EINVAL: Error = Error::Invalid; + pub const EOVERFLOW: Error = Error::Overflow; + pub type Result = std::result::Result; +} +pub mod fmt { + pub use std::fmt::{Formatter, Result}; +} +pub mod debugfs { + pub trait Writer { + fn write(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result; + } +} + +#[allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/agx_memory_stats.rs"] +mod memory; diff --git a/tools/asahi/neo-auxiliary-tests.py b/tools/asahi/neo-auxiliary-tests.py index 68236fe19d1927..ab6273432722cc 100644 --- a/tools/asahi/neo-auxiliary-tests.py +++ b/tools/asahi/neo-auxiliary-tests.py @@ -27,10 +27,13 @@ impl KernelMapping{pub fn iova(&self)->u64{self.va}} } mod storage{CONSTANTS} +mod agx_memory_stats{pub enum KernelCategory{RenderGeneration}} +use agx_memory_stats::KernelCategory; mod cfg{pub const AUXILIARY:std::ops::Range=0x100000..0x500000;} enum CpuMap{WriteCombined} struct KernelObject{va:u64,size:usize} impl KernelObject{ + fn classify_memory(&self,_c:KernelCategory){} fn gpu_va(&self)->u64{self.va} fn map_range(&self,_v:&mmu::Vm,_r:Range,_a:Range,_align:u64,_p:u32,_g:u32)->Result{Err(-28)} } From f1d9e5cccaacadd72057c6df146f6ac388d61667 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:08:43 -0400 Subject: [PATCH 03/22] drm/asahi: export validated G17 firmware GPU utilization Decode the four utilization percentages independently and use the aggregate reported by firmware. Order power witnesses in the firmware clock domain, clear load after power-off, and require fresh utilization after wake. Mark device loss and statistics-stream failure unavailable rather than retaining a plausible busy value. Expose a read-only per-device snapshot and document its validity and units. Keep completed power residency separate from execution time. Full raw payload capture is enabled only by an explicit diagnostic read and expires after two seconds, preserving the normal report-consumption path. Add decoder, snapshot-concurrency and capture-expiry controls, plus a load reader which tolerates a transient snapshot collision. On Neo the default path sustains the earlier compute throughput; explicit raw capture costs about three percent in the matched comparison. --- Documentation/ABI/testing/sysfs-driver-asahi | 54 +++++ .../ABI/testing/sysfs-driver-asahi-agx-stats | 3 + drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/asahi_telemetry.c | 100 +++++++++ drivers/gpu/drm/asahi/driver.rs | 18 +- drivers/gpu/drm/asahi/g17/channel.rs | 57 ++++- drivers/gpu/drm/asahi/g17/load.rs | 195 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 56 +++++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 +- drivers/gpu/drm/asahi/g17/statistics.rs | 128 ++++++++++++ drivers/gpu/drm/asahi/g17/telemetry.rs | 108 ++++++++++ tools/asahi/g17-load-tests.rs | 185 +++++++++++++++++ tools/asahi/g17-statistics-tests.rs | 85 ++++++++ .../asahi/g17-telemetry-registration-tests.py | 132 ++++++++++++ tools/asahi/gpu-load-tests.py | 32 +++ tools/asahi/gpu-load.py | 55 +++++ 16 files changed, 1193 insertions(+), 20 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-driver-asahi create mode 100644 drivers/gpu/drm/asahi/asahi_telemetry.c create mode 100644 drivers/gpu/drm/asahi/g17/load.rs create mode 100644 drivers/gpu/drm/asahi/g17/statistics.rs create mode 100644 drivers/gpu/drm/asahi/g17/telemetry.rs create mode 100644 tools/asahi/g17-load-tests.rs create mode 100644 tools/asahi/g17-statistics-tests.rs create mode 100644 tools/asahi/g17-telemetry-registration-tests.py create mode 100644 tools/asahi/gpu-load-tests.py create mode 100755 tools/asahi/gpu-load.py diff --git a/Documentation/ABI/testing/sysfs-driver-asahi b/Documentation/ABI/testing/sysfs-driver-asahi new file mode 100644 index 00000000000000..0c6442c88667a7 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-asahi @@ -0,0 +1,54 @@ +What: /sys/devices/platform/.../*.gpu/agx_stats +Date: October 2026 +Contact: Aurora Silicon developers +Description: + Read-only, per-device firmware GPU utilization on T8140 G17P. + With asahi.stats_export=0, the file reports unsupported. + The default value 1 enables the version 2 export described here. + Earlier GPU generations use the separate legacy agx_stats format. + Version 2 is a whitespace-delimited key/value format. Readers + must check the version and load_valid before using percentages. + + version: 2 + source: firmware + load_valid: 1 if the current utilization is available, else 0 + busy_percent: firmware aggregate utilization, percent (0..100) + util1..util3: firmware engine percentages (0..100); engine labels + are not assigned without independent validation + util4: same aggregate as busy_percent; do not sum engine values + gpu_power_known: whether a timestamped power state was received + gpu_powered: 0 when firmware reports the GPU powered off, else 1 + samples: accepted primary utilization reports since probe + sample_received_ns: host CLOCK_MONOTONIC receive timestamp + sample_age_ns: time since the retained utilization report + sample_timestamp_ticks: 64-bit firmware utilization timestamp + power_timestamp_ticks: newest secondary power-state timestamp + clock_hz: 24000000 (timestamp and residency ticks) + powered_ticks: completed powered intervals, accumulated from + firmware power-off records since probe + unpowered_ticks: completed unpowered intervals, accumulated from + firmware power-on records since probe + invalid_samples: malformed utilization/power records rejected + + A timestamped power-off witness overrides retained utilization + with zero. After wake, an old utilization report is unavailable + until a report for the new interval arrives. Device loss or + shutdown invalidates load until a new device is probed. Receive timestamps + do not order events across firmware rings: firmware timestamps + determine which state is newer. Suspend can separate the host + and firmware clocks; do not subtract values from those domains. + + Power residency is not GPU busy time. Residency counters omit + the currently open power interval and saturate at U64_MAX. + Neither these counters nor the percentages are per-client DRM + engine execution time. No per-client execution counter is + synthesized from queue admission, wall time or job spans. + + Reading debugfs asahi-g17/statistics requests two seconds of full + raw-payload capture, including undecoded record kinds. Repeat + reads to keep diagnostic capture active. It is disabled by + default and is not needed for normal utilization reporting. + Trace records and uncaptured statistics only read required words. + Captured counts exclude records outside the capture window. That diagnostic format is not a + stable userspace ABI. Optional telemetry registration failure + does not prevent GPU initialization and is logged by the driver. diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 11a964e6f17d81..1db3281eb64c4a 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -2,6 +2,9 @@ What: /sys/class/drm/card*/device/agx_stats Date: October 2026 Contact: Asahi Linux GPU driver maintainers Description: + Legacy GPU format. For T8140 G17P, see sysfs-driver-asahi + for the version 2 format and power-state validity rules. + Read-only dump of the AGX firmware statistics decoded by the drm/asahi driver. The driver already polls the firmware statistics ring (see `drivers/gpu/drm/asahi/channel.rs::StatsChannel::poll`, diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index a3aff32eea1608..7d8eb6c4376f0d 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o +asahi-y := asahi_core.o sysfs.o asahi_telemetry.o diff --git a/drivers/gpu/drm/asahi/asahi_telemetry.c b/drivers/gpu/drm/asahi/asahi_telemetry.c new file mode 100644 index 00000000000000..bf46d02ddb678c --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi_telemetry.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* A device attribute with an owned Rust snapshot callback. */ +#include +#include +#include +#include +#include +#include + +struct asahi_load_export { + u32 valid, power_known, powered, utilization[4], clock_hz; + u64 samples, received_ns, timestamp_ticks, power_timestamp_ticks; + u64 powered_ticks, unpowered_ticks, invalid, age_ns; +}; + +typedef int (*asahi_snapshot_fn)(const void *, void *); +struct asahi_telemetry_attribute { + struct device_attribute attr; + const void *data; + asahi_snapshot_fn snapshot; +}; + +static ssize_t agx_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_telemetry_attribute *stats = + container_of(attr, struct asahi_telemetry_attribute, attr); + struct asahi_load_export value; + int ret = stats->snapshot(stats->data, &value); + + if (ret) + return ret; + return sysfs_emit(buf, + "version 2\nsource firmware\nload_valid %u\n" + "gpu_power_known %u\ngpu_powered %u\n" + "util1 %u\nutil2 %u\nutil3 %u\nutil4 %u\nbusy_percent %u\n" + "samples %llu\nsample_received_ns %llu\nsample_age_ns %llu\n" + "sample_timestamp_ticks %llu\npower_timestamp_ticks %llu\n" + "clock_hz %u\npowered_ticks %llu\nunpowered_ticks %llu\n" + "invalid_samples %llu\n", + value.valid, value.power_known, value.powered, + value.utilization[0], value.utilization[1], value.utilization[2], + value.utilization[3], value.utilization[3], + (unsigned long long)value.samples, + (unsigned long long)value.received_ns, + (unsigned long long)value.age_ns, + (unsigned long long)value.timestamp_ticks, + (unsigned long long)value.power_timestamp_ticks, + value.clock_hz, + (unsigned long long)value.powered_ticks, + (unsigned long long)value.unpowered_ticks, + (unsigned long long)value.invalid); +} + +int asahi_telemetry_register(struct device *dev, const void *data, + asahi_snapshot_fn snapshot, void **handle); +void asahi_telemetry_unregister(struct device *dev, void *handle); + +int asahi_telemetry_register(struct device *dev, const void *data, + asahi_snapshot_fn snapshot, void **handle) +{ + struct asahi_telemetry_attribute *stats; + int ret; + + if (!dev || !data || !snapshot || !handle) + return -EINVAL; + BUILD_BUG_ON(sizeof(struct asahi_load_export) != 96); + BUILD_BUG_ON(offsetof(struct asahi_load_export, samples) != 32); + stats = kzalloc(sizeof(*stats), GFP_KERNEL); + if (!stats) + return -ENOMEM; + sysfs_attr_init(&stats->attr.attr); + stats->attr.attr.name = "agx_stats"; + stats->attr.attr.mode = 0444; + stats->attr.show = agx_stats_show; + stats->data = data; + stats->snapshot = snapshot; + ret = device_create_file(dev, &stats->attr); + if (ret) { + kfree(stats); + return ret; + } + *handle = stats; + return 0; +} +EXPORT_SYMBOL_GPL(asahi_telemetry_register); + +void asahi_telemetry_unregister(struct device *dev, void *handle) +{ + struct asahi_telemetry_attribute *stats = handle; + + if (!stats) + return; + device_remove_file(dev, &stats->attr); + kfree(stats); +} +EXPORT_SYMBOL_GPL(asahi_telemetry_unregister); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Asahi firmware telemetry sysfs registration"); diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index df45f4a8ca91d1..3dddb7fb5e62ae 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -5,7 +5,6 @@ use crate::module_parameters; use kernel::bindings; use kernel::{ - c_str, device::{ Core, DeviceContext, // @@ -333,14 +332,19 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + // Enabled G17 telemetry owns agx_stats through Gpu's registration. + // Only the disabled-export placeholder belongs to the platform wrapper. let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { - Ok(handle) => handle, - Err(error) => { - gpu.shutdown(); - return Err(error); + let raw_stats = if *module_parameters::stats_export.value() == 0 { + match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + Ok(handle) => handle, + Err(error) => { + gpu.shutdown(); + return Err(error); + } } + } else { + core::ptr::null_mut() }; Ok(AsahiPlatformDriverData { _device: reg.device().into(), diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a1661..1468bc7b3f46dd 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -56,20 +56,26 @@ impl Rings { } pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { - self.drain_report(dev, init, role, ReportRing::Trace); + self.drain_report(dev, init, role, ReportRing::Trace, None); } - pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role) { - self.drain_report(dev, init, role, ReportRing::Statistics); + pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role, + statistics: &super::statistics::Statistics) { + self.drain_report(dev, init, role, ReportRing::Statistics, Some(statistics)); } - fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { + fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing, + statistics: Option<&super::statistics::Statistics>) { let quarantined = &mut self.quarantined[role as usize][ring as usize]; if *quarantined { return; } - if let Err(error) = Self::consume_report(init, role, ring) { + if let Err(error) = Self::consume_report(init, role, ring, statistics) { *quarantined = true; + if let Some(statistics) = statistics { + // Without both report streams, retained load is no longer current. + statistics.load.invalidate(); + } dev_err!( dev.as_ref(), "G17 {:?} {:?} ring stopped: {:?}\n", @@ -80,7 +86,8 @@ impl Rings { } } - fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { + fn consume_report(init: &InitData, role: Role, ring: ReportRing, + statistics: Option<&super::statistics::Statistics>) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; @@ -94,14 +101,42 @@ impl Rings { } while consumer != producer { let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - let tag = match ring { - // Copy the entire trace entry before returning its slot. - ReportRing::Trace => init.status_read_words::(role, offset)?[0], - ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], - }; + let tag = init.status_word(role, offset)?.load(Ordering::Relaxed); if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { return Err(EIO); } + if let Some(statistics) = statistics { + const { assert!(super::statistics::WORDS == EVENT_WORDS) }; + let pending = statistics.capture_pending(); + let now_ns = if pending || (role == Role::Primary && tag == 0x13) { + super::now_ns() + } else { 0 }; + if pending && statistics.capture_enabled(now_ns) { + // Diagnostic capture retains every word before returning the ring slot. + let words = init.status_read_words::(role, offset)?; + if words[0] != tag { return Err(EIO); } + statistics.record(role as usize, now_ns, &words); + } else { + // The production decoder reads only fields it consumes. Trace records + // and unsupported statistics need no payload copy or receive timestamp. + let mut words = [0; EVENT_WORDS]; + words[0] = tag; + match (role, tag) { + (Role::Primary, 0x13) => { + words[1..7].copy_from_slice(&init.status_read_words::<6>(role, offset + 4)?); + } + (Role::Secondary, 0x19) => { + words[1..3].copy_from_slice(&init.status_read_words::<2>(role, offset + 4)?); + words[6] = init.status_word(role, offset + 24)?.load(Ordering::Relaxed); + } + (Role::Secondary, 0x11 | 0x12) => { + words[1..3].copy_from_slice(&init.status_read_words::<2>(role, offset + 4)?); + } + _ => {} + } + statistics.load.record(role as usize, now_ns, &words); + } + } consumer = (consumer + 1) & (capacity - 1); fence(Ordering::SeqCst); consumer_word.store(consumer, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/load.rs b/drivers/gpu/drm/asahi/g17/load.rs new file mode 100644 index 00000000000000..9ed2f3a1312369 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/load.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide firmware utilization. This is not per-client execution time. +//! +//! The primary utilization record contains four independently reported percentages; +//! the aggregate is not the sum of the engine values. Power state comes from the +//! secondary's timestamped power record, never its coprocessor busy flag. Only +//! explicitly written fields are read: unused record tails retain old ring contents. + +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; + +pub(crate) const CLOCK_HZ: u64 = 24_000_000; +const UTILIZATION: u32 = 0x13; +const POWER_STATE: u32 = 0x19; +const POWER_ON: u32 = 0x11; +const POWER_OFF: u32 = 0x12; + +pub(crate) struct Load { + invalidated: AtomicBool, + sequence: AtomicU64, + samples: AtomicU64, + received_ns: AtomicU64, + ticks: AtomicU64, + utilization: [AtomicU32; 4], + power_known: AtomicU32, + powered: AtomicU32, + power_ticks: AtomicU64, + on_ticks: AtomicU64, + powered_ticks: AtomicU64, + unpowered_ticks: AtomicU64, + invalid: AtomicU64, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Report { + pub(crate) invalidated: bool, + pub(crate) samples: u64, + pub(crate) received_ns: u64, + pub(crate) ticks: u64, + pub(crate) utilization: [u32; 4], + pub(crate) power_known: bool, + pub(crate) powered: bool, + pub(crate) power_ticks: u64, + pub(crate) on_ticks: u64, + pub(crate) powered_ticks: u64, + pub(crate) unpowered_ticks: u64, + pub(crate) invalid: u64, +} + +impl Report { + /// An off witness at or after the sample overrides its retained percentages. + /// After wake, an old sample is unavailable until the new interval is sampled. + pub(crate) fn current(&self) -> Option<[u32; 4]> { + if self.invalidated { + return None; + } + if self.power_known && !self.powered && self.power_ticks >= self.ticks { + return Some([0; 4]); + } + if self.samples == 0 || self.ticks < self.on_ticks { + return None; + } + Some(self.utilization) + } +} + +fn ticks(words: &[u32; 18]) -> u64 { + u64::from(words[1]) | (u64::from(words[2]) << 32) +} + +impl Load { + pub(crate) const fn new() -> Self { + Self { + invalidated: AtomicBool::new(false), + sequence: AtomicU64::new(0), + samples: AtomicU64::new(0), + received_ns: AtomicU64::new(0), + ticks: AtomicU64::new(0), + utilization: [const { AtomicU32::new(0) }; 4], + power_known: AtomicU32::new(0), + powered: AtomicU32::new(0), + power_ticks: AtomicU64::new(0), + on_ticks: AtomicU64::new(0), + powered_ticks: AtomicU64::new(0), + unpowered_ticks: AtomicU64::new(0), + invalid: AtomicU64::new(0), + } + } + + /// The firmware receive worker is the sole writer, serialized by the device lock. + pub(crate) fn record(&self, role: usize, now_ns: u64, words: &[u32; 18]) { + let tag = words[0]; + if !matches!( + (role, tag), + (0, UTILIZATION) | (1, POWER_STATE | POWER_ON | POWER_OFF) + ) { + return; + } + if role == 1 && tag == POWER_STATE && words[6] <= 1 { + let timestamp = ticks(words); + let powered = u32::from(words[6] == 0); + if timestamp != 0 + && self.power_known.load(Ordering::Relaxed) != 0 + && self.powered.load(Ordering::Relaxed) == powered + { + // The state and wake epoch are unchanged. Only this atomic timestamp + // advances; a concurrent transition still changes the sequence below. + // Avoid rewriting the complete snapshot for every repeated report. + if timestamp > self.power_ticks.load(Ordering::Relaxed) { + self.power_ticks.store(timestamp, Ordering::Relaxed); + } + return; + } + } + self.sequence.fetch_add(1, Ordering::AcqRel); + match (role, tag) { + (0, UTILIZATION) => { + let timestamp = ticks(words); + if timestamp == 0 || words[3..7].iter().any(|&percent| percent > 100) { + self.invalid.fetch_add(1, Ordering::Relaxed); + } else if timestamp > self.ticks.load(Ordering::Relaxed) { + for (out, value) in self.utilization.iter().zip(&words[3..7]) { + out.store(*value, Ordering::Relaxed); + } + self.received_ns.store(now_ns, Ordering::Relaxed); + self.ticks.store(timestamp, Ordering::Relaxed); + self.samples.fetch_add(1, Ordering::Relaxed); + } + } + (1, POWER_STATE) => { + let timestamp = ticks(words); + if timestamp == 0 || words[6] > 1 { + self.invalid.fetch_add(1, Ordering::Relaxed); + } else if timestamp > self.power_ticks.load(Ordering::Relaxed) { + let powered = u32::from(words[6] == 0); + if powered != 0 + && (self.power_known.load(Ordering::Relaxed) == 0 + || self.powered.load(Ordering::Relaxed) == 0) + { + self.on_ticks.store(timestamp, Ordering::Relaxed); + } + self.powered.store(powered, Ordering::Relaxed); + self.power_known.store(1, Ordering::Relaxed); + self.power_ticks.store(timestamp, Ordering::Relaxed); + } + } + (1, POWER_ON | POWER_OFF) => { + let counter = if tag == POWER_OFF { + &self.powered_ticks + } else { + &self.unpowered_ticks + }; + counter.store( + counter.load(Ordering::Relaxed).saturating_add(ticks(words)), + Ordering::Relaxed, + ); + } + _ => {} + } + self.sequence.fetch_add(1, Ordering::Release); + } + + /// Fatal device loss and shutdown cannot leave the last busy value current. + pub(crate) fn invalidate(&self) { + self.invalidated.store(true, Ordering::Release); + } + + pub(crate) fn snapshot(&self) -> Option { + for _ in 0..8 { + let sequence = self.sequence.load(Ordering::Acquire); + if sequence & 1 != 0 { + continue; + } + let report = Report { + invalidated: self.invalidated.load(Ordering::Acquire), + samples: self.samples.load(Ordering::Relaxed), + received_ns: self.received_ns.load(Ordering::Relaxed), + ticks: self.ticks.load(Ordering::Relaxed), + utilization: core::array::from_fn(|i| self.utilization[i].load(Ordering::Relaxed)), + power_known: self.power_known.load(Ordering::Relaxed) != 0, + powered: self.powered.load(Ordering::Relaxed) != 0, + power_ticks: self.power_ticks.load(Ordering::Relaxed), + on_ticks: self.on_ticks.load(Ordering::Relaxed), + powered_ticks: self.powered_ticks.load(Ordering::Relaxed), + unpowered_ticks: self.unpowered_ticks.load(Ordering::Relaxed), + invalid: self.invalid.load(Ordering::Relaxed), + }; + fence(Ordering::Acquire); + if sequence == self.sequence.load(Ordering::Relaxed) { + return Some(report); + } + } + None + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 937eb4fa5a5959..206912ae3a6568 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -42,6 +42,9 @@ mod queue; mod recovery; mod runtime; pub(crate) mod status; +mod statistics; +mod load; +mod telemetry; mod teardown; mod timeout; mod validation; @@ -611,6 +614,7 @@ struct Shared { descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, + statistics: Arc, #[pin] feed_work: Work, preparations: Arc, @@ -791,6 +795,7 @@ impl Shared { /// Marks the device lost without a crash log. fn lose_device_quietly(self: &Arc) { + self.statistics.load.invalidate(); self.preparations.fail(); if !self.crashed.swap(true, Ordering::AcqRel) { dev_err!(self.dev.as_ref(), "GPU runtime stopped, device lost\n"); @@ -802,6 +807,7 @@ impl Shared { /// /// The firmware cannot be restarted: outstanding and future work fails. fn lose_device(self: &Arc, role: Role, crashlog: Option<&[u8]>) { + self.statistics.load.invalidate(); self.preparations.fail(); self.crashed.store(true, Ordering::Release); dev_err!( @@ -825,6 +831,35 @@ pub(crate) struct Gpu { ids: gpu::SequenceIDs, shared: Arc, _memory: Pin>>, + _statistics: Pin>>, + _telemetry: Option, +} + +struct StatisticsView(Arc); + +impl kernel::debugfs::Writer for StatisticsView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.0.request_capture(now_ns()); + writeln!(f, "version=2 payload=raw_firmware_words received_clock=monotonic_ns capture_window_ns={}", statistics::CAPTURE_NS)?; + for role in 0..2 { + for tag in 16..=30 { + let Some(report) = self.0.snapshot(role, tag) else { + writeln!(f, "role={} tag={:#x} busy=1", role, tag)?; + continue; + }; + if report.count == 0 { + continue; + } + write!(f, "role={} tag={:#x} count={} received_ns={} words=", + role, tag, report.count, report.received_ns)?; + for word in report.words { + write!(f, "{:08x} ", word)?; + } + writeln!(f)?; + } + } + Ok(()) + } } impl Gpu { @@ -858,6 +893,19 @@ impl Gpu { let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; + let statistics = Arc::new(statistics::Statistics::new(), GFP_KERNEL)?; + let report_statistics = statistics.clone(); + let telemetry = if *crate::module_parameters::stats_export.value() == 0 { + None + } else { + match telemetry::Telemetry::new(pdev.as_ref(), statistics.clone()) { + Ok(telemetry) => Some(telemetry), + Err(error) => { + dev_warn!(pdev.as_ref(), "GPU telemetry registration failed: {:?}\n", error); + None + } + } + }; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let shared = Arc::pin_init( pin_init!(Shared { @@ -868,6 +916,7 @@ impl Gpu { descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, + statistics, feed_work <- new_work!("g17::Shared::feed_work"), preparations, work <- new_work!("g17::Shared::work"), @@ -888,6 +937,10 @@ impl Gpu { directory.read_only_file(c"memory", crate::agx_memory_stats::View), GFP_KERNEL, )?; + let statistics = KBox::pin_init( + directory.read_only_file(c"statistics", StatisticsView(report_statistics)), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -897,6 +950,8 @@ impl Gpu { ids: Default::default(), shared, _memory: memory, + _statistics: statistics, + _telemetry: telemetry, }, GFP_KERNEL, )?; @@ -1036,6 +1091,7 @@ impl Gpu { /// Stops the firmware and releases everything it used. pub(crate) fn shutdown(&self) { + self.shared.statistics.load.invalidate(); self.shared.feed.begin_shutdown(); self.shared.preparations.remove(); self.shared.stop_queues(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3ae1d0a609a335..bf19862884df1a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -34,7 +34,8 @@ impl event::Transport for Service<'_> { let firmware = &mut self.firmware; firmware .rings - .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role, + &firmware.primary.state.shared.statistics); } fn drain( &mut self, diff --git a/drivers/gpu/drm/asahi/g17/statistics.rs b/drivers/gpu/drm/asahi/g17/statistics.rs new file mode 100644 index 00000000000000..e5d5106e99499d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/statistics.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Observational copies of firmware statistics, independent of admission and power policy. +//! +//! The payload is deliberately unscaled until its units have been established for this +//! firmware. Readers never touch the firmware rings or take the device mutex. + +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; + +pub(crate) const WORDS: usize = 18; +const FIRST_TAG: u32 = 16; +const TAGS: usize = 15; +const ROLES: usize = 2; +pub(crate) const CAPTURE_NS: u64 = 2_000_000_000; + +struct Slot { + sequence: AtomicU64, + count: AtomicU64, + received_ns: AtomicU64, + words: [AtomicU32; WORDS], +} + +impl Slot { + const fn new() -> Self { + Self { + sequence: AtomicU64::new(0), + count: AtomicU64::new(0), + received_ns: AtomicU64::new(0), + words: [const { AtomicU32::new(0) }; WORDS], + } + } +} + +pub(crate) struct Statistics { + capture_until_ns: AtomicU64, + pub(crate) load: super::load::Load, + slots: [Slot; ROLES * TAGS], +} + +pub(crate) struct Report { + pub(crate) count: u64, + pub(crate) received_ns: u64, + pub(crate) words: [u32; WORDS], +} + +impl Statistics { + pub(crate) const fn new() -> Self { + Self { + capture_until_ns: AtomicU64::new(0), + load: super::load::Load::new(), + slots: [const { Slot::new() }; ROLES * TAGS], + } + } + + /// A diagnostic read requests a bounded full-payload capture; normal load polling does not. + pub(crate) fn request_capture(&self, now_ns: u64) { + self.capture_until_ns + .fetch_max(now_ns.saturating_add(CAPTURE_NS), Ordering::Relaxed); + } + + pub(crate) fn capture_pending(&self) -> bool { + self.capture_until_ns.load(Ordering::Relaxed) != 0 + } + + pub(crate) fn capture_enabled(&self, now_ns: u64) -> bool { + let deadline = self.capture_until_ns.load(Ordering::Relaxed); + if deadline == 0 { + return false; + } + if now_ns < deadline { + return true; + } + // A reader can renew the request concurrently with expiry. + match self.capture_until_ns.compare_exchange( + deadline, + 0, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => false, + Err(renewed) => now_ns < renewed, + } + } + + /// Called only by the serialized firmware event worker, before returning the ring slot. + pub(crate) fn record(&self, role: usize, received_ns: u64, words: &[u32; WORDS]) { + let Some(tag) = words[0].checked_sub(FIRST_TAG) else { + return; + }; + if role >= ROLES || tag as usize >= TAGS { + return; + } + self.load.record(role, received_ns, words); + let slot = &self.slots[role * TAGS + tag as usize]; + slot.sequence.fetch_add(1, Ordering::AcqRel); + for (out, value) in slot.words.iter().zip(words) { + out.store(*value, Ordering::Relaxed); + } + slot.received_ns.store(received_ns, Ordering::Relaxed); + slot.count.fetch_add(1, Ordering::Relaxed); + slot.sequence.fetch_add(1, Ordering::Release); + } + + /// A bounded, nonblocking snapshot. Contention is distinct from an unseen record. + pub(crate) fn snapshot(&self, role: usize, tag: u32) -> Option { + let tag = tag.checked_sub(FIRST_TAG)? as usize; + if role >= ROLES || tag >= TAGS { + return None; + } + let slot = &self.slots[role * TAGS + tag]; + for _ in 0..8 { + let sequence = slot.sequence.load(Ordering::Acquire); + if sequence & 1 != 0 { + continue; + } + let report = Report { + count: slot.count.load(Ordering::Relaxed), + received_ns: slot.received_ns.load(Ordering::Relaxed), + words: core::array::from_fn(|i| slot.words[i].load(Ordering::Relaxed)), + }; + fence(Ordering::Acquire); + if sequence == slot.sequence.load(Ordering::Relaxed) { + return Some(report); + } + } + None + } +} diff --git a/drivers/gpu/drm/asahi/g17/telemetry.rs b/drivers/gpu/drm/asahi/g17/telemetry.rs new file mode 100644 index 00000000000000..8b3d470911abb0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/telemetry.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-device sysfs lifetime. Readers retain only the atomic statistics allocation. + +use super::statistics::Statistics; +use core::ffi::{c_int, c_void}; +use kernel::{device::Device, prelude::*, sync::Arc}; + +/// Private callback contract shared with `asahi_telemetry.c`, not a userspace ABI. +#[repr(C)] +struct Export { + valid: u32, + power_known: u32, + powered: u32, + utilization: [u32; 4], + clock_hz: u32, + samples: u64, + received_ns: u64, + timestamp_ticks: u64, + power_timestamp_ticks: u64, + powered_ticks: u64, + unpowered_ticks: u64, + invalid: u64, + age_ns: u64, +} + +const _: () = { + assert!(core::mem::size_of::() == 96); + assert!(core::mem::offset_of!(Export, samples) == 32); +}; + +unsafe extern "C" { + fn asahi_telemetry_register( + dev: *mut c_void, + data: *const c_void, + snapshot: unsafe extern "C" fn(*const c_void, *mut c_void) -> c_int, + handle: *mut *mut c_void, + ) -> c_int; + fn asahi_telemetry_unregister(dev: *mut c_void, handle: *mut c_void); +} + +unsafe extern "C" fn snapshot(data: *const c_void, out: *mut c_void) -> c_int { + // SAFETY: registration retains this exact allocation until removal drains readers. + let stats = unsafe { &*data.cast::() }; + let Some(report) = stats.load.snapshot() else { + return EAGAIN.to_errno(); + }; + let current = report.current(); + // SAFETY: the C callback provides exclusive, properly aligned stack output storage. + unsafe { + out.cast::().write(Export { + valid: u32::from(current.is_some()), + power_known: u32::from(report.power_known), + powered: u32::from(report.powered), + utilization: current.unwrap_or([0; 4]), + clock_hz: super::load::CLOCK_HZ as u32, + samples: report.samples, + received_ns: report.received_ns, + timestamp_ticks: report.ticks, + power_timestamp_ticks: report.power_ticks, + powered_ticks: report.powered_ticks, + unpowered_ticks: report.unpowered_ticks, + invalid: report.invalid, + age_ns: super::now_ns().saturating_sub(report.received_ns), + }) + }; + 0 +} + +pub(super) struct Telemetry { + dev: *mut c_void, + handle: *mut c_void, + _statistics: Arc, +} + +// SAFETY: raw pointers are used only for attribute removal; readers access atomics. +unsafe impl Send for Telemetry {} +// SAFETY: registration state is immutable until its exclusive drop removes the attribute. +unsafe impl Sync for Telemetry {} + +impl Telemetry { + pub(super) fn new(dev: &Device, statistics: Arc) -> Result { + let raw = dev.as_raw().cast(); + let mut handle = core::ptr::null_mut(); + // SAFETY: the bound platform device outlives this registration; the Arc owns data. + let ret = unsafe { + asahi_telemetry_register( + raw, + (&*statistics as *const Statistics).cast(), + snapshot, + &mut handle, + ) + }; + kernel::error::to_result(ret)?; + Ok(Self { + dev: raw, + handle, + _statistics: statistics, + }) + } +} + +impl Drop for Telemetry { + fn drop(&mut self) { + // SAFETY: removal drains callback users before the Arc and its allocation are dropped. + unsafe { asahi_telemetry_unregister(self.dev, self.handle) }; + } +} diff --git a/tools/asahi/g17-load-tests.rs b/tools/asahi/g17-load-tests.rs new file mode 100644 index 00000000000000..b04b992114a25d --- /dev/null +++ b/tools/asahi/g17-load-tests.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Tests the production decoder against power transitions, inter-ring ordering and overlap. +#[path = "../../drivers/gpu/drm/asahi/g17/load.rs"] +mod load; +use load::Load; +fn record(tag: u32, timestamp: u64) -> [u32; 18] { + let mut w = [0; 18]; + w[0] = tag; + w[1] = timestamp as u32; + w[2] = (timestamp >> 32) as u32; + w +} +fn util(timestamp: u64, values: [u32; 4]) -> [u32; 18] { + let mut w = record(0x13, timestamp); + w[3..7].copy_from_slice(&values); + w +} +fn power(timestamp: u64, off: bool) -> [u32; 18] { + let mut w = record(0x19, timestamp); + w[5] = u32::from(!off); + w[6] = u32::from(off); + w +} +#[test] +fn aggregate_is_firmware_union_not_engine_sum() { + let s = Load::new(); + assert!(s.snapshot().unwrap().current().is_none()); + // Captured mixed rendering/compute: engine activity overlaps. + s.record(0, 1000, &util(100, [69, 93, 74, 95])); + assert_eq!(s.snapshot().unwrap().current(), Some([69, 93, 74, 95])); +} +#[test] +fn power_off_clears_retained_load_and_wake_requires_a_fresh_sample() { + let s = Load::new(); + s.record(0, 1, &util(100, [10, 90, 0, 90])); + s.record(1, 2, &power(110, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0; 4])); + s.record(1, 3, &power(120, false)); + assert!(s.snapshot().unwrap().current().is_none()); + s.record(0, 4, &util(130, [0, 0, 94, 94])); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 0, 94, 94])); +} +#[test] +fn late_old_ring_records_do_not_clear_newer_work() { + let s = Load::new(); + s.record(0, 10, &util(200, [0, 0, 90, 90])); + s.record(1, 11, &power(100, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 0, 90, 90])); + s.record(1, 12, &power(190, false)); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 0, 90, 90])); + s.record(1, 13, &power(150, true)); + assert_eq!(s.snapshot().unwrap().power_ticks, 190); +} +#[test] +fn invalid_samples_and_wrong_sources_cannot_replace_a_valid_sample() { + let s = Load::new(); + s.record(0, 1, &util(100, [1, 20, 0, 20])); + s.record(1, 2, &util(200, [2, 100, 0, 100])); + s.record(0, 3, &power(200, true)); + s.record(0, 4, &util(300, [1, 101, 0, 20])); + s.record(0, 5, &util(90, [1, 90, 0, 90])); + let r = s.snapshot().unwrap(); + assert_eq!(r.samples, 1); + assert_eq!(r.invalid, 1); + assert_eq!(r.current(), Some([1, 20, 0, 20])); +} +#[test] +fn unwritten_tails_and_coprocessor_busy_are_not_gpu_utilization() { + let s = Load::new(); + let mut w = util(100, [1, 20, 0, 20]); + w[7..].fill(u32::MAX); + s.record(0, 1, &w); + let mut busy = record(0x1a, 200); + busy[3] = 0; + s.record(0, 2, &busy); + s.record(1, 3, &busy); + assert_eq!(s.snapshot().unwrap().current(), Some([1, 20, 0, 20])); +} +#[test] +fn timestamps_keep_all_64_bits_and_power_residency_is_separate_from_busy() { + let s = Load::new(); + let timestamp = (1u64 << 32) + 100; + s.record(0, 1, &util(timestamp, [0, 0, 25, 25])); + s.record(1, 2, &record(0x11, 48_000_000)); + s.record(1, 3, &record(0x12, 24_000_000)); + let r = s.snapshot().unwrap(); + assert_eq!(r.ticks, timestamp); + assert_eq!(r.powered_ticks, load::CLOCK_HZ); + assert_eq!(r.unpowered_ticks, 2 * load::CLOCK_HZ); + assert_eq!(r.current(), Some([0, 0, 25, 25])); + s.record(1, 4, &record(0x12, u64::MAX)); + assert_eq!(s.snapshot().unwrap().powered_ticks, u64::MAX); +} +#[test] +fn concurrent_snapshots_are_consistent() { + let s = Load::new(); + std::thread::scope(|scope| { + scope.spawn(|| { + for i in 1..=50_000 { + let v = i % 101; + s.record(0, i as u64, &util(i as u64, [v; 4])); + } + }); + for _ in 0..50_000 { + if let Some(r) = s.snapshot() { + if r.samples != 0 { + assert_eq!(r.received_ns, r.ticks); + assert_eq!(r.utilization, [(r.ticks % 101) as u32; 4]); + } + } + } + }); + assert_eq!(s.snapshot().unwrap().samples, 50_000); +} + +#[test] +fn device_loss_stays_unavailable_even_when_queued_records_arrive() { + let s = Load::new(); + s.record(0, 1, &util(100, [10, 90, 0, 90])); + s.invalidate(); + assert!(s.snapshot().unwrap().current().is_none()); + s.record(1, 2, &power(200, true)); + s.record(0, 3, &util(300, [0, 0, 95, 95])); + assert!(s.snapshot().unwrap().current().is_none()); +} + +#[test] +fn same_state_reports_refresh_the_witness_without_resetting_the_wake_epoch() { + let s = Load::new(); + s.record(0, 1, &util(110, [0, 90, 0, 90])); + s.record(1, 2, &power(100, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 90, 0, 90])); + s.record(1, 3, &power(120, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0; 4])); + s.record(1, 4, &power(130, false)); + s.record(1, 5, &power(150, false)); + s.record(0, 6, &util(140, [0, 20, 0, 20])); + let r = s.snapshot().unwrap(); + assert_eq!(r.on_ticks, 130); + assert_eq!(r.power_ticks, 150); + assert_eq!(r.current(), Some([0, 20, 0, 20])); +} +#[test] +fn concurrent_repeated_power_reports_preserve_state_and_epoch_consistency() { + let s = Load::new(); + std::thread::scope(|scope| { + scope.spawn(|| { + for i in 1..=20_000u64 { + let base = i * 20; + s.record(1, base, &power(base, false)); + s.record(0, base + 1, &util(base + 1, [1, 90, 0, 90])); + for j in 2..10 { + s.record(1, base + j, &power(base + j, false)); + } + s.record(1, base + 10, &power(base + 10, true)); + for j in 11..20 { + s.record(1, base + j, &power(base + j, true)); + } + } + }); + for _ in 0..100_000 { + if let Some(r) = s.snapshot() { + if r.power_known { + assert!(r.power_ticks >= r.on_ticks); + } + if r.power_known && !r.powered { + assert_eq!(r.current(), Some([0; 4])); + } + if let Some(v) = r.current() { + assert!(v == [0; 4] || v == [1, 90, 0, 90]); + } + } + } + }); +} + +#[test] +fn unrelated_power_payload_words_do_not_change_decoding() { + let s = Load::new(); + let mut w = power(100, true); + w[5] = u32::MAX; + s.record(1, 1, &w); + assert_eq!(s.snapshot().unwrap().current(), Some([0; 4])); + assert_eq!(s.snapshot().unwrap().invalid, 0); +} diff --git a/tools/asahi/g17-statistics-tests.rs b/tools/asahi/g17-statistics-tests.rs new file mode 100644 index 00000000000000..3b4f49314dae4b --- /dev/null +++ b/tools/asahi/g17-statistics-tests.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Host tests for the production firmware-report snapshot, including concurrent readers. + +#[path = "../../drivers/gpu/drm/asahi/g17/load.rs"] +mod load; + +#[path = "../../drivers/gpu/drm/asahi/g17/statistics.rs"] +mod statistics; + +use statistics::{Statistics, WORDS}; + +#[test] +fn retains_full_payload_and_separates_sources() { + let stats = Statistics::new(); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 0); + let mut words = core::array::from_fn(|i| i as u32); + words[0] = 0x14; + stats.record(0, 1234, &words); + let report = stats.snapshot(0, 0x14).unwrap(); + assert_eq!(report.words, words); + assert_eq!(report.received_ns, 1234); + assert_eq!(report.count, 1); + assert_eq!(stats.snapshot(1, 0x14).unwrap().count, 0); + assert_eq!(stats.snapshot(0, 0x19).unwrap().count, 0); + words[17] = u32::MAX; + stats.record(0, 4567, &words); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 2); + assert_eq!(stats.snapshot(0, 0x14).unwrap().words, words); +} + +#[test] +fn rejects_out_of_range_sources_without_aliasing() { + let stats = Statistics::new(); + for tag in [0, 15, 31, u32::MAX] { + let mut words = [0; WORDS]; + words[0] = tag; + stats.record(0, 1, &words); + assert!(stats.snapshot(0, tag).is_none()); + } + let mut words = [0; WORDS]; + words[0] = 0x14; + stats.record(2, 1, &words); + assert!(stats.snapshot(2, 0x14).is_none()); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 0); +} + +#[test] +fn readers_never_observe_a_torn_report() { + let stats = Statistics::new(); + std::thread::scope(|scope| { + scope.spawn(|| { + for sequence in 1..=50_000 { + let mut words = [sequence; WORDS]; + words[0] = 0x14; + stats.record(0, sequence as u64, &words); + } + }); + for _ in 0..50_000 { + if let Some(report) = stats.snapshot(0, 0x14) { + if report.count != 0 { + assert_eq!(report.received_ns, report.count); + assert_eq!(report.words[0], 0x14); + assert!(report.words[1..] + .iter() + .all(|&word| word as u64 == report.count)); + } + } + } + }); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 50_000); +} + +#[test] +fn full_payload_capture_is_requested_explicitly_and_expires() { + let stats = Statistics::new(); + assert!(!stats.capture_pending()); + assert!(!stats.capture_enabled(1)); + stats.request_capture(100); + assert!(stats.capture_pending()); + assert!(stats.capture_enabled(101)); + stats.request_capture(200); + assert!(stats.capture_enabled(100 + statistics::CAPTURE_NS)); + assert!(!stats.capture_enabled(200 + statistics::CAPTURE_NS)); + assert!(!stats.capture_pending()); +} diff --git a/tools/asahi/g17-telemetry-registration-tests.py b/tools/asahi/g17-telemetry-registration-tests.py new file mode 100644 index 00000000000000..cf5d7b7b9968c9 --- /dev/null +++ b/tools/asahi/g17-telemetry-registration-tests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check actual probe registration branches against a shared sysfs namespace. + +The namespace and optional-registration failures are host boundary stubs, not +kernel execution. A negative control restores the duplicate placeholder probe. +""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / "drivers/gpu/drm/asahi" + + +def fixture(): + gpu = (ROOT / "g17/mod.rs").read_text() + probe = (ROOT / "driver.rs").read_text().split(" fn probe_g17", 1)[1] + telemetry = gpu[gpu.index(" let telemetry ="):gpu.index(" let active_cores =")] + platform = probe[probe.index(" let raw_stats ="):probe.index(" Ok(AsahiPlatformDriverData")] + names = [re.search(r'attr\.attr\.name = "([^"]+)"', (ROOT / name).read_text()).group(1) + for name in ["sysfs.c", "asahi_telemetry.c"]] + assert names == ["agx_stats", "agx_stats"], names + return r''' +#![allow(dead_code, non_upper_case_globals)] +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering::SeqCst}; +type Result = std::result::Result; +static EXPORT: AtomicU8 = AtomicU8::new(1); +static PRESENT: AtomicBool = AtomicBool::new(false); +static FAIL: AtomicBool = AtomicBool::new(false); +static STOPPED: AtomicBool = AtomicBool::new(false); +static DUPLICATE: AtomicBool = AtomicBool::new(false); +macro_rules! dev_warn { ($dev:expr, $($t:tt)*) => {{ let _ = $dev; let _ = format_args!($($t)*); }}; } +mod module_parameters { + pub struct Parameter; + impl Parameter { + pub fn value(&self) -> Box { Box::new(super::EXPORT.load(super::SeqCst)) } + } + pub static stats_export: Parameter = Parameter; +} +struct Device; +impl AsRef for Device { fn as_ref(&self) -> &Self { self } } +impl Device { fn as_raw(&self) -> *mut () { std::ptr::dangling_mut() } } +struct Gpu; +impl Gpu { fn shutdown(&self) { STOPPED.store(true, SeqCst); } } +fn create() -> Result<*mut ()> { + if FAIL.load(SeqCst) { return Err(-12); } + if PRESENT.swap(true, SeqCst) { + DUPLICATE.store(true, SeqCst); + return Err(-17); + } + Ok(std::ptr::dangling_mut()) +} +fn remove() { assert!(PRESENT.swap(false, SeqCst)); } +mod telemetry { + use super::*; + pub struct Telemetry; + impl Telemetry { + pub fn new(_: &Device, _: u32) -> Result { create()?; Ok(Self) } + } + impl Drop for Telemetry { fn drop(&mut self) { remove(); } } +} +mod sysfs_exports { + use super::*; + pub fn register(_: *mut (), snapshot: *const (), enabled: bool) -> Result<*mut ()> { + assert!(snapshot.is_null() && !enabled); + create() + } +} +fn gpu_registration(pdev: &Device) -> Option { + let statistics = 0u32; +TELEMETRY + telemetry +} +fn platform_registration(pdev: &Device, gpu: &Gpu) -> Result<*mut ()> { + let raw_dev = pdev.as_raw(); +PLATFORM + Ok(raw_stats) +} +fn main() { + for export in [1, 0, 2, 0] { + for fail in [false, true] { + EXPORT.store(export, SeqCst); FAIL.store(fail, SeqCst); + STOPPED.store(false, SeqCst); + let telemetry = gpu_registration(&Device); + let platform = platform_registration(&Device, &Gpu); + assert!(!DUPLICATE.load(SeqCst), "both probe owners created agx_stats"); + if export != 0 { + assert_eq!(telemetry.is_some(), !fail); + assert!(platform.unwrap().is_null()); + assert!(!STOPPED.load(SeqCst), "optional telemetry stopped GPU probe"); + } else if fail { + assert_eq!(platform, Err(-12)); + assert!(STOPPED.load(SeqCst)); + } else { + assert!(telemetry.is_none()); + assert!(!platform.unwrap().is_null()); + remove(); + } + drop(telemetry); + assert!(!PRESENT.load(SeqCst), "registration outlived its owner"); + } + } + println!("PASS enabled/disabled probe ownership, allocation failures and repeated teardown"); +} +'''.replace("TELEMETRY", telemetry).replace("PLATFORM", platform) + + +def main(): + code = fixture() + with tempfile.TemporaryDirectory(prefix="g17-telemetry-registration-") as temporary: + root = Path(temporary) + for mutant in [False, True]: + source = root / f"registration-{mutant}.rs" + source.write_text(code.replace( + "let raw_stats = if *module_parameters::stats_export.value() == 0", + "let raw_stats = if true") if mutant else code) + binary = source.with_suffix("") + subprocess.run([os.environ.get("RUSTC", "rustc"), "--edition=2021", + str(source), "-o", str(binary)], check=True) + result = subprocess.run([str(binary)], text=True, capture_output=True) + if mutant: + assert result.returncode != 0 and "both probe owners" in result.stderr + print("PASS negative control rejects duplicate agx_stats probe registration") + else: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/gpu-load-tests.py b/tools/asahi/gpu-load-tests.py new file mode 100644 index 00000000000000..ecd1d502193283 --- /dev/null +++ b/tools/asahi/gpu-load-tests.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""A transient kernel snapshot collision must not terminate load monitoring.""" +import errno +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import Mock + +spec = importlib.util.spec_from_file_location('gpu_load', Path(__file__).with_name('gpu-load.py')) +gpu_load = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gpu_load) + + +class LoadTests(unittest.TestCase): + def test_busy_snapshot_is_unavailable(self): + path = Mock() + path.parent.resolve.return_value = '/sys/devices/platform/gpu' + path.read_text.side_effect = BlockingIOError(errno.EAGAIN, 'snapshot busy') + sample = gpu_load.snapshot(path) + self.assertFalse(sample['load_valid']) + self.assertIsNone(sample['busy_percent']) + + def test_other_read_errors_are_reported(self): + path = Mock() + path.read_text.side_effect = PermissionError(errno.EACCES, 'permission denied') + with self.assertRaises(PermissionError): + gpu_load.snapshot(path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/asahi/gpu-load.py b/tools/asahi/gpu-load.py new file mode 100755 index 00000000000000..5fd51b8ff57254 --- /dev/null +++ b/tools/asahi/gpu-load.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Read device-wide firmware load without confusing missing data with zero load.""" +import argparse +import json +from pathlib import Path +import time + + +def snapshot(path): + try: + text = path.read_text() + except BlockingIOError: + # A snapshot can lose its bounded retries to the firmware writer. + # Preserve an unavailable sample instead of terminating a monitor. + return {'device': str(path.parent.resolve()), 'source': 'firmware', + 'load_valid': False, 'busy_percent': None, + 'error': 'statistics snapshot busy'} + fields = dict(line.split(None, 1) for line in text.splitlines() + if len(line.split(None, 1)) == 2) + if fields.get('version') != '2' or fields.get('source') != 'firmware': + raise ValueError(f'{path}: unsupported statistics format') + valid = fields.get('load_valid') == '1' + return { + 'device': str(path.parent.resolve()), + 'source': 'firmware', + 'load_valid': valid, + 'busy_percent': int(fields['busy_percent']) if valid else None, + 'firmware_percentages': [int(fields[f'util{i}']) for i in range(1, 5)] if valid else None, + 'gpu_powered': fields['gpu_powered'] == '1' if fields['gpu_power_known'] == '1' else None, + 'samples': int(fields['samples']), + 'sample_age_seconds': int(fields['sample_age_ns']) / 1e9 if int(fields['samples']) else None, + 'invalid_samples': int(fields['invalid_samples']), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--path', type=Path) + parser.add_argument('--count', type=int, default=1) + parser.add_argument('--interval', type=float, default=1.0) + args = parser.parse_args() + if args.count < 1 or args.interval <= 0: + parser.error('count and interval must be positive') + paths = [args.path] if args.path else sorted(Path('/sys/class/drm').glob('renderD*/device/agx_stats')) + if not paths: + parser.error('no GPU statistics attribute found') + for index in range(args.count): + print(json.dumps([snapshot(path) for path in paths]), flush=True) + if index + 1 != args.count: + time.sleep(args.interval) + + +if __name__ == '__main__': + main() From c61760ecddda6ee51b6ea19ef036a0e919cf559e Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:26:00 -0400 Subject: [PATCH 04/22] drm/asahi: reclaim unused G17 render warming under queue pressure --- drivers/gpu/drm/asahi/g17/kick.rs | 28 ++- drivers/gpu/drm/asahi/g17/queue/admission.rs | 129 +++++++++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 14 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 12 +- drivers/gpu/drm/asahi/g17/runtime.rs | 14 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 121 +++++++++++- .../drm/asahi/g17/runtime/render/prepare.rs | 4 +- .../drm/asahi/g17/runtime/render/submit.rs | 88 ++++++++- tools/asahi/g17-resource-tests.py | 178 ++++++++++++++++++ 9 files changed, 548 insertions(+), 40 deletions(-) create mode 100644 tools/asahi/g17-resource-tests.py diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86de..87d37ad527a11b 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -17,7 +17,7 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. +/// Minimum render-channel capacity protected from compute-only allocation. const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a @@ -83,10 +83,21 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Published identities always retain their original lifetime rules. + if engine == DataMaster::Compute { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| { + matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) + }) + .count(); + if QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE.saturating_sub(render) { + return Err(ENOSPC); + } } let index = preferred .map(usize::from) @@ -127,6 +138,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f6..6e99160729034f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -115,18 +115,20 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -175,8 +177,51 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; + } + } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { @@ -188,9 +233,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -420,8 +487,33 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -468,6 +560,17 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e402..4290bd136b5dab 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -152,6 +152,7 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, descriptor_flags: [u32; 2], installed: [bool; 2], @@ -239,6 +240,7 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, descriptor_flags, installed: [false; 2], @@ -503,9 +505,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -730,9 +740,6 @@ impl Pair { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -1736,6 +1743,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 39941de815e077..e811d7e5c9073d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -48,11 +48,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -190,6 +199,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9f..bcf62844c689e0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -624,7 +624,19 @@ impl Backend { Rebind::Grow => { drop(state); return self.grow_compute(required); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + drop(state); + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; (firmware.uat.clone(), Ok(reservation)) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf20455261..30da1b09eedfac 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -55,6 +55,12 @@ struct Entry { published: bool, } +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -198,6 +204,113 @@ impl RenderState { } impl Registry { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); @@ -262,12 +375,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 06bea3c84fa7ad..14c4ce8c90754d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -568,9 +568,11 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { + match self.build_render_pair(sibling.slot(), packet, true) { Err(error) if error == EBUSY || error == EAGAIN => {} result => result?, } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c5..abea0b4c06f290 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -23,6 +23,83 @@ impl Backend { self.render_with_lease(packet, None, deferred) } + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + ) -> Result { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + loop { + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + match pool.acquire_render(self.owner, Request::of(packet)) { + Ok(lease) => lease, + Err(error) if error == ENOSPC => { + let reclaimed = { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .reclaim_unused_render_spare(Some(self.owner), false)? + }; + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => return Ok(lease), + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + drop(state); + if reclaimed.is_none() && !ready { + return Err(error); + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } + } + fn render_with_lease( &self, packet: &Arc, @@ -37,16 +114,7 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; + let admission = self.acquire_render_with_ids(packet, early)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py new file mode 100644 index 00000000000000..b9383b6a7da7db --- /dev/null +++ b/tools/asahi/g17-resource-tests.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual G17 admission, queue-ID, and unused-graph eligibility methods. + +Run with Python and rustc; temporary host binaries are removed after the checks. +""" +from pathlib import Path +import os +import re +import subprocess +import tempfile + +DRIVER = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi' + +def admission_tests(root, source): + s=(source/'g17/queue/admission.rs').read_text();cfg=(source/'hw/t8140.rs').read_text() + def item(start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_PER_OWNER','RENDER_DEPTH']) + mmu=(source/'mmu.rs').read_text() + limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) + header='''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + type KVec = Vec; struct InFlight; + mod cfg {\n'''+constants+'}\n'+limits+'\n' + methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' + fn main(){ + let mut state=State::new(); + for owner in 1..=32 { + assert!(state.refusal(owner).is_ok()); + let slot=state.acquire(owner).expect("existing owner fits"); + state.release(slot); + } + assert_eq!(state.active,0,"all submitted work retired"); + let retained=state.owners.iter().filter(|o|**o!=0).count(); + assert_eq!(retained,63); + assert_eq!(state.refusal(33),Err(ENOSPC)); + assert!(state.acquire(33).is_none()); + println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); + // Yield one unused spare for each additional logical owner. The runtime + // separately proves that these physical graphs have never been published. + for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); + assert_eq!(state.acquire(owner),Some(slot as u8));state.release(slot as u8); + } + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner).unwrap();state.release(slot);} + assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); + println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); + let mut guarded=State::new();let first=guarded.acquire(1).unwrap();guarded.release(first); + let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; + guarded.take_idle(spare,1); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); + guarded.release(spare as u8); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(3),Some(2)),"owner mismatch"); + guarded.quarantined |= 1u64< = std::result::Result; + const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} + #[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} + #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} + '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name)+'\n' for name in ['reserve','publish','can_cancel_unpublished','cancel_unpublished'])+'}\n' + test=''' + fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],compute_publication:0,leased:0}} + fn main(){ + for render in [0,2,16,32,112,126] { + let mut q=fresh(); + for i in 0..render {q.reserve(i as u64+1,if i%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();} + let mut compute=0; + while q.reserve(1000+compute,DataMaster::Compute,None).is_ok(){compute+=1;} + assert_eq!(compute as usize,QID_COUNT-render.max(COMPUTE_RENDER_RESERVE)); + } + let mut q=fresh();let first=q.reserve(1,DataMaster::Tiling,None).unwrap(); + assert!(q.can_cancel_unpublished(first));q.publish(first).unwrap(); + assert!(!q.can_cancel_unpublished(first));assert_eq!(q.cancel_unpublished(first),Err(EINVAL));assert_eq!(q.leased,1); + let unused=q.reserve(2,DataMaster::Fragment,None).unwrap();q.cancel_unpublished(unused).unwrap();assert_eq!(q.leased,1); + let next=q.reserve(3,DataMaster::Fragment,None).unwrap();assert_eq!(next.qid,unused.qid); + assert!(!q.can_cancel_unpublished(unused));assert_eq!(q.cancel_unpublished(unused),Err(EINVAL)); + println!("queue IDs: six render/compute capacity boundaries; compute-only render reserve retained; published and stale identities protected"); + } + ''' + fixture=root/'queue-id-capacity-controls.rs';fixture.write_text(header+test) + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'queue-id-capacity-controls')],check=True) + subprocess.run([str(root/'queue-id-capacity-controls')],check=True) + s=(source/'queue/render/parked.rs').read_text();guard=item(s,'pub(crate) fn can_yield_unused') + fixture=root/'unused-render-guard-controls.rs';fixture.write_text('''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + struct Context; + struct Pair {owner:Option,ordinal:u64,registered:Option,outer_started:u8,previous:Option<()>,graph_started:bool,backing:Result} + impl Pair { + fn context(&self)->&Context {&Context} + fn can_park_backing(&self,_:&Context)->Result{self.backing} + '''+guard+''' + } + fn fresh()->Pair {Pair {owner:Some(1),ordinal:0,registered:None,outer_started:0,previous:None,graph_started:false,backing:Ok(true)}} + fn main(){ + assert_eq!(fresh().can_yield_unused(),Ok(true)); + let mut p=fresh();p.owner=None;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.ordinal=1;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.registered=Some(2);assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.outer_started=4;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.graph_started=true;assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Ok(false);assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Err(-5);assert_eq!(p.can_yield_unused(),Err(-5)); + println!("unused graph: eight eligibility/error controls protect every publication boundary"); + } + ''') + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) + subprocess.run([str(root/'unused-render-guard-controls')],check=True) + +if __name__ == '__main__': + with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: + admission_tests(Path(temporary), DRIVER) + queue_tests(Path(temporary), DRIVER) From d14aac52817a8b1d650cc9dcf3a274b6e0a27404 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:39:10 -0400 Subject: [PATCH 05/22] drm/asahi: recover unused initial G17 queue IDs for compute Signed-off-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/asahi/g17/kick.rs | 46 ++++++++++++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 4 +-- tools/asahi/g17-resource-tests.py | 39 +++++++++++++++++++++-- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 87d37ad527a11b..e2a9bb643b892d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -47,6 +47,7 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], + initial_render: [Option; 2], compute_publication: u64, leased: usize, } @@ -55,11 +56,29 @@ impl QueueIds { pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), + initial_render: [None; 2], compute_publication: 0, leased: 0, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -83,6 +102,18 @@ impl QueueIds { Err(EINVAL) }; } + let initial = if engine == DataMaster::Compute && self.leased == QID_COUNT { + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + } else { + None + }; // Existing render identities already satisfy this reservation. Counting // them twice can refuse compute even while physical QIDs remain free. // Published identities always retain their original lifetime rules. @@ -95,7 +126,13 @@ impl QueueIds { matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) }) .count(); - if QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE.saturating_sub(render) { + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let replacement = usize::from(initial.is_some()); + if QID_COUNT - self.leased + replacement + <= COMPUTE_RENDER_RESERVE.saturating_sub(render - replacement) + { return Err(ENOSPC); } } @@ -103,6 +140,7 @@ impl QueueIds { .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) + .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) .ok_or(ENOSPC)?; let id = Id { qid: index as u8, @@ -113,7 +151,11 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index bcf62844c689e0..40d2a54b59e636 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -74,9 +74,7 @@ impl Registry { ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index b9383b6a7da7db..c21ab81fdca111 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -123,9 +123,9 @@ def item(s,start): #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} #[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} - '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name)+'\n' for name in ['reserve','publish','can_cancel_unpublished','cancel_unpublished'])+'}\n' + '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name+'(')+'\n' for name in ['reserve_initial_render','reserve','publish','can_cancel_unpublished','cancel_unpublished'])+'}\n' test=''' - fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],compute_publication:0,leased:0}} + fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} fn main(){ for render in [0,2,16,32,112,126] { let mut q=fresh(); @@ -141,6 +141,41 @@ def item(s,start): let next=q.reserve(3,DataMaster::Fragment,None).unwrap();assert_eq!(next.qid,unused.qid); assert!(!q.can_cancel_unpublished(unused));assert_eq!(q.cancel_unpublished(unused),Err(EINVAL)); println!("queue IDs: six render/compute capacity boundaries; compute-only render reserve retained; published and stale identities protected"); + // Reproduce the hardware snapshot: 124 installed render identities, two + // installed compute identities, and two initial reservations fill128 IDs. + let mut q=fresh();q.reserve_initial_render().unwrap(); + assert_eq!(q.reserve_initial_render(),Err(EINVAL)); + let initial=q.initial_render; + for owner in 1000..1002 {let id=q.reserve(owner,DataMaster::Compute,Some(4)).unwrap();q.publish(id).unwrap();} + let mut installed=Vec::new(); + for owner in 3..127 {let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();q.publish(id).unwrap();installed.push(id);} + assert_eq!(q.leased,QID_COUNT); + for (n,old) in initial.into_iter().enumerate() { + let old=old.unwrap();let id=q.reserve(2000+n as u64,DataMaster::Compute,None).unwrap(); + assert_eq!(id.qid,old.qid);assert_eq!(q.leased,QID_COUNT); + assert!(!q.can_cancel_unpublished(old));assert_eq!(q.cancel_unpublished(old),Err(EINVAL)); + q.publish(id).unwrap(); + } + assert_eq!(q.reserve(3000,DataMaster::Compute,None),Err(ENOSPC)); + for id in installed {assert!(!q.can_cancel_unpublished(id));assert_eq!(q.entries[id.qid as usize].unwrap().id,id);} + // A bootstrap token that became published is never replaced. + let mut q=fresh();q.reserve_initial_render().unwrap();let protected=q.initial_render[0].unwrap();q.publish(protected).unwrap(); + for owner in 3..129 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + assert_eq!(q.leased,QID_COUNT); + assert_eq!(q.reserve(1000,DataMaster::Compute,None).unwrap().qid,1); + assert_eq!(q.reserve(1001,DataMaster::Compute,None),Err(ENOSPC)); + assert_eq!(q.entries[0].unwrap().id,protected); + // Replacement must not consume the protected render minimum. + let mut q=fresh();q.reserve_initial_render().unwrap(); + for owner in 3..17 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + for owner in 1000..1112 {q.reserve(owner,DataMaster::Compute,None).unwrap();} + assert_eq!(q.leased,QID_COUNT);assert_eq!(q.reserve(2000,DataMaster::Compute,None),Err(ENOSPC)); + assert!(q.initial_render.iter().all(Option::is_some)); + // Ordinary free capacity leaves the initialization reservations untouched. + let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; + let id=q.reserve(1000,DataMaster::Compute,Some(4)).unwrap();assert_eq!(id.qid,4);assert_eq!(q.initial_render,initial); + println!("initial reservations: full-namespace compute admission restored; physical count unchanged; published and stale identities plus render minimum protected"); + } ''' fixture=root/'queue-id-capacity-controls.rs';fixture.write_text(header+test) From 8f429c0693cfed05f652a166da976f1f8d87ad31 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:29:26 -0400 Subject: [PATCH 06/22] drm/asahi: find common addresses for G17 render pool backing A canonical low address may still be occupied by a retained alias in another VM. RenderBacking::new previously propagated that fixed-address collision as ENOSPC, failing accepted render work during shared-VM queue replacement even though backing memory and other addresses were available. Place fresh backing in the canonical and both alias VMs before initializing the pool tables. Keep ordinary best-fit placement on the fast path; after a collision, search lowest-address holes in the refusing VM and retry with the same backing. Roll partial aliases back before releasing the canonical reservation. Keep guards, physical capacities, generation depth and firmware retirement requirements intact. Add source-derived controls for rollback, non-capacity errors, guards and best-fit holes, plus an independent occupancy oracle covering all 262144 combinations of three six-page address spaces. On Neo G17P B1, the first-run 40-client/shared-VM reproduction, changing pixel checks, three simultaneous 300-second workloads, concurrent layered rendering, Vulkan multisample readbacks and four compute owners pass with no kernel GPU errors. A further 2000-context shared-VM test passes and returns tracked kernel memory below its starting level after VM close. Signed-off-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/asahi/g17/freelist.rs | 21 ++-- drivers/gpu/drm/asahi/g17/object.rs | 77 ++++++++++++++ drivers/gpu/drm/asahi/mmu.rs | 33 +++++- tools/asahi/g17-resource-tests.py | 144 ++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3ec0d50033e793..f5ed7a61524937 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -76,12 +76,15 @@ pub(crate) struct Pages { } impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( + let backing = alloc.lower( TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; + Self::from_backing(backing, blocks, layout) + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { backing.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; @@ -145,14 +148,14 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let (backing, [global, client]) = alloc.lower_with_aliases( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + )?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 664e513ac4c2cb..eefa63692a7a15 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -164,6 +164,17 @@ impl KernelObject { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) } + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) + } + /// Maps an exact backing subrange, retaining an explicit unmapped guard. pub(crate) fn map_range( &self, @@ -344,6 +355,72 @@ impl Allocator<'_> { cpu, ) } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + )? + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + )? + }; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + )?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } + } } /// Families whose upper mappings remain cached and installed until firmware diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4ea797d398a8b1..f4fb0b4bb50f8b 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1859,6 +1859,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1916,7 +1947,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index c21ab81fdca111..73d5fa52b98333 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -207,7 +207,151 @@ def item(s,start): subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) subprocess.run([str(root/'unused-render-guard-controls')],check=True) +def shared_alias_tests(root, source): + s=(source/'g17/object.rs').read_text() + at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + method=s[at:end] + fixture=root/'shared-alias-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::{cell::RefCell,rc::Rc,ops::Range}; +type Result = std::result::Result;type Error=i32; +const EINVAL:i32=-22;const EOVERFLOW:i32=-75;const ENOSPC:i32=-28;const EIO:i32=-5; +mod hw {pub mod t8140 {pub mod dynamic {pub const LOWER:std::ops::Range=0..256;}}} +mod mmu {pub const UAT_PGSZ:usize=16;pub type Prot=u8;pub type Vm=super::Vm;pub type KernelMapping=super::Mapping;} +#[derive(Clone,Copy)] enum CpuMap {WriteCombined} +#[derive(Clone)] struct Region {vm:usize,start:u64,size:u64,object:u64} +#[derive(Default)] struct State {regions:Vec,allocations:u64,attempts:usize,fault:Option} +struct Vm {id:usize,state:Rc>} +struct Uat {lower:Vm} +impl Uat {fn kernel_lower_vm(&self)->&Vm {&self.lower}} +struct Device {state:Rc>} +struct Mapping {vm:usize,address:u64,size:u64,object:u64,state:Rc>} +impl Mapping {fn iova(&self)->u64 {self.address}} +impl Drop for Mapping {fn drop(&mut self) { + let mut state=self.state.borrow_mut(); + if self.vm==0 {assert!(!state.regions.iter().any(|r|r.vm!=0&&r.object==self.object&&r.start==self.address),"aliases must retire before canonical reservation");} + let index=state.regions.iter().position(|r|r.vm==self.vm&&r.start==self.address&&r.object==self.object).unwrap(); + state.regions.remove(index); +}} +fn overlaps(s:&State,vm:usize,start:u64,size:u64)->bool {s.regions.iter().any(|r|r.vm==vm&&r.startResult { + let mut state=vm.state.borrow_mut(); + if state.fault==Some(vm.id) {return Err(EIO)} + if overlaps(&state,vm.id,address,size) {return Err(ENOSPC)} + state.regions.push(Region {vm:vm.id,start:address,size,object}); + Ok(Mapping {vm:vm.id,address,size,object,state:vm.state.clone()}) +} +struct ObjectRef {id:u64,size:usize} +impl ObjectRef { + fn map_mode(&self,vm:&Vm,range:Range,align:u64,guard:bool,lowest:bool)->Result { + if vm.id==0 {assert!(guard,"canonical guard must be preserved");vm.state.borrow_mut().attempts+=1;} + let length=self.size as u64+if guard {16}else{0}; + let mut used:Vec<_>=vm.state.borrow().regions.iter().filter(|r|r.vm==vm.id).cloned().collect();used.sort_by_key(|r|r.start); + let mut cursor=range.start;let mut holes=Vec::new(); + for r in used { + if r.start>cursor {holes.push(cursor..r.start.min(range.end));} + cursor=cursor.max(r.start+r.size); + } + if cursorclaim(vm,address,length,self.id),None=>Err(ENOSPC)} + } + fn map_into_range(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:bool)->Result { + self.map_mode(vm,range,align,guard,false) + } +} +struct KernelObject {obj:ObjectRef,mapping:Option} +impl KernelObject { + fn backing(dev:&Device,size:usize,_cpu:CpuMap)->Result {let mut s=dev.state.borrow_mut();s.allocations+=1;Ok(Self {obj:ObjectRef {id:s.allocations,size},mapping:None})} + fn map_alias(&self,vm:&Vm,address:u64,_prot:u8)->Result {claim(vm,address,self.obj.size as u64,self.obj.id)} + fn alias_in_lowest(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:usize)->Result {self.obj.map_mode(vm,range,align,guard!=0,true)} +} +struct Allocator<'a> {dev:&'a Device,uat:&'a Uat} +impl Allocator<'_> { +'''+method+''' +} +fn case(obstacles:&[(usize,u64,u64)],fault:Option,size:usize,expected:std::result::Result,attempts:usize) { + let state=Rc::new(RefCell::new(State::default())); + for (vm,start,len) in obstacles {state.borrow_mut().regions.push(Region {vm:*vm,start:*start,size:*len,object:0});} + state.borrow_mut().fault=fault; + let dev=Device {state:state.clone()};let uat=Uat {lower:Vm {id:0,state:state.clone()}}; + let first=Vm {id:1,state:state.clone()};let second=Vm {id:2,state:state.clone()}; + let alloc=Allocator {dev:&dev,uat:&uat}; + let result=alloc.lower_with_aliases(size,16,3,CpuMap::WriteCombined,[&first,&second]); + match (expected,result) { + (Ok(address),Ok((object,aliases)))=>{ + assert_eq!(object.mapping.as_ref().unwrap().iova(),address); + assert!(aliases.iter().all(|m|m.iova()==address&&m.object==object.obj.id)); + assert_eq!(state.borrow().regions.iter().filter(|r|r.object!=0).count(),3); + drop(aliases);drop(object); + }, + (Err(expected),Err(actual))=>assert_eq!(actual,expected), + _=>panic!("unexpected common-address placement result"), + } + assert_eq!(state.borrow().regions.len(),obstacles.len(),"failed attempts leak no reservations or aliases"); + assert_eq!(state.borrow().allocations,if size==0||size%16!=0 {0}else{1},"all retries reuse one backing"); + assert_eq!(state.borrow().attempts,attempts); +} +fn exhaustive_common_holes() { + for canonical in 0u32..64 {for global in 0u32..64 {for client in 0u32..64 { + let masks=[canonical,global,client]; + let available=(0..4).any(|page|canonical&(7<{ + assert!(available,"allocator found a false common hole"); + let address=object.mapping.as_ref().unwrap().iova();assert_eq!(address%16,0);let page=address/16;assert!(page<4); + assert!(canonical&(7<{assert_eq!(error,ENOSPC);assert!(!available,"allocator skipped a common hole");}, + } + assert_eq!(state.borrow().allocations,1);assert_eq!(state.borrow().regions.len(),original); + }}} + println!("shared aliases: all262144 three-VM occupancy combinations match the independent common-hole oracle"); +} +fn main() { + case(&[],None,64,Ok(0),1); + case(&[(1,0,16)],None,64,Ok(16),2); + case(&[(2,0,16)],None,64,Ok(16),2); + case(&[(1,0,16),(2,64,16)],None,64,Ok(80),3); + case(&[(0,0,16)],None,64,Ok(16),1); + case(&[(2,0,256)],None,64,Err(ENOSPC),1); + case(&[(2,0,176)],None,64,Ok(176),2); + case(&[],Some(1),64,Err(EIO),1); + case(&[],Some(2),64,Err(EIO),1); + case(&[],Some(0),64,Err(EIO),1); + case(&[],None,256,Err(ENOSPC),1); + case(&[],None,0,Err(EINVAL),0); + case(&[],None,15,Err(EINVAL),0); + // Best-fit initially selects the small late hole, occupied in the client. + // The larger earlier canonical hole must remain eligible after that collision. + case(&[(0,128,16),(2,144,112)],None,64,Ok(0),2); + case(&[(1,0,256)],None,64,Err(ENOSPC),1); + case(&[(0,128,128),(2,0,128)],None,64,Err(ENOSPC),2); + println!("shared aliases: 16 best-fit, collision, rollback, lifetime, backing-reuse, guard and error controls pass"); + exhaustive_common_holes(); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'shared-alias-controls')],check=True) + subprocess.run([str(root/'shared-alias-controls')],check=True) + if __name__ == '__main__': with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: admission_tests(Path(temporary), DRIVER) queue_tests(Path(temporary), DRIVER) + shared_alias_tests(Path(temporary), DRIVER) From a5806fdc14736a14daace495ebe39153e5e55f0b Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:20:52 -0400 Subject: [PATCH 07/22] drm/asahi: wait for pending G17 compute pool releases A released compute queue cannot be rebound while its USC pool release is still awaiting control-consumer observation. At a full physical QID namespace, all retained queues can occupy this state together: admission then returns ENOSPC even though no owner or command holds those queues. Keep the event worker polling until every published pool release has been observed. Classify namespace exhaustion with a pending release as temporary backpressure and let admission retry outside the device mutex and the preparation lease. Bound that wait and return EBUSY if it does not settle. Preserve the firmware release witness, physical identity lifetimes, warm retention policy and normal admission with spare capacity. Source-derived controls cover the captured four-queue transition, wake removal after settlement, genuine exhaustion and error propagation; both original failure modes fail the same checks as negative controls. On Neo G17P, the full cold-start pressure/geometry/multisample gate and three simultaneous 300-second workloads pass. Twelve further four-client compute launches, including the ten-second pool-expiry boundary, verify 420,110,336 changing integer words with no kernel GPU errors. Signed-off-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 6 + drivers/gpu/drm/asahi/g17/runtime.rs | 32 +++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 +- tools/asahi/g17-pool-release-tests.py | 169 ++++++++++++++++++++ 4 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 tools/asahi/g17-pool-release-tests.py diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb7609..0593352863e76e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -626,6 +626,12 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 40d2a54b59e636..4e6a455d6b1ceb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -273,6 +273,20 @@ impl Registry { Ok(freed) } + /// A retained queue in a pool-release transaction temporarily cannot bind. + /// Distinguish that state from permanent physical-identity exhaustion. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC + && self.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_pending()) + { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { @@ -294,7 +308,8 @@ impl Registry { let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, @@ -595,6 +610,8 @@ impl Backend { } fn build_compute(&self, required: usize) -> Result { + const POOL_RELEASE_WAIT_NS: u64 = 200_000_000; + let mut pool_release_wait = None; loop { match self.shared_compute() { Err(error) if error == EAGAIN => continue, @@ -624,6 +641,19 @@ impl Backend { Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, + Err(error) if error == EAGAIN => { + // The event worker owns release observation and + // drops vacated aliases/backing off this mutex. + let since = *pool_release_wait.get_or_insert(now_ns()); + drop(state); + drop(lease); + self.shared.queue_events(); + if now_ns().saturating_sub(since) >= POOL_RELEASE_WAIT_NS { + return Err(EBUSY); + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; drop(state); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index bf19862884df1a..fef9463d699cc6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -420,7 +420,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } diff --git a/tools/asahi/g17-pool-release-tests.py b/tools/asahi/g17-pool-release-tests.py new file mode 100644 index 00000000000000..f8fcee86d9ed17 --- /dev/null +++ b/tools/asahi/g17-pool-release-tests.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production pool-release admission and worker wake predicates. + +Firmware and allocation boundaries are host stubs. This is not GPU execution. +Both original failure modes must fail the same state-transition assertions. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def fixture(): + runtime = (ROOT / 'runtime.rs').read_text() + compute = (ROOT / 'queue/compute.rs').read_text() + events = (ROOT / 'runtime/events.rs').read_text() + predicate = '.any(|queue| queue.retirement_polling() || queue.pool_release_pending())' + assert events.count(predicate) == 1 + methods = '\n'.join(item(runtime, name) for name in [ + 'fn compute_admission_error(', 'pub(super) fn reserve_compute(']).replace('pub(super) ', '') + pending = item(compute, 'pub(crate) fn pool_release_pending(') + return r''' +#![allow(dead_code)] +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENOSPC: Error = -28; +const EAGAIN: Error = -11; +const FIRST_COMPUTE_QID: u8 = 4; +#[derive(Clone, Copy, PartialEq, Eq)] +enum DataMaster { Compute } +#[derive(Clone, Copy)] +enum PoolPhase { Populated, ReleaseSent(u32), Vacant } +mod compute { + use super::*; + pub(super) struct Queue { + pub owner: Option, + pub pool_phase: PoolPhase, + pub retire_pending: bool, + } + impl Queue { + pub(super) fn owner(&self) -> Option { self.owner } + pub(super) fn retirement_polling(&self) -> bool { self.retire_pending } +PENDING + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Id { qid: u8 } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ComputeReservation { slot: usize, id: Id, owner: u64 } +struct ComputeEntry { reservation: ComputeReservation, queue: Option> } +struct QueueIds { free: bool, fail: Option } +impl QueueIds { + fn reserve(&mut self, _: u64, _: DataMaster, preferred: Option) -> Result { + if let Some(error) = self.fail { return Err(error); } + if !self.free { return Err(ENOSPC); } + self.free = false; + Ok(Id { qid: preferred.unwrap() }) + } +} +struct Identities { next: u64 } +impl Identities { + fn reserve(&mut self, count: u64) -> Result> { + let first = self.next; self.next += count; + Ok(first..self.next) + } +} +struct Registry { + compute: Vec>, + identities: Identities, + qids: QueueIds, +} +impl Registry { +METHODS + fn poll(&self) -> bool { + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref())PREDICATE + } +} +fn state(phase: PoolPhase, free: bool) -> Registry { + let mut entries = Vec::new(); + for slot in 0..4 { + entries.push(Some(ComputeEntry { + reservation: ComputeReservation { slot, id: Id { qid: slot as u8 }, owner: slot as u64 + 1 }, + queue: Some(Box::new(compute::Queue { owner: None, pool_phase: phase, retire_pending: false })), + })); + } + entries.push(None); + Registry { compute: entries, identities: Identities { next: 100 }, qids: QueueIds { free, fail: None } } +} +fn main() { + // Captured failure: four idle released queues, full physical namespace, + // control-release cursor still awaiting observation, no retirement event. + let mut registry = state(PoolPhase::ReleaseSent(3), false); + assert!(registry.poll(), "release needs a worker wake without new GPU events"); + for _ in 0..8 { + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "pending release is temporary admission backpressure"); + assert!(registry.compute[4].is_none(), "failed admission must not install a reservation"); + assert!(!registry.qids.free); + } + // Only the firmware witness may move the retained graph to vacant. + for entry in registry.compute.iter_mut().flatten() { + entry.queue.as_mut().unwrap().pool_phase = PoolPhase::Vacant; + } + assert!(!registry.poll(), "settled idle queues must stop polling"); + assert_eq!(registry.compute_admission_error(ENOSPC), ENOSPC); + for phase in [PoolPhase::Populated, PoolPhase::Vacant] { + let mut registry = state(phase, false); + assert!(!registry.poll()); + assert_eq!(registry.reserve_compute(200), Err(ENOSPC), "real exhaustion remains terminal"); + } + // Pending releases do not throttle spare physical capacity or hide errors. + let mut registry = state(PoolPhase::ReleaseSent(u32::MAX), true); + let reservation = registry.reserve_compute(200).unwrap(); + assert_eq!(reservation.slot, 4); + assert_eq!(reservation.id.qid, FIRST_COMPUTE_QID); + assert!(registry.compute[4].is_some()); + for error in [EINVAL, -5, -12, EAGAIN] { + let mut registry = state(PoolPhase::ReleaseSent(3), false); + registry.qids.fail = Some(error); + assert_eq!(registry.reserve_compute(200), Err(error)); + } + let mut registry = state(PoolPhase::Populated, false); + registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().retire_pending = true; + assert!(registry.poll(), "existing command-retirement polling remains enabled"); + println!("PASS captured four-queue transition, settled wake removal, full/free identity capacity and error preservation"); +} +'''.replace('PENDING', pending).replace('METHODS', methods).replace('PREDICATE', predicate) + + +def main(): + code = fixture() + with tempfile.TemporaryDirectory(prefix='g17-pool-release-') as directory: + root = Path(directory) + for mutation in ['fixed', 'no-wake', 'terminal-admission']: + current = code + if mutation == 'no-wake': + current = current.replace('queue.retirement_polling() || queue.pool_release_pending()', 'queue.retirement_polling()') + elif mutation == 'terminal-admission': + current = current.replace('.map_err(|error| self.compute_admission_error(error))?', '?') + path = root / (mutation + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if mutation == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + expected = 'release needs a worker wake' if mutation == 'no-wake' else 'pending release is temporary' + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', mutation) + + +if __name__ == '__main__': + main() From c3799227f43a50f8d109af59f72f64102df899bc Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:16:28 -0400 Subject: [PATCH 08/22] driver core: track devres node membership for direct removal Rust Devres drops remove an exact node, but devres_node_remove scans the entire device list to find it. Repeated removal of older GPU SG-table resources therefore does quadratic work while holding devres_lock. After large concurrent GPU workloads this keeps exiting clients CPU-bound for roughly a minute. Track the device-list owner in each node and remove a matching node directly under the device lock. Clear membership on every detach path, including replacement, ordinary and action removal, group removal, and moves to the private release-all/group callback list. A concurrent drop must leave nodes already owned by that release path alone. Preserve callback ordering and lifetime handling. Use single-copy membership accesses so a simultaneous comparison using a different device does not race through an ordinary pointer access. The membership field adds one pointer per resource node. Source-derived ASan/UBSan controls cover the removal APIs, replacement, open/closed/nested groups, private-list ownership and 1000 concurrent removal/detachment/different-device lookup races. Retaining stale private list ownership makes the negative control fail. The combined series passes Neo G17P concurrent GPU stress, expiry cycles and retained-VM context churn. Four clients performing 30 seconds of checked compute work finish and exit in 31.46 seconds total, versus 86.97 seconds with the same compact target layout before this correction. Signed-off-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/base/base.h | 2 ++ drivers/base/devres.c | 29 ++++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/drivers/base/base.h b/drivers/base/base.h index a5b7abc10ff02c..d16026fdb2ef00 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -222,6 +222,8 @@ typedef void (*dr_node_free_t)(struct devres_node *node); struct devres_node { struct list_head entry; + /* Device-list membership, cleared under devres_lock before detachment. */ + struct device *owner; dr_node_release_t release; dr_node_free_t free_node; const char *name; diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 9d9842fc5a19f9..26e5518b6482b3 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -41,6 +41,7 @@ void devres_node_init(struct devres_node *node, dr_node_free_t free_node) { INIT_LIST_HEAD(&node->entry); + WRITE_ONCE(node->owner, NULL); node->release = release; node->free_node = free_node; } @@ -156,6 +157,7 @@ static void add_dr(struct device *dev, struct devres_node *node) devres_log(dev, node, "ADD"); BUG_ON(!list_empty(&node->entry)); list_add_tail(&node->entry, &dev->devres_head); + WRITE_ONCE(node->owner, dev); } static void replace_dr(struct device *dev, @@ -164,6 +166,8 @@ static void replace_dr(struct device *dev, devres_log(dev, old, "REPLACE"); BUG_ON(!list_empty(&new->entry)); list_replace(&old->entry, &new->entry); + WRITE_ONCE(old->owner, NULL); + WRITE_ONCE(new->owner, dev); } /** @@ -367,18 +371,14 @@ EXPORT_SYMBOL_GPL(devres_get); bool devres_node_remove(struct device *dev, struct devres_node *node) { - struct devres_node *__node; - guard(spinlock_irqsave)(&dev->devres_lock); - list_for_each_entry_reverse(__node, &dev->devres_head, entry) { - if (__node == node) { - list_del_init(&node->entry); - devres_log(dev, node, "REM"); - return true; - } - } - - return false; + /* A release-all/group pass may own a detached node on its private list. */ + if (READ_ONCE(node->owner) != dev) + return false; + list_del_init(&node->entry); + WRITE_ONCE(node->owner, NULL); + devres_log(dev, node, "REM"); + return true; } /** @@ -405,6 +405,7 @@ void *devres_remove(struct device *dev, dr_release_t release, dr = find_dr(dev, release, match, match_data); if (dr) { list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr->data; } @@ -500,6 +501,7 @@ static int remove_nodes(struct device *dev, if (&node->entry == first) first = first->next; list_move_tail(&node->entry, todo); + WRITE_ONCE(node->owner, NULL); cnt++; } } @@ -530,7 +532,9 @@ static int remove_nodes(struct device *dev, * nodes are always before both. */ list_move_tail(&grp->node[0].entry, todo); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); } } @@ -685,7 +689,9 @@ void devres_remove_group(struct device *dev, void *id) grp = find_group(dev, id); if (grp) { list_del_init(&grp->node[0].entry); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); devres_log(dev, &grp->node[0], "REM"); } else WARN_ON(1); @@ -849,6 +855,7 @@ static struct devres_action *remove_action(struct device *dev, return ERR_PTR(-ENOENT); list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr; From aa5996f8c90dec3307aef6a8f45c11376ab4bdc3 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:16:28 -0400 Subject: [PATCH 09/22] drm/asahi: reduce minimum G17 target record backing The target-record stride already accounts for its computed layout and UAT alignment, but its 256 KiB minimum wastes backing on smaller targets. Use one 32 KiB TVB page as the minimum instead, keeping the computed used size, consumer bounds, growth handling and all sixteen generations unchanged. Queue capacities, batching and synchronization policy are unchanged. Source-derived controls validate 388800 layouts against the original capacity and consumer bounds; 114426 need less backing. At forty held GPU clients with 124 target objects on Neo, allocation extents fall from 272236544 to 44695552 bytes, saving 217 MiB for that workload. The compact layout passes changing-pixel and layered readbacks, target growth, concurrent render/compute/mixed stress, Vulkan multisample checks, compute pool expiry and 2000 retained-VM contexts. Brilliant Diamond also reaches its language menu around 30 FPS in a bounded emulator run; broader gameplay and application performance validation remain separate. Signed-off-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/asahi/hw/t8140.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f22..648d1ed9919b82 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -273,7 +273,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; From 13ddec8f7a4790ca9855a82038736dc0bcbc1b51 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:53 -0400 Subject: [PATCH 10/22] drm/asahi: keep G17 queue identity contention from failing clients G17 queue IDs never change engine once published (the driver has no firmware operation that uninstalls one), and the driver rebinds a physical render pair or compute queue only after its logical owner closes. A live logical queue therefore keeps its compute queue and up to two render pairs while idle. Several Vulkan devices with eight queues each (the synchronization multi_queue cases, six processes), or 50 held single-queue clients, exceed both pools with idle owners alone: - Render: 63 admission slots, two per owner. A new owner is refused in acquire_render_with_ids() (admission ENOSPC, or no QIDs for a fresh pair) and its accepted job fails at publication ("reported at g17/runtime/backend.rs:39"), even when the holders are only warming spares or finishing a job. - Compute: reserve_compute() fails and build_compute() returns ENOSPC from the SUBMIT ioctl, which is fatal to the client. - Compute could take any QID beyond a 16-identity render minimum. Those QIDs stay compute for the device lifetime, so open queues could be left without a render pair. Treat held identities as contention: - Compute takes any free QID beyond a render floor of one pair for each open logical queue (existing pairs included). Below that floor it only replaces the unpublished initial reservations, which no render pair can use, and cancels unused render reservations only beyond the floor. Contention-free clients keep one compute queue each, as before. - When neither a fresh QID nor a released queue exists, publication may move an idle installed compute queue of another live owner (no active, replayed, quarantined, exiting, growing or superseded state; populated pool large enough) through the existing bind_owner() path, preferring the same VM and the least recently used queue. It publishes under the same device mutex hold. Render surplus held by installed pairs never becomes compute capacity, so it does not prevent the takeover. The previous owner keeps its context and binds again at its next publication. - Submission no longer fails when no physical compute queue is free. It leaves binding to publication, which retries for at most the job timeout and then fails the packet like any other unpublished error. One binder per owner prevents a submission and a publication from giving the same owner two queues. - A refused render owner retries reclamation after each release for a bounded time instead of failing at once: unused warming first, then the least recently used idle installed pair of a live owner that keeps another installed pair. The donor claims no other pair while it is open, not even the one it gave away: its work-state channel ordinals stay bound to the pairs it already used, and the rebind displaces its graph with the context still open, so that graph stays retained for the donor's scheduler until its own release, armed only by its close. The recipient returns its empty reservations and ends with exactly that pair. - A VM keeps one parameter-buffer view for each of its graphs on a pair instead of one per pair. The displaced graph of an open queue keeps its view until that queue's release, so with one view per pair any other queue of the same VM (one DRM file with several queues) offered the freed pair could not install a graph there before the donor closed. - The render admission wait for a pick is bounded instead of waiting indefinitely where the scheduler watchdog cannot fail the job. - A compute queue rebound from a detached owner that closed without publishing now returns to service instead of keeping that owner, its QID and VM aliases until processor stop. Published identities keep their lifetime rules: no QID is cancelled or changes engine, and every rebind still requires the queue or pair to be idle with no pending firmware witness, as for a closed owner's queue. Without contention, publication is unchanged apart from a registry lookup under the already held device mutex, and no queue changes owner. The resource host tests cover the floor for open queues, initial-identity replacement, render forfeit guards and recency, compute takeover states, the unpublished close, 40 held plus 12 fresh clients sharing 20 compute queues with all QIDs leased and render surplus beyond the floor, the survivors of 47 render clients after closes restore that surplus, a compositor that gives its idle sibling to a held client and keeps drawing on its other pair while the recipient holds it and after it exits, and the other queues of a donor's VM installing and drawing on the freed pair; the close, admission, donor and shared-VM tests fail without their fixes. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/kick.rs | 133 +++- drivers/gpu/drm/asahi/g17/queue/admission.rs | 107 ++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 51 ++ .../drm/asahi/g17/queue/compute/retirement.rs | 7 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 8 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 13 + drivers/gpu/drm/asahi/g17/runtime.rs | 197 ++++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 60 ++ .../drm/asahi/g17/runtime/render/prepare.rs | 5 +- .../drm/asahi/g17/runtime/render/submit.rs | 51 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 50 +- tools/asahi/g17-pool-release-tests.py | 10 +- tools/asahi/g17-resource-tests.py | 749 +++++++++++++++++- 13 files changed, 1322 insertions(+), 119 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index e2a9bb643b892d..55c3ab317a9897 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -17,8 +17,18 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Minimum render-channel capacity protected from compute-only allocation. -const COMPUTE_RENDER_RESERVE: usize = 16; +/// Minimum render identities protected from compute-only allocation. +const MIN_RENDER_IDENTITIES: usize = 16; + +/// Render identities compute leaves free while `queues` logical queues are +/// open: one pair for each of them, existing pairs included. A published QID +/// never changes engine (no firmware operation uninstalls one) and a live owner +/// keeps its last render pair, so render capacity handed to compute could not +/// be recovered while those owners live. Compute takes any QID beyond this and +/// changes owners only when none remains. +fn render_floor(queues: usize) -> usize { + MIN_RENDER_IDENTITIES.max(queues.saturating_mul(2)).min(QID_COUNT - 2) +} /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -80,12 +90,33 @@ impl QueueIds { } /// Reuses only an exact owner/engine match. Preferred placement is for - /// bootstrapping; it does not reserve a partition for an engine. + /// bootstrapping; it does not reserve a partition for an engine. Compute + /// reservations use `reserve_compute`, which knows the open queues. pub(crate) fn reserve( &mut self, owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, 0) + } + + /// A compute reservation while `queues` logical queues are open. + pub(crate) fn reserve_compute( + &mut self, + owner: u64, + preferred: Option, + queues: usize, + ) -> Result { + self.reserve_for(owner, DataMaster::Compute, preferred, queues) + } + + fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -102,46 +133,38 @@ impl QueueIds { Err(EINVAL) }; } - let initial = if engine == DataMaster::Compute && self.leased == QID_COUNT { - self.initial_render - .iter() - .copied() - .enumerate() - .find_map(|(slot, id)| { - id.filter(|id| self.can_cancel_unpublished(*id)) - .map(|id| (slot, id)) - }) + // Existing render pairs already count towards the reserve. The initial + // reservations do not: no render pair can use them, so compute may + // replace an unpublished one instead of taking reserved capacity. + // Published identities always retain their original lifetime rules. + let initial = if engine == DataMaster::Compute { + let need = render_floor(queues).saturating_sub(self.pair_identities()); + if QID_COUNT - self.leased > need { + None + } else { + Some( + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + .ok_or(ENOSPC)?, + ) + } } else { None }; - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. - // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute { - let render = self - .entries - .iter() - .flatten() - .filter(|entry| { - matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) - }) - .count(); - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let replacement = usize::from(initial.is_some()); - if QID_COUNT - self.leased + replacement - <= COMPUTE_RENDER_RESERVE.saturating_sub(render - replacement) - { - return Err(ENOSPC); - } - } - let index = preferred - .map(usize::from) - .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) - .ok_or(ENOSPC)?; + let index = match initial { + Some((_, id)) => usize::from(id.qid), + None => preferred + .map(usize::from) + .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?, + }; let id = Id { qid: index as u8, owner, @@ -159,6 +182,36 @@ impl QueueIds { Ok(id) } + /// Tiling and fragment identities of render pairs, excluding the initial + /// reservations, which no pair uses. + fn pair_identities(&self) -> usize { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count(); + render.saturating_sub(self.initial_render.iter().flatten().count()) + } + + /// Render identities beyond the reserve for `queues` open queues. Only + /// these become compute capacity when an unused render reservation is + /// cancelled. + pub(crate) fn render_surplus(&self, queues: usize) -> usize { + self.pair_identities().saturating_sub(render_floor(queues)) + } + + /// A compute reservation would succeed now: a fresh QID beyond the render + /// reserve, or a replaceable initial reservation. + pub(crate) fn compute_available(&self, queues: usize) -> bool { + QID_COUNT - self.leased > render_floor(queues).saturating_sub(self.pair_identities()) + || self + .initial_render + .iter() + .flatten() + .any(|id| self.can_cancel_unpublished(*id)) + } + /// Ordered under the device mutex; no atomic operation on the submission path. pub(crate) fn next_compute_publication(&mut self) -> Result { self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 6e99160729034f..8b9fbd08d4c6fb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -46,6 +46,10 @@ struct InFlight { } const QUARANTINE_WAIT_MS: u32 = 1000; +/// Bound on waiting for a lease of this owner's own pairs (all at depth) or a +/// transiently borrowed free slot. Every awaited lease belongs to published +/// work, which retires or is failed by its watchdog well within this time. +const PICK_WAIT_MS: u32 = 4 * crate::g17::job::TIMEOUT_MS; struct State { active: u64, @@ -58,6 +62,13 @@ struct State { epoch: u64, footprints: KVec, next_ticket: u64, + /// Live owners that gave a sibling pair to a refused owner. They keep + /// their remaining pair and never receive another while they are open, + /// not even the one they gave away. Their work-state channel ordinals stay + /// bound to the pairs they published on, and the displaced graph of the + /// given pair stays retained for their scheduler until its own release: + /// their scheduler must not get a second graph on that pair before then. + forfeited: [u64; cfg::RENDER_SLOTS], } impl State { const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; @@ -73,6 +84,7 @@ impl State { epoch: 0, footprints: KVec::new(), next_ticket: 1, + forfeited: [0; cfg::RENDER_SLOTS], } } fn exhausted(&self) -> bool { @@ -159,7 +171,9 @@ impl State { Some(first) } fn claim_sibling(&mut self, owner: u64) { - if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER + || self.forfeited.contains(&owner) + { return; } let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; @@ -223,6 +237,64 @@ impl State { self.epochs[slot] = 0; true } + /// A refused owner without any pair takes the least recently used idle + /// pair of a live owner which keeps another usable pair. `candidates` are + /// the slots whose physical pair the device lock proved idle, with the + /// owner bound to that pair; `empty` are the slots without any physical + /// identities. The recipient gives back its empty, unborrowed slots and ends + /// with exactly the received pair. Returns the slot and the donor. + fn forfeit_sibling( + &mut self, + candidates: &[(u8, u64)], + empty: u64, + owner: u64, + ) -> Option<(u8, u64)> { + if owner == 0 + || (0..cfg::RENDER_SLOTS).any(|slot| { + self.owners[slot] == owner + && (empty & (1u64 << slot) == 0 + || self.refs[slot] != 0 + || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0) + }) + { + return None; + } + let record = self.forfeited.iter().position(|value| *value == 0)?; + let blocked = self.active | self.quarantined | self.kill_hold | self.released; + let (slot, donor) = candidates + .iter() + .copied() + .filter(|(slot, donor)| { + let slot = usize::from(*slot); + let bit = 1u64 << slot; + slot != 0 + && slot < cfg::RENDER_SLOTS + && *donor != 0 + && *donor != owner + && self.owners[slot] == *donor + && self.refs[slot] == 0 + && blocked & bit == 0 + && self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == *donor + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + }) + }) + .min_by_key(|(slot, _)| (self.epochs[usize::from(*slot)], *slot))?; + if !self.forfeited.contains(&donor) { + self.forfeited[record] = donor; + } + for other in 0..cfg::RENDER_SLOTS { + if self.owners[other] == owner { + self.owners[other] = 0; + self.released &= !(1u64 << other); + self.epochs[other] = 0; + } + } + self.owners[usize::from(slot)] = owner; + self.epochs[usize::from(slot)] = 0; + Some((slot, donor)) + } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { return None; @@ -362,6 +434,10 @@ impl Pool { GFP_KERNEL, ) } + /// Open logical queues, each of which may need a render pair. + pub(crate) fn queues(&self) -> usize { + self.queues.load(Ordering::Acquire) as usize + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { @@ -390,6 +466,7 @@ impl Pool { let mut state = self.state.lock(); let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let mut pick_budget = kernel::time::msecs_to_jiffies(PICK_WAIT_MS); let mut expired = false; let mut ticket = None; let result = (|| loop { @@ -454,8 +531,20 @@ impl Pool { } CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, } - } else if self.released.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); + } else { + // Never wait unboundedly in publication: the scheduler watchdog + // cannot fail a packet that has not claimed hardware yet. + if pick_budget == 0 { + return Err(EBUSY); + } + pick_budget = match self + .released + .wait_interruptible_timeout(&mut state, pick_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; } })(); state.forget_waiter(ticket); @@ -515,6 +604,15 @@ impl Pool { pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { self.state.lock().owners.get(slot) == Some(&owner) } + /// The caller holds the device mutex, which proved every candidate pair idle. + pub(crate) fn forfeit_sibling( + &self, + candidates: &[(u8, u64)], + empty: u64, + owner: u64, + ) -> Option<(u8, u64)> { + self.state.lock().forfeit_sibling(candidates, empty, owner) + } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); for slot in 0..cfg::RENDER_SLOTS { @@ -522,6 +620,9 @@ impl Pool { state.released |= 1u64 << slot; } } + for value in state.forfeited.iter_mut().filter(|value| **value == owner) { + *value = 0; + } drop(state); self.released.notify_all(); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 0593352863e76e..259a1f9f8e0524 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1140,6 +1140,17 @@ impl Queue { } self.retirement_ready = true; } + // An owner that rebound a detached queue and closed without + // publishing left no command for a witness to retire. Without + // this the queue keeps that owner, its QID and its VM aliases + // until processor stop. + if self.previous.is_none() + && self.replays.len == 0 + && self.exit.is_none() + && !self.growing + { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1147,6 +1158,46 @@ impl Queue { } Ok(false) } + /// Installed, populated and idle under a live owner: no retained, replayed, + /// quarantined, exiting, growing or superseded state. Publication on this + /// queue holds the device mutex, which the caller also holds, so no command + /// can become visible between this check and a takeover. + pub(crate) fn idle_owned(&self) -> bool { + self.owner.is_some() + && !self.released + && self.installed + && !self.growing + && self.pool_phase == PoolPhase::Populated + && !self.needs_fresh_backing + && self.exit.is_none() + && !self.quarantined + && !self.awaiting_witness + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.binding.is_some() + } + /// Releases an idle live owner exactly as its logical close would, so that + /// `bind_owner` can move the queue. The owner keeps its context and takes a + /// queue again before its next publication. + pub(crate) fn yield_idle_owner(&mut self) -> Result { + if !self.idle_owned() { + return Err(EBUSY); + } + self.released = true; + self.retirement_ready = true; + self.owner = None; + self.released_at = now_ns(); + Ok(()) + } + /// The bound owner's VM status, for preferring a same-VM takeover. + pub(crate) fn bound_status(&self) -> Option<&Arc> { + self.binding.as_ref().map(|binding| binding.context.status()) + } + /// GPU end time of the last retired command, for least-recent selection. + pub(crate) fn last_end(&self) -> u64 { + self.last_end + } /// The last owner's file and VM are gone: its execution root was released /// after the scheduler-state release and command retirement were witnessed /// (`Context::release_execution`), and this queue is idle without an owner. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b4d4bcef24421f..f5b68c67cf9354 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -156,10 +156,11 @@ pub(crate) struct Batch { } impl Batch { - pub(crate) fn new() -> Result { - Ok(Self { + /// Initialized in place: the slot table is too large to stage on the probe stack. + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, - slots: [(0, 0); QIDS], + slots <- pin_init::init_zeroed(), open: None, }) } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 569b5e289e7d8e..7b8d401cf8dd71 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -129,7 +129,13 @@ impl Queue { fn ensure_compute(&mut self, scs_pages: u16) -> Result { // Admission is checked even when this queue already owns a scheduler. - self.backend.ensure_compute(scs_pages)?; + // Physical queues held by other live owners (ENOSPC) and a concurrent + // binder of this owner (EAGAIN) are contention, not client errors: + // publication binds before the command becomes visible, bounded. + match self.backend.ensure_compute(scs_pages) { + Err(error) if error == ENOSPC || error == EAGAIN => {} + result => result?, + } if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 4290bd136b5dab..3a106dbd524729 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1623,6 +1623,19 @@ impl Pair { && previous_ready && self.idle_for_context(self.context())?) } + /// Idle under a live owner and reusable as soon as that owner is released: + /// no prepared, active, replayed, quarantined, displaced or unsettled + /// superseded state, so a successor needs no firmware witness first. + pub(crate) fn forfeitable(&self) -> Result { + let previous_ready = self + .previous + .as_ref() + .is_none_or(|p| p.memory.is_none() && self.context_update && !p.ready); + Ok(self.owner.is_some() + && self.displaced.is_none() + && previous_ready + && self.idle_for_context(self.context())?) + } /// Logical close leaves the physical channel identity and backing retained. pub(crate) fn release_owner(&mut self) { self.owner = None; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4e6a455d6b1ceb..b5af3577cab9ec 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -33,6 +33,15 @@ const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Whether a publication with no queue may take another live owner's idle one: +/// when neither a fresh QID nor surplus render capacity remains, or once its +/// own off-lock attempt (`contended`) found no released queue, fresh QID or +/// reclaimable render spare. Surplus held by installed render pairs never +/// becomes compute capacity, so it alone must not keep a publication waiting. +fn takeover_allowed(compute_available: bool, render_surplus: usize, contended: bool) -> bool { + contended || (!compute_available && render_surplus < 2) +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -93,7 +102,7 @@ impl Registry { compute_building: false, compute_scheduler_building: false, replay_outer_wait: None, - observations: compute::retirement::Batch::new()?, + observations <- compute::retirement::Batch::new(), worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) @@ -146,6 +155,77 @@ impl Registry { Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) } + /// No fresh QID and no released queue remain: move the least recently used + /// idle queue of another live owner to `owner`, preferring one bound to the + /// same VM. Only a pool that already fits is taken; growth would have to + /// wait for a publication of the new owner. The previous owner keeps its + /// context and takes a queue again at its next publication. + pub(super) fn take_over_idle_compute( + &mut self, + owner: u64, + context: &Arc, + required: usize, + ) -> Result { + if !context.is_current() { + return Err(EFAULT); + } + let mut selected = None::<(bool, u64, usize)>; + for (index, entry) in self.compute.iter().enumerate() { + let Some(queue) = entry.as_ref().and_then(|entry| entry.queue.as_deref()) else { + continue; + }; + if queue.owner() == Some(owner) || !queue.idle_owned() || !queue.pool_fits(required) { + continue; + } + let foreign = !queue + .bound_status() + .is_some_and(|status| Arc::ptr_eq(status, context.status())); + let key = (foreign, queue.last_end(), index); + if selected.is_none_or(|best| key < best) { + selected = Some(key); + } + } + let Some((_, _, index)) = selected else { + return Ok(false); + }; + let queue = self.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)?; + queue.yield_idle_owner()?; + queue.bind_owner(owner, context.clone())?; + Ok(true) + } + + /// Publication's device-locked binding: a released queue that fits, or a + /// takeover when neither a fresh QID nor surplus render capacity remains, + /// or once an off-lock attempt of this publication (`contended`) found no + /// released queue, fresh QID or reclaimable render spare. Render surplus + /// held by installed pairs cannot become compute capacity, so it must not + /// keep a publication from taking an idle queue. False leaves + /// construction, re-arming or growth to the off-lock path. + pub(super) fn rebind_or_take_over_compute( + &mut self, + owner: u64, + context: &Arc, + required: usize, + contended: bool, + ) -> Result { + match self.rebind_compute(owner, context, required)? { + Rebind::Done => Ok(true), + Rebind::Fresh + if takeover_allowed( + self.qids.compute_available(self.admission.queues()), + self.qids.render_surplus(self.admission.queues()), + contended, + ) => + { + self.take_over_idle_compute(owner, context, required) + } + _ => Ok(false), + } + } + /// Installs fresh USC backing into the vacant queue `qid`; the caller writes /// the returned descriptor row before binding. EAGAIN if the queue was taken. pub(super) fn rearm_compute( @@ -308,7 +388,7 @@ impl Registry { let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .reserve_compute(identity, Some(FIRST_COMPUTE_QID), self.admission.queues()) .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { @@ -393,6 +473,17 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + /// One compute binder per owner: submission and publication can both bind, + /// and two concurrent binders could leave the owner with two queues. + binding: core::sync::atomic::AtomicBool, +} + +/// Clears the owner's compute-binder flag on every exit. +struct Binder<'a>(&'a core::sync::atomic::AtomicBool); +impl Drop for Binder<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } } impl Backend { @@ -442,6 +533,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + binding: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, )?, @@ -609,9 +701,12 @@ impl Backend { } } + /// Off-lock binding: a released queue, a re-armed vacant one, growth, or a + /// fresh QID. ENOSPC leaves the owner unbound. Taking over another live + /// owner's idle queue happens only in publication's device-locked rebind, + /// which publishes under the same mutex hold. Submission never waits for + /// or fails on physical capacity: it leaves binding to publication. fn build_compute(&self, required: usize) -> Result { - const POOL_RELEASE_WAIT_NS: u64 = 200_000_000; - let mut pool_release_wait = None; loop { match self.shared_compute() { Err(error) if error == EAGAIN => continue, @@ -641,25 +736,30 @@ impl Backend { Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, + // A pool release is still in flight: the event worker + // observes it and drops the vacated backing off this + // mutex. The caller retries after it. Err(error) if error == EAGAIN => { - // The event worker owns release observation and - // drops vacated aliases/backing off this mutex. - let since = *pool_release_wait.get_or_insert(now_ns()); drop(state); - drop(lease); self.shared.queue_events(); - if now_ns().saturating_sub(since) >= POOL_RELEASE_WAIT_NS { - return Err(EBUSY); - } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; + return Err(ENOSPC); } Err(error) if error == ENOSPC => { - let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; - drop(state); + // Cancelling an unused render reservation frees + // compute capacity only beyond the render reserve. + let surplus = firmware + .queues + .qids + .render_surplus(firmware.queues.admission.queues()); + let reclaimed = if surplus >= 2 { + firmware.queues.reclaim_unused_render_spare(None, true)? + } else { + None + }; if reclaimed.is_none() { return Err(error); } + drop(state); drop(reclaimed); continue; } @@ -1093,9 +1193,24 @@ impl super::Firmware { } impl Backend { + /// Submission-time binding: never takes over another owner's queue and + /// reports ENOSPC when no physical queue is free, leaving publication to bind. fn ensure_compute(&self, scs_pages: u16) -> Result { let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; + self.bind_compute(required) + } + + /// EAGAIN while another binder of this owner runs; it binds for both. + fn bind_compute(&self, required: usize) -> Result { use kernel::time::{delay::fsleep, Delta}; + if self + .binding + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(EAGAIN); + } + let _binder = Binder(&self.binding); const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; for _ in 0..SPARED_WAIT_MS { @@ -1129,8 +1244,15 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Err(EINVAL); + }; + let required = + super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; let mut recovery_polls = 0; let mut outer_wait = None; + let mut binding_wait = None; + let mut contended = false; loop { if packet.completion.status().get() != 0 { return Err(EIO); @@ -1166,6 +1288,51 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; + if !firmware + .queues + .compute(self.owner) + .is_some_and(|queue| queue.pool_fits(required)) + { + // Submission found no free physical queue, or another owner + // took this idle one. Bind again; a takeover publishes under + // the same mutex hold. Capacity frees whenever any queue's + // published work retires or times out, so the wait is bounded. + let now = now_ns(); + let since = *binding_wait.get_or_insert(now); + if now.saturating_sub(since) >= WORK_RING_WAIT_NS { + return Err(EBUSY); + } + // An off-lock binder of this owner may hold a reservation or + // a queue being re-armed: binding here could give the owner a + // second queue. Its flag is raised before its first decision. + let taken = !self.binding.load(Ordering::Acquire) + && firmware.queues.compute(self.owner).is_none() + && firmware.queues.rebind_or_take_over_compute( + self.owner, + &self.context, + required, + contended, + )?; + if !taken { + drop(state); + drop(lease); + // Replays collected above belong to other packets; do not + // hold their fences while this one waits for a queue. + deferred.finish(); + match self.bind_compute(required) { + Ok(()) => {} + Err(error) if [ENOSPC, EBUSY, EAGAIN].contains(&error) => { + // No released queue, fresh QID or spare: the next + // locked attempt may take an idle queue over. + contended |= error != EAGAIN; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } + continue; + } + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; let result = firmware.publish_compute( self.owner, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 30da1b09eedfac..2f8beb932130df 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -303,6 +303,66 @@ impl Registry { Ok(None) } + /// Every admission slot is held by a live owner and no unused graph can + /// yield: move an idle installed pair from a live owner which keeps another + /// installed pair to `owner`. Only the owner's logical binding is released; + /// QIDs, backing and the old context's retained graph follow the existing + /// rebinding rules, and its scheduler is displaced as still live. + pub(super) fn forfeit_idle_render_sibling(&mut self, owner: u64) -> Result { + let mut candidates: KVec<(u8, u64)> = KVec::with_capacity(RENDER_SLOTS, GFP_KERNEL)?; + let mut empty = 0u64; + for slot in 1..RENDER_SLOTS { + let Some(entry) = self.render.entries[slot].as_ref() else { + empty |= 1u64 << slot; + continue; + }; + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + || entry.closed + || entry.parked.is_some() + { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + let Some(donor) = pair.owner() else { + continue; + }; + if donor == owner || donor != entry.reservation.owner || !pair.forfeitable()? { + continue; + } + let installed = self + .render + .entries + .iter() + .flatten() + .filter(|other| { + other.pair.as_ref().is_some_and(|pair| pair.owner() == Some(donor)) + }) + .count(); + if installed < 2 { + continue; + } + candidates + .push_within_capacity((slot as u8, donor)) + .map_err(|_| EIO)?; + } + let Some((slot, _)) = self.admission.forfeit_sibling(&candidates, empty, owner) else { + return Ok(false); + }; + self.render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .release_owner(); + Ok(true) + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 14c4ce8c90754d..0ed99ed04552de 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -463,6 +463,9 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // A pair forfeited by a live owner still names that owner's + // scheduler; only its logical close may arm the release. + let owner_closed = entry.closed; if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { @@ -470,7 +473,7 @@ impl super::super::Backend { } firmware.queues.teardown.displace( pair.context(), - true, + owner_closed, super::super::now_ns(), )?; pair.bind_owner( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index abea0b4c06f290..730d69059908cb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,7 +14,25 @@ use kernel::{ time::{delay::fsleep, Delta}, }; +/// Bound on waiting for render capacity held by other live owners. Capacity +/// frees when a warming graph finishes construction, when a sibling's published +/// work retires or times out, or when an owner closes. +const RENDER_CAPACITY_WAIT_NS: u64 = 4 * crate::g17::job::TIMEOUT_MS as u64 * 1_000_000; + impl Backend { + /// Every slot or QID is held by live owners whose spares are still being + /// built or whose siblings are busy. Retry reclamation after their next + /// release instead of failing the packet; fail only after the bound. + fn wait_render_capacity(&self, since: &mut Option) -> Result { + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= RENDER_CAPACITY_WAIT_NS { + return Err(ENOSPC); + } + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + Ok(()) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -35,6 +53,7 @@ impl Backend { let state = self.shared.state.lock(); (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() }; + let mut capacity_wait = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -48,16 +67,20 @@ impl Backend { match pool.acquire_render(self.owner, Request::of(packet)) { Ok(lease) => lease, Err(error) if error == ENOSPC => { - let reclaimed = { + // Live owners hold every slot. Prefer unused warming, + // then an idle sibling of an owner keeping another pair. + let (reclaimed, forfeited) = { let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .reclaim_unused_render_spare(Some(self.owner), false)? + let queues = &mut (*state).as_deref_mut().ok_or(ENODEV)?.queues; + let reclaimed = + queues.reclaim_unused_render_spare(Some(self.owner), false)?; + let forfeited = reclaimed.is_none() + && queues.forfeit_idle_render_sibling(self.owner)?; + (reclaimed, forfeited) }; - if reclaimed.is_none() { - return Err(error); + if reclaimed.is_none() && !forfeited { + self.wait_render_capacity(&mut capacity_wait)?; + continue; } drop(reclaimed); continue; @@ -83,15 +106,23 @@ impl Backend { return Err(error); } if !lease.cancel_empty(self.owner) { + drop(state); + self.wait_render_capacity(&mut capacity_wait)?; continue; } let reclaimed = firmware .queues .reclaim_unused_render_spare(Some(self.owner), true)?; let ready = firmware.queues.has_render_ids(self.owner); + // Every remaining QID belongs to an installed graph: take an + // idle installed sibling, whose identities move with it. + let forfeited = reclaimed.is_none() + && !ready + && firmware.queues.forfeit_idle_render_sibling(self.owner)?; drop(state); - if reclaimed.is_none() && !ready { - return Err(error); + if reclaimed.is_none() && !ready && !forfeited { + self.wait_render_capacity(&mut capacity_wait)?; + continue; } drop(reclaimed); } diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 98006b9459c98b..cc81e78e97f355 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -19,7 +19,11 @@ pub(super) struct DriverMappings { mappings: KVec, reserved: KVec>, compute: Option, - render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], + /// Parameter-buffer views by render pair, one for each graph of this VM + /// that keeps one: a graph displaced from an open queue keeps its view + /// until that queue's scheduler release, while another queue of the same + /// VM installs its own graph on the pair. + render: [KVec; crate::hw::t8140::queues::RENDER_SLOTS], } struct PoolMappings { @@ -37,7 +41,7 @@ impl DriverMappings { self.compute.iter().flat_map(|pool| pool.mappings.iter()), self.render .iter() - .flatten() + .flat_map(|views| views.iter()) .flat_map(|pool| pool.mappings.iter()), ), ) @@ -143,7 +147,7 @@ impl Vm { mappings, reserved, compute: None, - render: core::array::from_fn(|_| None), + render: core::array::from_fn(|_| KVec::new()), }); Ok(()) } @@ -171,12 +175,13 @@ impl Vm { .as_ref()? .render .get(usize::from(slot))? - .as_ref() - .filter(|pool| pool.id == id) + .iter() + .find(|pool| pool.id == id) .map(|pool| pool.blocks) } - /// Retains a new generation after all references to a replaced pool retire. + /// Retains the view of a new graph on pair `slot`. Views of other graphs + /// of this VM on the pair stay until their own graphs are dropped. pub(crate) fn install_render_pool_mappings( &self, slot: u8, @@ -190,16 +195,20 @@ impl Vm { let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; let mut driver = lifetime.driver.lock(); let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; - let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; - if pool.is_some() { + let views = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if views.iter().any(|pool| pool.id == id) { return Err(EBUSY); } - *pool = Some(PoolMappings { - id, - blocks, - mappings, - }); - Ok(()) + // Reserve first: no refusal drops the aliases while the lock is held, + // since unmapping them takes GPUVM locks. + views.reserve(1, GFP_KERNEL)?; + views + .push_within_capacity(PoolMappings { + id, + blocks, + mappings, + }) + .map_err(|_| EIO) } /// Consumes aliases only after the exact cached owner and complete append @@ -220,8 +229,7 @@ impl Vm { let mut driver = lifetime.driver.lock(); let pool = Option::as_mut(&mut *driver) .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .and_then(Option::as_mut) - .filter(|pool| pool.id == id) + .and_then(|views| views.iter_mut().find(|pool| pool.id == id)) .ok_or(EINVAL)?; if pool.blocks != previous { return Err(EBUSY); @@ -258,8 +266,7 @@ impl Vm { let mut driver = lifetime.driver.lock(); let pool = Option::as_mut(&mut *driver) .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .and_then(Option::as_mut) - .filter(|pool| pool.id == id) + .and_then(|views| views.iter_mut().find(|pool| pool.id == id)) .ok_or(EINVAL)?; if pool.blocks != blocks || pool.mappings.len() < count { return Err(EBUSY); @@ -280,10 +287,9 @@ impl Vm { } let previous = self.lifetime.as_ref().and_then(|lifetime| { let mut driver = lifetime.driver.lock(); - Option::as_mut(&mut *driver) - .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .filter(|pool| pool.as_ref().is_some_and(|pool| pool.id == id)) - .and_then(Option::take) + let views = Option::as_mut(&mut *driver)?.render.get_mut(usize::from(slot))?; + let index = views.iter().position(|pool| pool.id == id)?; + views.remove(index).ok() }); drop(previous); } diff --git a/tools/asahi/g17-pool-release-tests.py b/tools/asahi/g17-pool-release-tests.py index f8fcee86d9ed17..20039f9cc753ea 100644 --- a/tools/asahi/g17-pool-release-tests.py +++ b/tools/asahi/g17-pool-release-tests.py @@ -71,6 +71,13 @@ def fixture(): self.free = false; Ok(Id { qid: preferred.unwrap() }) } + fn reserve_compute(&mut self, owner: u64, preferred: Option, _queues: usize) -> Result { + self.reserve(owner, DataMaster::Compute, preferred) + } +} +struct Admission; +impl Admission { + fn queues(&self) -> usize { 0 } } struct Identities { next: u64 } impl Identities { @@ -83,6 +90,7 @@ def fixture(): compute: Vec>, identities: Identities, qids: QueueIds, + admission: Admission, } impl Registry { METHODS @@ -99,7 +107,7 @@ def fixture(): })); } entries.push(None); - Registry { compute: entries, identities: Identities { next: 100 }, qids: QueueIds { free, fail: None } } + Registry { compute: entries, identities: Identities { next: 100 }, qids: QueueIds { free, fail: None }, admission: Admission } } fn main() { // Captured failure: four idle released queues, full physical namespace, diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 73d5fa52b98333..88931b1894e263 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -28,7 +28,7 @@ def item(start): macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } type KVec = Vec; struct InFlight; mod cfg {\n'''+constants+'}\n'+limits+'\n' - methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn forfeit_sibling','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' fn main(){ let mut state=State::new(); @@ -100,6 +100,72 @@ def item(start): assert!(state.refusal(33).is_ok()); assert_eq!(state.acquire(33),Some(slot as u8)); println!("control: explicitly released idle ownership admits a new owner"); + // Live owners whose siblings were published hold every slot: no spare + // can yield. The runtime proves each candidate pair idle and installed. + let candidates=|s:&State|->Vec<(u8,u64)> {(1..cfg::RENDER_SLOTS).filter(|slot|s.owners[*slot]!=0).map(|slot|(slot as u8,s.owners[slot])).collect()}; + let mut live=State::new(); + for owner in 1..=32 {let slot=live.acquire(owner).unwrap();live.release(slot);} + assert_eq!(live.refusal(33),Err(ENOSPC)); + let mut gifts=Vec::new(); + for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { + assert_eq!(live.refusal(owner),Err(ENOSPC)); + let (slot,donor)=live.forfeit_sibling(&candidates(&live),0,owner).expect("an owner with two pairs donates one"); + assert_eq!(live.owners[slot as usize],owner);assert_eq!(live.epochs[slot as usize],0); + assert_eq!(live.owners.iter().filter(|o|**o==donor).count(),1,"donor keeps exactly one pair"); + assert_eq!(live.acquire(owner),Some(slot));live.release(slot); + gifts.push((slot as usize,donor)); + } + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + assert!(live.refusal(owner).is_ok());let slot=live.acquire(owner).unwrap();live.release(slot); + } + println!("forfeit: all{} live logical owners render while idle siblings move; every donor keeps a pair",MAX_EXECUTION_CONTEXTS); + // A donor never receives another pair while open: neither free capacity + // that appears later nor the pair it gave away once its recipient closes. + let (given,donor)=gifts[0]; + let kept=live.owners.iter().position(|o|*o==donor).unwrap(); + let closed=live.owners.iter().position(|o|*o==u64::from(MAX_EXECUTION_CONTEXTS)).unwrap(); + assert_ne!(given,closed,"control: the released pair was not this donor's"); + live.released|=1u64<Vec {(0..cfg::RENDER_SLOTS).filter(|s|g.owners[*s]==o).collect()}; + let (a,b)=(pairs(&g,1)[0],pairs(&g,1)[1]); + g.refs[b]=1;assert_eq!(g.forfeit_sibling(&[(b as u8,1)],0,9),None,"borrowed");g.refs[b]=0; + g.active|=1u64< = std::result::Result; const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; @@ -123,58 +190,88 @@ def item(s,start): #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} #[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} - '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name+'(')+'\n' for name in ['reserve_initial_render','reserve','publish','can_cancel_unpublished','cancel_unpublished'])+'}\n' + '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,name+'(')+'\n' for name in ['pub(crate) fn reserve_initial_render','pub(crate) fn reserve','pub(crate) fn reserve_compute','fn reserve_for','fn pair_identities','pub(crate) fn render_surplus','pub(crate) fn compute_available','pub(crate) fn publish','pub(crate) fn can_cancel_unpublished','pub(crate) fn cancel_unpublished'])+'}\n' test=''' fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} fn main(){ - for render in [0,2,16,32,112,126] { - let mut q=fresh(); - for i in 0..render {q.reserve(i as u64+1,if i%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();} - let mut compute=0; - while q.reserve(1000+compute,DataMaster::Compute,None).is_ok(){compute+=1;} - assert_eq!(compute as usize,QID_COUNT-render.max(COMPUTE_RENDER_RESERVE)); + let floor=|queues:usize| MIN_RENDER_IDENTITIES.max(2*queues).min(QID_COUNT-2); + assert_eq!(render_floor(0),16);assert_eq!(render_floor(26),52);assert_eq!(render_floor(58),116);assert_eq!(render_floor(1000),QID_COUNT-2); + let mut cases=0; + for queues in [0usize,8,20,26,54,58] { + for render in [0usize,2,12,16,32,52,108,116,126] { + let mut q=fresh(); + for i in 0..render {q.reserve(i as u64+1,if i%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();} + assert_eq!(q.render_surplus(queues),render.saturating_sub(floor(queues))); + let mut compute=0; + loop { + let available=q.compute_available(queues); + let result=q.reserve_compute(1000+compute,None,queues); + assert_eq!(available,result.is_ok(),"availability predicts admission"); + if result.is_err() {assert_eq!(result,Err(ENOSPC));break;} + compute+=1; + } + assert_eq!(compute as usize,QID_COUNT-render.max(floor(queues))); + // Render keeps a pair for every open logical queue. + let mut pairs=render; + while q.reserve(5000+pairs as u64,DataMaster::Tiling,None).is_ok() {pairs+=1;} + assert!(pairs>=floor(queues),"render capacity {} below {} open queues",pairs,queues); + cases+=1; + } } + // compute20: 20 compute clients beside a desktop of six owners with pairs. + let mut q=fresh();q.reserve_initial_render().unwrap(); + for owner in 3..15 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + for owner in 100..126 {q.reserve_compute(owner,Some(4),26).unwrap();} + assert!(q.compute_available(26),"26 compute queues while QIDs are free: no sharing"); + println!("queue IDs: {} open-queue/render/compute capacity boundaries; a render pair for every open queue; contention-free compute keeps one QID per owner",cases); let mut q=fresh();let first=q.reserve(1,DataMaster::Tiling,None).unwrap(); assert!(q.can_cancel_unpublished(first));q.publish(first).unwrap(); assert!(!q.can_cancel_unpublished(first));assert_eq!(q.cancel_unpublished(first),Err(EINVAL));assert_eq!(q.leased,1); let unused=q.reserve(2,DataMaster::Fragment,None).unwrap();q.cancel_unpublished(unused).unwrap();assert_eq!(q.leased,1); let next=q.reserve(3,DataMaster::Fragment,None).unwrap();assert_eq!(next.qid,unused.qid); assert!(!q.can_cancel_unpublished(unused));assert_eq!(q.cancel_unpublished(unused),Err(EINVAL)); - println!("queue IDs: six render/compute capacity boundaries; compute-only render reserve retained; published and stale identities protected"); + println!("queue IDs: published and stale identities protected"); // Reproduce the hardware snapshot: 124 installed render identities, two // installed compute identities, and two initial reservations fill128 IDs. let mut q=fresh();q.reserve_initial_render().unwrap(); assert_eq!(q.reserve_initial_render(),Err(EINVAL)); let initial=q.initial_render; - for owner in 1000..1002 {let id=q.reserve(owner,DataMaster::Compute,Some(4)).unwrap();q.publish(id).unwrap();} + for owner in 1000..1002 {let id=q.reserve_compute(owner,Some(4),0).unwrap();q.publish(id).unwrap();} let mut installed=Vec::new(); for owner in 3..127 {let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();q.publish(id).unwrap();installed.push(id);} assert_eq!(q.leased,QID_COUNT); for (n,old) in initial.into_iter().enumerate() { - let old=old.unwrap();let id=q.reserve(2000+n as u64,DataMaster::Compute,None).unwrap(); + let old=old.unwrap();let id=q.reserve_compute(2000+n as u64,None,0).unwrap(); assert_eq!(id.qid,old.qid);assert_eq!(q.leased,QID_COUNT); assert!(!q.can_cancel_unpublished(old));assert_eq!(q.cancel_unpublished(old),Err(EINVAL)); q.publish(id).unwrap(); } - assert_eq!(q.reserve(3000,DataMaster::Compute,None),Err(ENOSPC)); + assert_eq!(q.reserve_compute(3000,None,0),Err(ENOSPC)); for id in installed {assert!(!q.can_cancel_unpublished(id));assert_eq!(q.entries[id.qid as usize].unwrap().id,id);} // A bootstrap token that became published is never replaced. let mut q=fresh();q.reserve_initial_render().unwrap();let protected=q.initial_render[0].unwrap();q.publish(protected).unwrap(); for owner in 3..129 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} assert_eq!(q.leased,QID_COUNT); - assert_eq!(q.reserve(1000,DataMaster::Compute,None).unwrap().qid,1); - assert_eq!(q.reserve(1001,DataMaster::Compute,None),Err(ENOSPC)); + assert_eq!(q.reserve_compute(1000,None,0).unwrap().qid,1); + assert_eq!(q.reserve_compute(1001,None,0),Err(ENOSPC)); assert_eq!(q.entries[0].unwrap().id,protected); - // Replacement must not consume the protected render minimum. - let mut q=fresh();q.reserve_initial_render().unwrap(); - for owner in 3..17 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} - for owner in 1000..1112 {q.reserve(owner,DataMaster::Compute,None).unwrap();} - assert_eq!(q.leased,QID_COUNT);assert_eq!(q.reserve(2000,DataMaster::Compute,None),Err(ENOSPC)); - assert!(q.initial_render.iter().all(Option::is_some)); + // Compute stops at the render reserve, then replaces only the bootstrap + // identities no render pair can use; the reserve itself stays free. + for queues in [0usize,40,58] { + let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; + for owner in 3..17 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + let mut compute=0; + while q.reserve_compute(1000+compute,None,queues).is_ok() {compute+=1;} + assert_eq!(compute as usize,QID_COUNT-2-render_floor(queues)+2); + assert!(q.initial_render.iter().all(Option::is_none)); + for old in initial.into_iter().flatten() {assert_eq!(q.entries[old.qid as usize].unwrap().id.engine,DataMaster::Compute);} + assert_eq!(QID_COUNT-q.leased,render_floor(queues)-14,"render can still install a pair for every open queue"); + assert!(!q.compute_available(queues)); + } // Ordinary free capacity leaves the initialization reservations untouched. let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; - let id=q.reserve(1000,DataMaster::Compute,Some(4)).unwrap();assert_eq!(id.qid,4);assert_eq!(q.initial_render,initial); - println!("initial reservations: full-namespace compute admission restored; physical count unchanged; published and stale identities plus render minimum protected"); + let id=q.reserve_compute(1000,Some(4),0).unwrap();assert_eq!(id.qid,4);assert_eq!(q.initial_render,initial); + println!("initial reservations: replaced only beyond the open-queue render floor; physical count unchanged; published and stale identities plus that floor protected"); } ''' @@ -207,6 +304,79 @@ def item(s,start): subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) subprocess.run([str(root/'unused-render-guard-controls')],check=True) +def compute_takeover_tests(root, source): + s=(source/'g17/queue/compute.rs').read_text() + def item(start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + methods='\n'.join(item('pub(crate) fn '+name+'(') for name in ['release_owner','idle_owned','yield_idle_owner','reusable']) + fixture=root/'compute-takeover-controls.rs' + fixture.write_text('''#![allow(dead_code)] +type Result = std::result::Result; +const EBUSY:i32=-16;const EIO:i32=-5; +fn now_ns()->u64 {42} +#[derive(Clone,Copy,PartialEq,Eq,Debug)] enum PoolPhase {Populated,ReleaseSent(u32),Vacant} +struct Fifo {len:usize} +#[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Binding {context:u32} +struct Previous {binding:Option,context:u32,publication:Option<(u64,u64)>} +struct Graph {owner:u32} +impl Graph {fn set_owner(&mut self,context:&u32)->Result {self.owner=*context;Ok(())}} +struct Queue {owner:Option,released:bool,installed:bool,growing:bool,pool_phase:PoolPhase, + needs_fresh_backing:bool,exit:Option<()>,quarantined:bool,awaiting_witness:bool,active:Fifo,replays:Fifo, + previous:Option,binding:Option,retirement_ready:bool,released_at:u64, + retired_by_teardown:bool,graph:Graph} +impl Queue { +'''+methods+''' +} +fn idle()->Queue {Queue {owner:Some(1),released:false,installed:true,growing:false,pool_phase:PoolPhase::Populated, + needs_fresh_backing:false,exit:None,quarantined:false,awaiting_witness:false,active:Fifo {len:0},replays:Fifo {len:0}, + previous:None,binding:Some(Binding {context:1}),retirement_ready:true,released_at:0,retired_by_teardown:false,graph:Graph {owner:1}}} +fn main(){ + let mut q=idle();assert!(q.idle_owned());assert!(!q.reusable()); + q.yield_idle_owner().unwrap(); + assert!(q.reusable(),"a yielded queue is exactly a closed owner's reusable queue"); + assert_eq!((q.owner,q.released,q.retirement_ready,q.released_at),(None,true,true,42)); + assert_eq!(q.binding,Some(Binding {context:1}),"the old binding stays until bind_owner supersedes it"); + // A queue bound at submission whose packet has not published is also idle. + let mut q=idle();q.retirement_ready=false;assert!(q.idle_owned());q.yield_idle_owner().unwrap();assert!(q.reusable()); + let cases:[(&str,fn(&mut Queue));13]=[ + ("released",|q|q.released=true),("never installed",|q|q.installed=false),("growing",|q|q.growing=true), + ("pool release",|q|q.pool_phase=PoolPhase::ReleaseSent(1)),("vacant",|q|q.pool_phase=PoolPhase::Vacant), + ("fresh backing",|q|q.needs_fresh_backing=true),("exit",|q|q.exit=Some(())),("quarantined",|q|q.quarantined=true), + ("witness",|q|q.awaiting_witness=true),("active",|q|q.active.len=1),("replay",|q|q.replays.len=1), + ("superseded",|q|q.previous=Some(Previous {binding:None,context:0,publication:None})),("detached",|q|q.binding=None), + ]; + for (name,change) in cases { + let mut q=idle();change(&mut q); + assert!(!q.idle_owned(),"{} must not be taken over",name); + let before=(q.owner,q.released,q.retirement_ready,q.released_at); + assert_eq!(q.yield_idle_owner(),Err(EBUSY)); + assert_eq!((q.owner,q.released,q.retirement_ready,q.released_at),before,"{}: refusal changes nothing",name); + } + let mut q=idle();q.owner=None;assert!(!q.idle_owned(),"no owner"); + println!("compute takeover: idle live queue yields into reusable state; 14 busy/transitional states refused unchanged"); + // An owner that rebound a detached queue and closed without publishing. + let mut q=idle();q.retirement_ready=false;assert_eq!(q.release_owner(),Ok(false)); + assert_eq!(q.owner,None);assert!(q.reusable(),"QID returns to service at close"); + // Controls: retained work, a superseded binding and a pending replay keep the owner. + let mut q=idle();q.retirement_ready=false;q.active.len=1;q.release_owner().unwrap();assert_eq!(q.owner,Some(1)); + let mut q=idle();q.retirement_ready=false;q.replays.len=1;q.release_owner().unwrap();assert_eq!(q.owner,Some(1)); + let mut q=idle();q.retirement_ready=false;q.exit=Some(());q.release_owner().unwrap();assert_eq!(q.owner,Some(1)); + let mut q=idle();q.retirement_ready=false;q.binding=Some(Binding {context:2}); + q.previous=Some(Previous {binding:Some(Binding {context:1}),context:1,publication:None}); + q.release_owner().unwrap();assert_eq!((q.owner,q.binding,q.graph.owner),(None,Some(Binding {context:1}),1),"unpublished successor reverts"); + let mut q=idle();q.retirement_ready=false; + q.previous=Some(Previous {binding:None,context:1,publication:Some((1,1))}); + q.release_owner().unwrap();assert_eq!(q.owner,Some(1),"published successor waits for its witness"); + let mut q=idle();q.installed=false;q.retirement_ready=false;assert_eq!(q.release_owner(),Ok(true),"never installed: cancel QID"); + println!("compute close: unpublished rebinding returns its QID; work, replay, exit and superseded controls retained"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'compute-takeover-controls')],check=True) + subprocess.run([str(root/'compute-takeover-controls')],check=True) + def shared_alias_tests(root, source): s=(source/'g17/object.rs').read_text() at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 @@ -350,8 +520,541 @@ def shared_alias_tests(root, source): subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'shared-alias-controls')],check=True) subprocess.run([str(root/'shared-alias-controls')],check=True) +def compute_admission_tests(root, source): + """held40 on hardware: 40 held clients and 12 fresh ones beside a desktop, + with every QID leased (108 render identities, 20 compute queues). Render + surplus beyond the open-queue floor cannot become compute capacity, so a + client without a queue must be able to take an idle one over.""" + g17 = source / "g17" + def item(s,start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + s=(g17/'kick.rs').read_text() + fw=(g17/'fw/kick.rs').read_text() + runtime=(g17/'runtime.rs').read_text() + constants=re.search(r'const QID_MAX: [^;]+;',fw).group(0)+'\n'+'\n'.join(re.search(r'const '+name+r': [^;]+;',s).group(0) for name in ['QID_COUNT','MIN_RENDER_IDENTITIES']) + constants+='\n'+item(s,'fn render_floor(') + header='''#![allow(dead_code, unused_macros)] +type Result = std::result::Result; +const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; +macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } +#[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} +#[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} +#[derive(Clone,Copy)] struct Identity {id:Id,published:bool} +'''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,name+'(')+'\n' for name in ['pub(crate) fn reserve','pub(crate) fn reserve_compute','fn reserve_for','fn pair_identities','pub(crate) fn render_surplus','pub(crate) fn compute_available','pub(crate) fn publish','pub(crate) fn can_cancel_unpublished'])+'}\n'+item(runtime,'fn takeover_allowed(')+'\n' + test=''' +fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} +// Physical compute queues: their bound owner, or none once released. +struct Queues {owners:Vec>,takeovers:usize} +// One publication: the locked rebind or takeover, then the off-lock attempt, +// as compute_packet alternates them. True once the client has a queue. +fn publish(q:&mut QueueIds,queues:&mut Queues,client:u64,open:usize)->bool { + let mut contended=false; + for _ in 0..3 { + if queues.owners.contains(&Some(client)) {return true;} + if let Some(slot)=queues.owners.iter_mut().find(|owner|owner.is_none()) {*slot=Some(client);return true;} + if takeover_allowed(q.compute_available(open),q.render_surplus(open),contended) { + // Every other owner's queue is idle between its iterations. + if let Some(slot)=queues.owners.iter_mut().find(|owner|owner.is_some_and(|o|o!=client)) { + *slot=Some(client);queues.takeovers+=1;return true; + } + } + match q.reserve_compute(10_000+client,None,open) { + Ok(id)=>{q.publish(id).unwrap();queues.owners.push(Some(client));return true;} + Err(error)=>{assert_eq!(error,ENOSPC);contended=true;} + } + } + false +} +fn main(){ + let mut q=fresh(); + for owner in 1..=108u64 { + let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap(); + q.publish(id).unwrap(); + } + let mut queues=Queues {owners:Vec::new(),takeovers:0}; + for n in 0..20u64 {let id=q.reserve_compute(1000+n,None,0).unwrap();q.publish(id).unwrap();queues.owners.push(None);} + assert_eq!(q.leased,QID_COUNT); + let desktop=11; + let mut open=desktop+40; + assert!(!q.compute_available(open)&&q.render_surplus(open)>=2,"snapshot: no fresh QID, render surplus beyond the floor"); + for round in 0..10 { + for client in 1..=40u64 { + assert!(publish(&mut q,&mut queues,client,open),"held client {} could not bind in round {}",client,round); + } + } + for fresh_client in 41..=52u64 { + open+=1; + for _ in 0..3 { + assert!(publish(&mut q,&mut queues,fresh_client,open),"fresh client {} could not bind",fresh_client); + for client in 1..=40u64 {assert!(publish(&mut q,&mut queues,client,open),"held client {} could not bind again",client);} + } + for owner in queues.owners.iter_mut().filter(|owner|**owner==Some(fresh_client)) {*owner=None;} + open-=1; + } + assert_eq!(queues.owners.len(),20,"no QID changed engine"); + assert!(queues.takeovers>0); + let held=queues.takeovers; + // render50: 47 clients fill the 58 contexts, so the render floor leaves no + // surplus. Eight fail render capacity and close; the floor drops and the + // surplus returns while the 39 others still need compute queues. + let mut q=fresh(); + for owner in 1..=108u64 { + let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap(); + q.publish(id).unwrap(); + } + let mut queues=Queues {owners:Vec::new(),takeovers:0}; + for n in 0..20u64 {let id=q.reserve_compute(1000+n,None,0).unwrap();q.publish(id).unwrap();queues.owners.push(None);} + let mut open=desktop+47; + assert!(q.render_surplus(open)<2,"58 open queues: the floor covers every render identity"); + for client in 1..=47u64 {assert!(publish(&mut q,&mut queues,client,open),"render50 client {} could not bind",client);} + for client in 40..=47u64 {for owner in queues.owners.iter_mut().filter(|owner|**owner==Some(client)) {*owner=None;}} + open-=8; + assert!(q.render_surplus(open)>=2&&!q.compute_available(open),"closes leave render surplus and no fresh QID"); + // Every survivor publishes while each other one holds a queue. + for round in 0..5 { + for client in 1..=39u64 { + assert!(publish(&mut q,&mut queues,client,open),"render50 survivor {} could not bind in round {}",client,round); + } + } + println!("compute admission: held40 snapshot, 40 held and 12 fresh clients share 20 queues by takeover ({} takeovers); render50 survivors keep binding after closes restore render surplus ({} takeovers); none starves",held,queues.takeovers); +} +''' + code=header+test + for name,old,new in [('fixed',None,None),('render-surplus-gate', + 'contended || (!compute_available && render_surplus < 2)', + '(!compute_available && render_surplus < 2)')]: + current=code + if old is not None: + assert current.count(old)==1, name + current=current.replace(old,new) + fixture=root/('compute-admission-'+name+'.rs');fixture.write_text(current) + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(fixture.with_suffix(''))],check=True) + result=subprocess.run([str(fixture.with_suffix(''))],capture_output=True,text=True) + if old is None: + assert result.returncode==0,result.stderr + print(result.stdout.strip()) + else: + assert result.returncode!=0 and 'could not bind' in result.stderr,result.stderr + print('PASS negative control:',name) + + +def forfeit_donor_tests(root, source): + """held40 with the compositor as a forfeit donor. Under full slot pressure + a refused held client takes the compositor's least recently used sibling; + the compositor keeps rendering two overlapping commands every frame, while + the recipient holds the pair and after it closes. Runs the actual admission + State; the registry side is modelled from the rules checked below.""" + g17 = source / 'g17' + def item(s, start): + at = s.index(start); begin = s.index('{', at); depth = 1; end = begin + 1 + while depth: + depth += (s[end] == '{') - (s[end] == '}'); end += 1 + return s[at:end] + def body(s, start): + return item(s, start) + # The modelled rules, anchored in the driver source. + pair = (g17 / 'queue/render.rs').read_text() + assert re.search(r'if foreign \{\s*self\.displaced = Some\(retired\);', + body(pair, 'pub(crate) fn bind_owner(')), \ + 'a foreign rebind displaces the old graph: revisit this model' + teardown = (g17 / 'runtime/render/teardown.rs').read_text() + assert re.search(r'if self\.retired\[index\]\.buffer_id\(\)\.is_some\(\) \{\s*index \+= 1;\s*continue;', + body(teardown, 'pub(in crate::g17::runtime) fn collect_reclaimable(')), \ + 'a displaced graph with memory stays until its context is detached: revisit this model' + assert 'pairs: [Option; 2],' in (g17 / 'context/identity.rs').read_text(), \ + 'a context has two render work-state channels: revisit this model' + + s = (g17 / 'queue/admission.rs').read_text(); cfg = (source / 'hw/t8140.rs').read_text() + constants = '\n'.join(re.search(r'pub\(crate\) const ' + name + r': [^;]+;', cfg).group(0) + for name in ['RENDER_SLOTS', 'RENDER_PAIRS_PER_OWNER', 'RENDER_DEPTH']) + mmu = (source / 'mmu.rs').read_text() + limits = '\n'.join(re.search(r'const ' + name + r': [^;]+;', mmu).group(0) + for name in ['EXECUTION_CONTEXT_MASK', 'MAX_EXECUTION_CONTEXTS']) + header = '''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + type KVec = Vec; struct InFlight; + mod cfg {\n''' + constants + '}\n' + limits + '\n' + methods = ['const fn new', 'fn exhausted', 'fn refusal', 'fn advance', 'fn take_idle', 'fn acquire', + 'fn claim_sibling', 'fn forfeit_sibling', 'fn release'] + code = header + item(s, 'struct State') + '\nimpl State {\n' + \ + re.search(r' const MASK: u64 = .*;', s).group(0) + '\n' + \ + '\n'.join(item(s, n) for n in methods) + '\n}\n' + ''' + use std::collections::{HashMap,HashSet}; + // Registry side of a publication: the graph installed on each pair, + // displaced graphs of open owners (retained for their scheduler until + // that owner's own release), and each context's two render work-state + // channels, bound to the pairs it published on. + struct Model {graph:[u64;64],retained:Vec<(u64,usize)>,channels:HashMap>,open:HashSet} + impl Model { + fn publish(&mut self,owner:u64,slot:usize) { + if self.graph[slot]!=owner { + if self.retained.contains(&(owner,slot)) { + panic!("owner {} would install a second graph on pair {} while its displaced graph there is retained for its scheduler",owner,slot); + } + let previous=self.graph[slot]; + if previous!=0 && self.open.contains(&previous) {self.retained.push((previous,slot));} + self.graph[slot]=owner; + } + let channels=self.channels.entry(owner).or_default(); + if !channels.contains(&slot) { + assert!(channels.len()<2,"owner {} has no work-state channel for pair {}: it published on {:?}",owner,slot,channels); + channels.push(slot); + } + } + fn close(&mut self,owner:u64) {self.open.remove(&owner);self.retained.retain(|(vm,_)|*vm!=owner);} + } + // Pool::release_owner on the State: logical close of every slot of `owner`. + fn close(s:&mut State,m:&mut Model,owner:u64) { + for slot in 0..cfg::RENDER_SLOTS {if s.owners[slot]==owner {s.released|=1u64<u8 { + let slot=s.acquire(owner).unwrap_or_else(||panic!("owner {} found no pair",owner)); + m.publish(owner,slot as usize); + slot + } + // Forfeit candidates: idle installed pairs of open owners with two installed pairs. + fn candidates(s:&State,m:&Model)->Vec<(u8,u64)> { + (1..cfg::RENDER_SLOTS).filter(|slot|{ + let donor=s.owners[*slot]; + donor!=0 && m.open.contains(&donor) && m.graph[*slot]==donor && s.refs[*slot]==0 + && (1..cfg::RENDER_SLOTS).filter(|other|s.owners[*other]==donor && m.graph[*other]==donor).count()>=2 + }).map(|slot|(slot as u8,s.owners[slot])).collect() + } + fn main(){ + const COMPOSITOR:u64=1; + let mut s=State::new(); + let mut m=Model{graph:[0;64],retained:Vec::new(),channels:HashMap::new(),open:HashSet::new()}; + // The desktop and earlier clients hold every pair: 31 owners render on two, one on one. + for owner in 1..=32 { + m.open.insert(owner); + let a=render(&mut s,&mut m,owner); + if owner<32 {let b=render(&mut s,&mut m,owner);assert_ne!(a,b);s.release(b);} + s.release(a); + } + assert_eq!(s.owners[1..].iter().filter(|o|**o!=0).count(),63); + // Everyone but the compositor renders on both pairs again; the compositor + // keeps drawing on one, so its sibling is the least recently used pair. + for owner in 2..=31 {let a=render(&mut s,&mut m,owner);let b=render(&mut s,&mut m,owner);s.release(b);s.release(a);} + let first=render(&mut s,&mut m,COMPOSITOR);s.release(first); + let mine:Vec=(1..cfg::RENDER_SLOTS).filter(|slot|s.owners[*slot]==COMPOSITOR).collect(); + assert_eq!(mine.len(),2); + // held40: refused clients take idle siblings of open owners. + let mut recipients=Vec::new(); + for owner in 100..140 { + m.open.insert(owner); + assert_eq!(s.refusal(owner),Err(ENOSPC)); + if let Some((slot,donor))=s.forfeit_sibling(&candidates(&s,&m),0,owner) {recipients.push((owner,slot as usize,donor));} + } + let &(holder,given,_)=recipients.iter().find(|r|r.2==COMPOSITOR).expect("the compositor donates its least recently used sibling"); + let kept=*mine.iter().find(|slot|**slot!=given).unwrap(); + for round in 0..10 { + for &(owner,_,_) in &recipients {let slot=render(&mut s,&mut m,owner);s.release(slot);} + // The compositor's recipient draws most recently of all. + let slot=render(&mut s,&mut m,holder);s.release(slot); + // Two overlapping compositor commands per frame. + let x=render(&mut s,&mut m,COMPOSITOR);let y=render(&mut s,&mut m,COMPOSITOR); + assert!(x as usize!=given && y as usize!=given,"round {}: the compositor uses its own pair, never the one its recipient holds",round); + s.release(y);s.release(x); + } + assert_eq!(s.owners[given],holder); + assert!(m.retained.contains(&(COMPOSITOR,given)),"control: the recipient displaced the compositor's graph"); + // The held clients exit; the compositor keeps drawing for 20 frames, + // beside fresh clients that reuse the released pairs. + for &(owner,_,_) in &recipients {close(&mut s,&mut m,owner);} + for owner in 100..140 {m.open.remove(&owner);} + for round in 0..20 { + let x=render(&mut s,&mut m,COMPOSITOR);let y=render(&mut s,&mut m,COMPOSITOR); + assert_eq!((x as usize,y as usize),(kept,kept),"round {}: the compositor stays on the pair it kept",round); + if round<12 { + let fresh=200+round as u64;m.open.insert(fresh); + let slot=render(&mut s,&mut m,fresh);s.release(slot);close(&mut s,&mut m,fresh); + } + s.release(y);s.release(x); + } + assert_eq!(s.owners.iter().filter(|o|**o==COMPOSITOR).count(),1,"a donor receives no other pair while open"); + // Once the compositor itself closes, its record clears. + close(&mut s,&mut m,COMPOSITOR); + assert!(!s.forfeited.contains(&COMPOSITOR)); + println!("forfeit donor: the compositor donates its idle sibling under held40, renders every frame on the pair it kept while the recipient holds the other and after it exits, and never takes back a pair its displaced graph still holds"); + } + ''' + # The donor rule before this fix: a donor takes back the pair it gave away + # once free. Its next command there waits for its own retained graph. + reclaim = [ + (' forfeited: [u64; cfg::RENDER_SLOTS],\n}', + ' forfeited: [u64; cfg::RENDER_SLOTS],\n forfeited_slots: [u8; cfg::RENDER_SLOTS],\n}'), + (' forfeited: [0; cfg::RENDER_SLOTS],\n', + ' forfeited: [0; cfg::RENDER_SLOTS],\n forfeited_slots: [0; cfg::RENDER_SLOTS],\n'), + (''' if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER + || self.forfeited.contains(&owner) + { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; +''', ''' if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + if let Some(record) = self.forfeited.iter().position(|id| *id == owner) { + let slot = usize::from(self.forfeited_slots[record]); + if eligible(slot) + && self.owners[slot] != owner + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + { + self.forfeited[record] = 0; + self.forfeited_slots[record] = 0; + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.epochs[slot] = 0; + } + return; + } +'''), + (''' if !self.forfeited.contains(&donor) { + self.forfeited[record] = donor; + } +''', ''' let record = self + .forfeited + .iter() + .position(|value| *value == donor) + .unwrap_or(record); + self.forfeited[record] = donor; + self.forfeited_slots[record] = slot; +'''), + ] + # Without the donor rule a donor claims any free sibling: a third pair for + # a context whose two work-state channels are bound. + unguarded = [(''' + || self.forfeited.contains(&owner) + { + return; + }''', ''' { + return; + }''')] + for name, edits, expected in [('fixed', [], None), + ('donor-reclaims-its-pair', reclaim, 'second graph on pair'), + ('donor-claims-any-sibling', unguarded, 'no work-state channel')]: + current = code + for old, new in edits: + assert current.count(old) == 1, (name, old[:50]) + current = current.replace(old, new) + fixture = root / ('forfeit-donor-' + name + '.rs'); fixture.write_text(current) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(fixture), '-o', + str(fixture.with_suffix(''))], check=True) + result = subprocess.run([str(fixture.with_suffix(''))], capture_output=True, text=True) + if expected is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + +def shared_vm_donor_tests(root, source): + """A VM with several render queues (one DRM file, as quickshell or a GPU + process with several GL contexts) under held40 pressure. One queue donates + its idle sibling; its displaced graph keeps that VM's view of the pair + until the queue's own scheduler release. The VM's other queues are then + offered the freed pair: an existing queue claiming a sibling, and a new + queue. Runs the actual admission State and the actual per-VM view records + of mmu/lifetime.rs; graph lifetimes follow the rules checked in + forfeit_donor_tests.""" + g17 = source / 'g17' + def item(s, start): + at = s.index(start); begin = s.index('{', at); depth = 1; end = begin + 1 + while depth: + depth += (s[end] == '{') - (s[end] == '}'); end += 1 + return s[at:end] + s = (g17 / 'queue/admission.rs').read_text(); cfg = (source / 'hw/t8140.rs').read_text() + constants = '\n'.join(re.search(r'pub\(crate\) const ' + name + r': [^;]+;', cfg).group(0) + for name in ['RENDER_SLOTS', 'RENDER_PAIRS_PER_OWNER', 'RENDER_DEPTH']) + mmu = (source / 'mmu.rs').read_text() + limits = '\n'.join(re.search(r'const ' + name + r': [^;]+;', mmu).group(0) + for name in ['EXECUTION_CONTEXT_MASK', 'MAX_EXECUTION_CONTEXTS']) + lifetime = (source / 'mmu/lifetime.rs').read_text() + views = item(lifetime, 'pub(super) struct DriverMappings') + '\n' + item(lifetime, 'struct PoolMappings') + '\n' + \ + item(lifetime, 'impl DriverMappings') + '\nimpl Vm {\n' + '\n'.join( + item(lifetime, 'pub(crate) fn ' + name + '(') for name in [ + 'install_driver_mappings', 'render_pool_tvb_blocks', 'install_render_pool_mappings', + 'append_render_pool_mappings_prepared', 'truncate_render_pool_mappings', + 'clear_render_pool_mappings_if_owner']) + '\n}\n' + views = views.replace('pub(super) ', '').replace('pub(crate) ', '') + header = '''#![allow(dead_code, unused_macros, unused_imports)] + use std::ops::Range; + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; const EINVAL:i32=-22; const EBUSY:i32=-16; const EOVERFLOW:i32=-75; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + struct InFlight; + #[derive(Clone,Copy)] struct Flags; const GFP_KERNEL:Flags=Flags; + struct KVec(Vec); + impl KVec { + const fn new()->Self {KVec(Vec::new())} + fn push(&mut self,v:T,_:Flags)->Result {self.0.push(v);Ok(())} + fn push_within_capacity(&mut self,v:T)->std::result::Result<(),T> {if self.0.len()Result {self.0.reserve(n);Ok(())} + fn remove(&mut self,i:usize)->std::result::Result {if iOption {self.0.pop()} + fn drain_all(&mut self)->std::vec::Drain<'_,T> {self.0.drain(..)} + fn len(&self)->usize {self.0.len()} + fn capacity(&self)->usize {self.0.capacity()} + fn is_empty(&self)->bool {self.0.is_empty()} + fn iter(&self)->std::slice::Iter<'_,T> {self.0.iter()} + fn iter_mut(&mut self)->std::slice::IterMut<'_,T> {self.0.iter_mut()} + } + struct KernelMapping(u64); + impl KernelMapping {fn iova_range(&self)->Range {self.0..self.0+1}} + struct Mutex(std::sync::Mutex); + impl Mutex {fn lock(&self)->std::sync::MutexGuard<'_,T> {self.0.lock().unwrap()}} + struct VmLifetime {driver:Mutex>} + struct Vm {lifetime:Option>} + mod hw {pub mod t8140 {pub mod queues {''' + re.search(r'pub\(crate\) const RENDER_SLOTS: [^;]+;', cfg).group(0).replace('pub(crate)', 'pub') + '''}}} + mod cfg {\n''' + constants + '}\n' + limits + '\n' + methods = ['const fn new', 'fn exhausted', 'fn refusal', 'fn advance', 'fn take_idle', 'fn acquire', + 'fn claim_sibling', 'fn forfeit_sibling', 'fn release'] + follow = ',None' if 'fn acquire(&mut self, owner: u64, follow' in s else '' + code = header + views + item(s, 'struct State') + '\nimpl State {\n' + \ + re.search(r' const MASK: u64 = .*;', s).group(0) + '\n' + \ + '\n'.join(item(s, n) for n in methods) + '\n}\n' + ''' + use std::collections::{HashMap,HashSet}; + // The graph on each pair (owner, view id); a parked pair has none. + // Displaced graphs of open queues keep their VM view until that queue's + // scheduler release; a closed queue's graphs are released and parked. + struct Model {graph:[(u64,u64);64],retained:Vec<(u64,usize,u64)>,vm:HashMap,vms:Vec,open:HashSet,next:u64} + impl Model { + fn view(&self,owner:u64)->&Vm {&self.vms[self.vm[&owner]]} + fn publish(&mut self,owner:u64,slot:usize) { + let (previous,id)=self.graph[slot]; + if previous==owner {return;} + if previous!=0 && self.vm[&previous]==self.vm[&owner] { + // Same VM: the pair is rebound without a new graph. + self.graph[slot]=(owner,id);return; + } + self.next+=1;let new=self.next; + if let Err(error)=self.view(owner).install_render_pool_mappings(slot as u8,new,1,KVec(vec![KernelMapping(new)])) { + assert_eq!(error,EBUSY); + panic!("queue {} (VM {}) stalls on pair {}: another graph of its VM keeps its view there until queue {:?} closes", + owner,self.vm[&owner],slot,self.retained.iter().filter(|r|r.1==slot).map(|r|r.0).collect::>()); + } + if previous!=0 { + if self.open.contains(&previous) {self.retained.push((previous,slot,id));} + else {self.view(previous).clear_render_pool_mappings_if_owner(slot as u8,id);} + } + self.graph[slot]=(owner,new); + } + fn close(&mut self,owner:u64) { + self.open.remove(&owner); + for slot in 0..64 { + let (o,id)=self.graph[slot]; + if o==owner {self.view(owner).clear_render_pool_mappings_if_owner(slot as u8,id);self.graph[slot]=(0,0);} + } + let mine:Vec<_>=self.retained.iter().filter(|r|r.0==owner).cloned().collect(); + for (o,slot,id) in mine {self.view(o).clear_render_pool_mappings_if_owner(slot as u8,id);} + self.retained.retain(|r|r.0!=owner); + } + } + fn close(s:&mut State,m:&mut Model,owner:u64) { + for slot in 0..cfg::RENDER_SLOTS {if s.owners[slot]==owner {s.released|=1u64<u8 { + let slot=s.acquire(owner FOLLOW).unwrap_or_else(||panic!("queue {} found no pair",owner)); + m.publish(owner,slot as usize); + slot + } + fn candidates(s:&State,m:&Model)->Vec<(u8,u64)> { + (1..cfg::RENDER_SLOTS).filter(|slot|{ + let donor=s.owners[*slot]; + donor!=0 && m.open.contains(&donor) && m.graph[*slot].0==donor && s.refs[*slot]==0 + && (1..cfg::RENDER_SLOTS).filter(|other|s.owners[*other]==donor && m.graph[*other].0==donor).count()>=2 + }).map(|slot|(slot as u8,s.owners[slot])).collect() + } + fn main(){ + // VM 1 hosts queues 1 (donor), 32 (one pair) and later 50 (new); every other queue has its own VM. + let (donor,sibling,late)=(1u64,32u64,50u64); + let mut s=State::new(); + let mut m=Model{graph:[(0,0);64],retained:Vec::new(),vm:HashMap::new(),vms:Vec::new(),open:HashSet::new(),next:0}; + let vm_of=|m:&mut Model,owner:u64,shared:bool| { + let index=if shared && !m.vms.is_empty() {0} else { + let vm=Vm{lifetime:Some(std::sync::Arc::new(VmLifetime{driver:Mutex(std::sync::Mutex::new(None))}))}; + vm.install_driver_mappings(KVec::new(),KVec::new()).unwrap();m.vms.push(vm);m.vms.len()-1}; + m.vm.insert(owner,index);m.open.insert(owner); + }; + for owner in 1..=32 { + vm_of(&mut m,owner,owner==donor||owner==sibling); + let a=render(&mut s,&mut m,owner); + if owner<32 {let b=render(&mut s,&mut m,owner);s.release(b);} + s.release(a); + } + for owner in 2..=31 {let a=render(&mut s,&mut m,owner);let b=render(&mut s,&mut m,owner);s.release(b);s.release(a);} + let first=render(&mut s,&mut m,donor);s.release(first); + let mut recipients=Vec::new(); + for owner in 100..140 { + vm_of(&mut m,owner,false); + assert_eq!(s.refusal(owner),Err(ENOSPC)); + if let Some((slot,from))=s.forfeit_sibling(&candidates(&s,&m),0,owner) {recipients.push((owner,slot as usize,from));} + } + let &(holder,given,_)=recipients.iter().find(|r|r.2==donor).expect("the VM's queue donates its least recently used sibling"); + for _ in 0..5 { + for &(owner,_,_) in &recipients {let slot=render(&mut s,&mut m,owner);s.release(slot);} + let x=render(&mut s,&mut m,donor);let y=render(&mut s,&mut m,donor);s.release(y);s.release(x); + } + let displaced=m.retained.iter().find(|r|r.0==donor&&r.1==given).expect("control: the donor's displaced graph keeps its view").2; + // Only the donor's recipient exits: the given pair is the one free slot. + close(&mut s,&mut m,holder); + // The VM's queue with one pair claims it as its sibling and draws on both. + for round in 0..10 { + let x=render(&mut s,&mut m,sibling);let y=render(&mut s,&mut m,sibling); + if round==0 {assert_eq!(y as usize,given,"control: the VM's other queue is offered the freed pair");} + s.release(y);s.release(x); + } + // That queue exits; a new queue of the same VM is offered the pair. + close(&mut s,&mut m,sibling); + vm_of(&mut m,late,true); + let x=render(&mut s,&mut m,late);let y=render(&mut s,&mut m,late); + assert!(x as usize==given||y as usize==given,"control: the new queue of the VM is offered the freed pair"); + s.release(y);s.release(x); + let view=m.view(donor); + assert_eq!(view.render_pool_tvb_blocks(given as u8,displaced),Some(1),"the donor's displaced graph keeps its view"); + assert_eq!(view.render_pool_tvb_blocks(given as u8,m.graph[given].1),Some(1),"the new graph has its own view"); + close(&mut s,&mut m,donor); + assert_eq!(m.view(late).render_pool_tvb_blocks(given as u8,displaced),None,"the donor's release drops only its view"); + assert_eq!(m.view(late).render_pool_tvb_blocks(given as u8,m.graph[given].1),Some(1)); + println!("shared VM: a queue of a multi-queue VM donates its sibling under held40; the VM's other queue and a new queue install and draw on the freed pair while the donor's displaced graph keeps its own view"); + } + '''.replace(' FOLLOW', follow) + one_view = [('if views.iter().any(|pool| pool.id == id) {', 'if !views.is_empty() {')] + for name, edits, expected in [('fixed', [], None), ('one-view-per-pair', one_view, 'stalls on pair')]: + current = code + for old, new in edits: + assert current.count(old) == 1, (name, old) + current = current.replace(old, new) + fixture = root / ('shared-vm-' + name + '.rs'); fixture.write_text(current) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(fixture), '-o', + str(fixture.with_suffix(''))], check=True) + result = subprocess.run([str(fixture.with_suffix(''))], capture_output=True, text=True) + if expected is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + if __name__ == '__main__': with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: admission_tests(Path(temporary), DRIVER) + forfeit_donor_tests(Path(temporary), DRIVER) + shared_vm_donor_tests(Path(temporary), DRIVER) queue_tests(Path(temporary), DRIVER) + compute_takeover_tests(Path(temporary), DRIVER) + compute_admission_tests(Path(temporary), DRIVER) shared_alias_tests(Path(temporary), DRIVER) From f4feef8201b20273c92b7b395f34f5b02c9f6449 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:53 -0400 Subject: [PATCH 11/22] drm/asahi: order G17 queue owner changes after scheduler releases When a logical queue closes, its scheduler release is published once its work is idle. The firmware consumes the release against the queues that scheduler last ran. After the host has seen it consumed, the teardown parks the scheduler's render pairs, and a later owner reinstalls a parked pair with fresh queue records. A successor on the same VM could find such a pair still installed after the release was published but before its consumption was seen. It then rebound the pair in place, rewriting the job-list head, pid and scheduler fields of queue records the firmware had run for the released scheduler, and published its first command, which carries the owner change. A GL client that recreates its context every few milliseconds next to three Vulkan clients slowed the release enough to open this window. Its successor published the first commands on both of its pairs inside it, and the GPU firmware crashed about 3 ms later without writing a crash log. At the crash the release had been consumed. Both pairs were rebound in place and their first commands for the new owner were still in flight. Order owner changes and scheduler releases: - A render pair whose context has an armed or unconsumed scheduler release is not rebound. The new owner retries until the teardown has parked the pair, then installs it with fresh records. The check is made when the rebind is chosen and again when it is committed, because a live donor may close while one of its pairs is prepared for another owner. - A scheduler release waits while a render pair or compute queue that changed owner away from that context still has the new owner's first command in flight. A pair records at publication which scheduler its queues last ran for. A borrowed pair keeps a snapshot of that state. - The first command after an owner change waits while the scheduler the pair last ran for is being released. This happens only when a live donor closes between giving up a pair and the recipient's first command. - The same holds for compute queues: a queue whose bound scheduler has an armed or unconsumed release is not rebound, re-armed for another owner or taken over, and the first command after a compute owner change waits while the scheduler the queue was bound to is being released. - A pair parked without a consumed release, an unused warm pair moved to another owner, keeps the scheduler it last ran for, so its next owner change still waits for that release. A park after the consumed release of the pair's scheduler clears it. The normal path is unchanged. A pair whose owner does not change compares one word at publication. A release is delayed only while an owner change is in flight. After a context churn, a same-VM successor's first render waits until the previous context's release has been consumed and its pairs parked, which successors on another VM already did. Measured on Neo with diagnostic switches that apply the first two rules: - four-client churn: 3 of 3 runs crashed within 10 to 120 s without them, 2 runs of 90 s passed with them - GLES: 15160 vs 15175 iterations, 5054 vs 5059 contexts per 90 s - Vulkan 4x MSAA render: 212643 vs 214050 iterations tools/asahi/g17-owner-release-order-tests.py compiles the release table, pair switch tracking and release readiness predicates against host stubs and checks the guard placement in the rebind and publication paths. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 15 + drivers/gpu/drm/asahi/g17/queue/render.rs | 25 ++ .../gpu/drm/asahi/g17/queue/render/parked.rs | 15 + drivers/gpu/drm/asahi/g17/runtime.rs | 27 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 6 + .../gpu/drm/asahi/g17/runtime/render/grow.rs | 3 + .../drm/asahi/g17/runtime/render/observe.rs | 2 + .../drm/asahi/g17/runtime/render/prepare.rs | 14 + .../drm/asahi/g17/runtime/render/submit.rs | 8 + .../drm/asahi/g17/runtime/render/teardown.rs | 13 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- drivers/gpu/drm/asahi/g17/teardown.rs | 15 + tools/asahi/g17-owner-release-order-tests.py | 260 ++++++++++++++++++ 13 files changed, 402 insertions(+), 3 deletions(-) create mode 100644 tools/asahi/g17-owner-release-order-tests.py diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 259a1f9f8e0524..86daa2bf591bfd 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -566,6 +566,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3a106dbd524729..1e28d2d37bb385 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -171,6 +171,12 @@ pub(crate) struct Pair { context_update: bool, /// A retired, never-executed group awaiting republication on this pair. replay: Option>, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] @@ -255,6 +261,8 @@ impl Pair { displaced: None, replay: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -788,6 +796,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -1585,6 +1598,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.is_some() || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index e811d7e5c9073d..c2ad99cb32523e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -31,6 +31,11 @@ pub(crate) struct ParkedPair { registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -91,6 +96,7 @@ impl Pair { registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -105,6 +111,13 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } @@ -225,6 +238,8 @@ impl ParkedPair { displaced: None, replay: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index b5af3577cab9ec..95f0ebe049de51 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -137,6 +137,7 @@ impl Registry { }); } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -144,6 +145,12 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -174,7 +181,11 @@ impl Registry { let Some(queue) = entry.as_ref().and_then(|entry| entry.queue.as_deref()) else { continue; }; - if queue.owner() == Some(owner) || !queue.idle_owned() || !queue.pool_fits(required) { + if queue.owner() == Some(owner) + || !queue.idle_owned() + || !queue.pool_fits(required) + || queue.context().is_some_and(|bound| self.teardown.releasing(bound)) + { continue; } let foreign = !queue @@ -245,6 +256,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -252,6 +264,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -1058,6 +1074,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 2f8beb932130df..0f569c0d03978b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -46,6 +46,9 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], closed: bool, pending_error: Option<(Arc, Error)>, @@ -163,6 +166,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -182,6 +186,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -401,6 +406,7 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], closed: false, pending_error: None, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index b283fb8e7d55ca..9e229855012e53 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,6 +271,7 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; } if firmware.queues.render.grow.active != Some(request) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c80018..51fe6428fae481 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 0ed99ed04552de..a06e3c91f5b9de 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -133,6 +133,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -164,6 +171,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -466,6 +475,10 @@ impl super::super::Backend { // A pair forfeited by a live owner still names that owner's // scheduler; only its logical close may arm the release. let owner_closed = entry.closed; + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { @@ -502,6 +515,7 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; (result, firmware.render_grow_ready()) }; drop(prepared); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 730d69059908cb..292a3a5265eb03 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -395,6 +395,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0c..eb04f803619558 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01c..f8020e8962f849 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b1..53518a76e91a47 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py new file mode 100644 index 00000000000000..0430a8a0c76597 --- /dev/null +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the ordering between physical-queue owner changes and scheduler releases. + +The production release table, pair switch tracking, release readiness and compute +switch predicate are compiled against host stubs. This is not GPU execution. The +captured firmware crash (a render pair rebound in place and kicked by a same-VM +successor while its previous scheduler's release was published and the pair not +yet parked) must fail the same assertions once either rule is removed. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def ordered(source, *needles): + """Every needle occurs, in this order, inside `source`.""" + position = 0 + for needle in needles: + found = source.find(needle, position) + assert found >= 0, f'{needle!r} missing or out of order' + position = found + len(needle) + + +def check_guards(): + prepare = (ROOT / 'runtime/render/prepare.rs').read_text() + build = item(prepare, 'fn build_render_pair(') + # The in-place rebind is refused while the pair's scheduler release is pending. + ordered(build, 'if !pair.reusable()?', 'firmware.queues.teardown.releasing(pair.context())', + 'return Err(EBUSY);', 'Build::Rebind {') + rebind = item(prepare, 'fn rebind_pair(') + # A live donor that closed while the binding was prepared is caught at commit. + ordered(rebind, 'firmware.queues.teardown.releasing(pair.context())', 'return Err(EBUSY);', + 'firmware.queues.teardown.displace(', 'pair.bind_owner(') + submit = (ROOT / 'runtime/render/submit.rs').read_text() + publish = item(submit, 'pub(super) fn publish_render(') + ordered(publish, '.owner_switch_from(&packet.context)', 'releasing_scheduler(scheduler)', + 'return Err(EBUSY);', 'acquire_owner(buffer_id)', 'pair.publish(') + render = (ROOT / 'queue/render.rs').read_text() + pair_publish = item(render, 'pub(crate) fn publish(') + # The switch is recorded only after the outer publication succeeded. + ordered(pair_publish, 'let outer_next = match result', 'self.ordinal = next_ordinal;', + 'self.switch = Some((self.executed, ordinal));', 'self.executed = scheduler;', + 'self.announce_publication(') + parked = (ROOT / 'queue/render/parked.rs').read_text() + # Parking keeps the last scheduler; only a consumed release clears it. + ordered(item(parked, 'pub(crate) fn park('), 'executed: self.executed,') + ordered(item(parked, 'pub(crate) fn commit('), 'executed: self.executed,', 'switch: None,') + detach = item((ROOT / 'runtime/render/teardown.rs').read_text(), 'fn detach_context(') + ordered(detach, '.park();', 'parked.release_consumed(context.scheduler_va());') + runtime = (ROOT / 'runtime.rs').read_text() + # A compute queue does not change owner while its scheduler is released. + ordered(item(runtime, 'pub(super) fn rebind_compute('), '.filter(|queue| queue.reusable())', + 'teardown.releasing(bound)', 'continue;', 'queue.bind_owner(owner, context.clone())?;') + ordered(item(runtime, 'pub(super) fn take_over_idle_compute('), 'self.teardown.releasing(bound)', + 'queue.yield_idle_owner()?;', 'queue.bind_owner(') + ordered(item(runtime, 'pub(super) fn bind_rearmed('), 'teardown.releasing(bound)', 'return Err(EAGAIN);', + 'queue.bind_owner(') + ordered(item(runtime, 'fn publish_compute('), '.pending_switch_from()', 'queues.teardown.releasing(previous)', + 'return Err(EAGAIN);', 'queue.publish(') + # Takeover only under publication's own device mutex hold. + assert 'take_over_idle_compute' not in item(runtime, 'fn build_compute(') + for name, path in [('take_pair', 'runtime/render.rs'), ('take_render_observation', 'runtime/render/observe.rs')]: + body = item((ROOT / path).read_text(), f'fn {name}(') + ordered(body, 'entry.borrowed_switch = pair.switch_in_flight();', 'entry.pair.take()') + print('PASS guard placement: rebind decision, rebind commit, first publication, switch record, borrows, parked scheduler, compute owner changes') + + +def fixture(): + teardown = (ROOT / 'teardown.rs').read_text() + assert ('#[derive(Clone, Copy, PartialEq, Eq)]\nenum Phase {\n Deferred,\n Draining,\n' + ' Released(u32),\n}') in teardown + pending = '\n'.join(item(teardown, name) for name in [ + 'pub(crate) fn displace(', 'pub(crate) fn close(', 'fn insert(', 'pub(crate) fn releasing(', + 'pub(crate) fn releasing_scheduler(', 'fn find(']).replace('pub(crate) ', '') + pending = pending.replace('self.entries.push(entry, GFP_KERNEL)?;', 'self.entries.push(entry);') + render = (ROOT / 'queue/render.rs').read_text() + switch_methods = '\n'.join(item(render, name) for name in [ + 'pub(crate) fn owner_switch_from(', 'pub(crate) fn switch_in_flight(']).replace('pub(crate) ', '') + record = render[render.index(' let scheduler = packet.context.scheduler_va();'): + render.index(' self.executed = scheduler;') + len(' self.executed = scheduler;')] + idle = item((ROOT / 'runtime/render/teardown.rs').read_text(), + 'pub(in crate::g17::runtime) fn teardown_idle(').replace('pub(in crate::g17::runtime) ', '') + compute_switch = item((ROOT / 'queue/compute.rs').read_text(), + 'pub(crate) fn switching_from(').replace('pub(crate) ', '') + compute_idle = item((ROOT / 'runtime/teardown.rs').read_text(), 'fn compute_idle(') + return r''' +#![allow(dead_code)] +use std::sync::Arc; +type Error = i32; +type Result = std::result::Result; +struct Context { scheduler: u64, published: bool } +impl Context { + fn scheduler_va(&self) -> u64 { self.scheduler } + fn is_published(&self) -> bool { self.published } +} +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { Deferred, Draining, Released(u32) } +struct Entry { context: Arc, phase: Phase, armed: u64, announce_pending: bool } +struct Pending { entries: Vec } +impl Pending { +PENDING +} +struct Ticket { ordinal: u64 } +struct Active { ticket: Ticket } +struct Packet { context: Arc } +struct Pair { executed: u64, switch: Option<(u64, u64)>, replay: Option<()>, active: Vec, idle: bool } +impl Pair { +SWITCH + /// Host stand-in for a successful publication of `ordinal` (stand-in, not production). + fn publish(&mut self, packet: &Packet, ordinal: u64) { +RECORD + self.active.push(Active { ticket: Ticket { ordinal } }); + } + fn idle_for_context(&self, _: &Arc) -> Result { Ok(self.idle) } +} +struct RenderEntry { + context: Option>, + preparing: Option<()>, + growing: bool, + pair: Option>, + borrowed_switch: Option, +} +struct RenderState { entries: Vec> } +impl RenderState { +IDLE +} +struct Previous { context: Arc, publication: Option<(u64, u64)> } +struct Queue { previous: Option, bound: Option> } +impl Queue { +COMPUTE_SWITCH + fn idle_for_context(&self, context: &Arc) -> bool { + !self.bound.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, context)) + } +} +struct ComputeEntry { queue: Option> } +struct Service { compute: Vec> } +impl Service { +COMPUTE_IDLE +} +fn context(scheduler: u64) -> Arc { Arc::new(Context { scheduler, published: true }) } +fn main() { + let (b, c, d) = (context(0x1000), context(0x2000), context(0x3000)); + // Captured crash: GL context B closes, its release is published, then the same-VM + // successor C finds B's pairs still installed (not parked) and rebinds them in place. + let mut table = Pending { entries: Vec::new() }; + table.close(&b, 1).unwrap(); + assert!(table.releasing(&b), "armed release refuses an owner change"); + table.entries[0].phase = Phase::Released(67); + assert!(table.releasing(&b), "published, unconsumed release refuses an owner change"); + assert!(table.releasing_scheduler(0x1000) && !table.releasing_scheduler(0x2000)); + // Consumption and detach remove the entry; the pair is parked and reinstalled fresh. + table.entries.clear(); + assert!(!table.releasing(&b) && !table.releasing_scheduler(0x1000)); + // A forfeit from a live owner is not a release; its later close arms one. + table.displace(&d, false, 2).unwrap(); + assert!(!table.releasing(&d), "a live displaced owner may still lose an idle pair"); + table.close(&d, 3).unwrap(); + assert!(table.releasing(&d), "the donor's close arms its release"); + // An unpublished context has no firmware scheduler state and no release. + let quiet = Arc::new(Context { scheduler: 0x4000, published: false }); + table.close(&quiet, 4).unwrap(); + assert!(!table.releasing(&quiet)); + + // Switch tracking across an owner change of one pair. + let mut pair = Pair { executed: 0, switch: None, replay: None, active: Vec::new(), idle: true }; + assert_eq!(pair.owner_switch_from(&c), None, "a fresh graph has no previous scheduler"); + pair.publish(&Packet { context: d.clone() }, 0); + pair.active.clear(); + assert_eq!(pair.owner_switch_from(&d), None, "the same owner never switches"); + assert_eq!(pair.owner_switch_from(&c), Some(0x3000)); + pair.publish(&Packet { context: c.clone() }, 1); + assert_eq!(pair.switch_in_flight(), Some(0x3000)); + let mut render = RenderState { entries: vec![Some(RenderEntry { + context: Some(c.clone()), preparing: None, growing: false, pair: Some(Box::new(pair)), borrowed_switch: None, + })] }; + assert!(!render.teardown_idle(&d).unwrap(), "release waits for the new owner's first command"); + assert!(render.teardown_idle(&b).unwrap(), "unrelated releases are not held"); + // A borrowed pair is judged by its snapshot; a stale snapshot outside a borrow is ignored. + let borrowed = render.entries[0].as_mut().unwrap(); + let pair = borrowed.pair.take().unwrap(); + borrowed.borrowed_switch = pair.switch_in_flight(); + borrowed.preparing = Some(()); + assert!(!render.teardown_idle(&d).unwrap(), "a borrowed switching pair still holds the release"); + let entry = render.entries[0].as_mut().unwrap(); + entry.preparing = None; + entry.pair = Some(pair); + entry.pair.as_mut().unwrap().active.clear(); + assert!(render.teardown_idle(&d).unwrap(), "retirement of the first command releases the hold"); + let entry = render.entries[0].as_mut().unwrap(); + entry.pair.as_mut().unwrap().active.push(Active { ticket: Ticket { ordinal: 1 } }); + entry.pair = None; + assert!(render.teardown_idle(&d).unwrap(), "a stale snapshot without a borrow holds nothing"); + + // Compute: only a published first command after the change holds the release. + let mut service = Service { compute: vec![Some(ComputeEntry { queue: Some(Box::new(Queue { + previous: Some(Previous { context: b.clone(), publication: None }), bound: Some(c.clone()), + })) })] }; + assert!(service.compute_idle(&b), "an unpublished rebind does not hold the release"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous.as_mut().unwrap().publication = Some((1, 0)); + assert!(!service.compute_idle(&b), "compute release waits for the new owner's first command"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous = None; + assert!(service.compute_idle(&b)); + assert!(!service.compute_idle(&c), "the bound owner still needs its own retirement"); + println!("PASS release table gate, pair switch hold, borrowed snapshot and compute switch hold"); +} +'''.replace('PENDING', pending).replace('SWITCH', switch_methods, 1).replace('RECORD', record) \ + .replace('IDLE', idle, 1).replace('COMPUTE_SWITCH', compute_switch).replace('COMPUTE_IDLE', compute_idle) + + +def main(): + check_guards() + code = fixture() + mutations = { + 'fixed': (None, None, None), + 'no-release-gate': ('.is_some_and(|index| self.entries[index].phase != Phase::Deferred)', + '.is_some_and(|_| false)', 'armed release refuses an owner change'), + 'no-render-hold': ('Some(pair) => pair.switch_in_flight() == scheduler,', 'Some(_) => false,', + "release waits for the new owner's first command"), + 'no-compute-hold': ('queue.idle_for_context(context) && !queue.switching_from(context)', + 'queue.idle_for_context(context)', + "compute release waits for the new owner's first command"), + } + with tempfile.TemporaryDirectory(prefix='g17-owner-release-') as directory: + root = Path(directory) + for name, (old, new, expected) in mutations.items(): + current = code + if old is not None: + assert current.count(old) == 1, name + current = current.replace(old, new) + path = root / (name + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], + check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if old is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() From 2da55b019a37468533448627196b32bc0b192f4b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:53 -0400 Subject: [PATCH 12/22] drm/asahi: wait for exhausted G17 render resources and name them A render command whose admission and queue IDs succeeded can still fail during graph construction or preparation with ENOSPC: parameter-buffer or counter identities, render free-list IDs, retired-binding room or the shared lower GPU VA window may all be held by other open owners during a burst of new clients. The packet then fails at publication within a fraction of a second although these resources free as warming graphs are reclaimed, published work retires and displaced scheduler releases complete. Construction of an optional warming sibling fails the command the same way, even though its own pair is ready. Retry such a refusal after a release for the same bounded time as admission, with the device mutex and preparation lease dropped; a failed build or discarded preparation leaves its reservation unchanged. Treat exhausted identities, VA or memory as a soft refusal for warming. On the publication path, a compute takeover that cannot map its aliases falls back to the off-lock binder instead of failing the command. A pair that is borrowed, still settling, or waiting for its previous scheduler's release to be consumed and the pair parked (a same-VM successor after a context churn) returned ETIMEDOUT after 5000 polls, about a second. Bound that wait by the same time instead, and kick the event worker, which observes the release and parks the pair, on every poll. Name each exhausted resource with its site once per module load, and log one census of render capacity (admission slots, render entries, retired bindings, scheduler releases by phase, queue IDs, compute queues and parameter-buffer, counter and free-list identities) when a render capacity wait first starts and when one first expires, so the remaining refusals can be attributed on hardware. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/buffer.rs | 26 +++++-- drivers/gpu/drm/asahi/g17/context.rs | 9 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 16 ++++- drivers/gpu/drm/asahi/g17/kick.rs | 34 ++++++++-- drivers/gpu/drm/asahi/g17/mod.rs | 20 ++++++ drivers/gpu/drm/asahi/g17/object.rs | 14 ++++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 11 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 27 +++++--- drivers/gpu/drm/asahi/g17/runtime.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 54 +++++++++++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 7 +- .../drm/asahi/g17/runtime/render/submit.rs | 68 +++++++++++++++---- drivers/gpu/drm/asahi/g17/teardown.rs | 13 ++++ 14 files changed, 264 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index c23d4cebb70968..c82bc579d451e4 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -70,6 +70,10 @@ impl BufferIds { self.next_token = token + 1; token } + /// Identities held by an owner or a reference, for diagnostics. + pub(crate) fn in_use(&self) -> usize { + self.entries.iter().filter(|entry| !entry.free()).count() + } pub(crate) fn allocate_render(&mut self) -> Result<(u8, u64)> { let id = if self.returned_count != 0 { self.returned_count -= 1; @@ -78,7 +82,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| resource_exhausted!("parameter-buffer IDs"))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -111,7 +115,10 @@ impl BufferIds { if !entry.owner { return Err(EINVAL); } - entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; + entry.references = entry + .references + .checked_add(1) + .ok_or_else(|| resource_exhausted!("parameter-buffer references"))?; Ok((id, entry.token)) } /// Preflight shared completion bookkeeping before any of its counters change. @@ -168,7 +175,7 @@ impl Scenes { (0..fw::SCENES) .map(|offset| (self.next + offset) % fw::SCENES) .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) - .ok_or(ENOSPC) + .ok_or_else(|| resource_exhausted!("render scene slots")) } pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { if self.select(ordinal)? != scene { @@ -240,7 +247,7 @@ impl Growth { let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { - return Err(ENOSPC); + return Err(resource_exhausted!("tiler heap block slots")); } let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; Ok(Self { @@ -320,7 +327,7 @@ pub(crate) fn compact_alias( Err(error) => return Err(error), } } - Err(ENOSPC) + Err(resource_exhausted!("client compact GPU VA")) } /// Allocated aliases remain private until the owner installs them in its VM @@ -465,6 +472,13 @@ impl MetricsIds { ], } } + /// Leased counter slots and their total, for diagnostics. + pub(crate) fn census(&self) -> (usize, usize) { + let used = self.used[0].load(Ordering::Relaxed).count_ones() + + (self.used[1].load(Ordering::Relaxed) & ((1u64 << (fw::METRICS_SLOTS - 64)) - 1)) + .count_ones(); + (used as usize, fw::METRICS_SLOTS) + } } pub(crate) struct MetricsLease { @@ -505,7 +519,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| resource_exhausted!("performance-counter slots"))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index d4e9eaa16a2d1d..333a6a3fcf54d8 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(resource_exhausted!("QoS IDs")); } *used |= 1u128 << id; Ok(QosLease { @@ -388,7 +388,12 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context + .work + .lock() + .render + .get(pair) + .ok_or_else(|| resource_exhausted!("context render channels")) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index f5ed7a61524937..bcb221c1529de8 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -29,6 +29,11 @@ impl RenderIds { used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], } } + /// Allocated render pool IDs, for diagnostics. + pub(crate) fn in_use(&self) -> usize { + (self.used[0].load(Ordering::Relaxed).count_ones() + + (self.used[1].load(Ordering::Relaxed) & !(1u64 << 63)).count_ones()) as usize + } } struct RenderId { slot: u16, @@ -59,7 +64,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(resource_exhausted!("render free-list IDs")) } } impl Drop for RenderId { @@ -154,7 +159,14 @@ impl RenderBacking { mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, [render_global, client], - )?; + ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA (render free list)") + } else { + error + } + })?; let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 55c3ab317a9897..f288fa643a4eaa 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -182,6 +182,17 @@ impl QueueIds { Ok(id) } + /// Leased IDs, render pair identities and compute identities, for diagnostics. + pub(crate) fn census(&self) -> (usize, usize, usize) { + let compute = self + .entries + .iter() + .flatten() + .filter(|entry| entry.id.engine == DataMaster::Compute) + .count(); + (self.leased, self.pair_identities(), compute) + } + /// Tiling and fragment identities of render pairs, excluding the initial /// reservations, which no pair uses. fn pair_identities(&self) -> usize { @@ -349,12 +360,20 @@ impl Queue { )?; let mut range = t8140::dynamic::LOWER; let (low, client) = loop { - let low = high.alias_in( - alloc.uat.kernel_lower_vm(), - range.clone(), - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - )?; + let low = high + .alias_in( + alloc.uat.kernel_lower_vm(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA (render kick ring)") + } else { + error + } + })?; let end = low .iova() .checked_add(KICK_RING_SIZE as u64) @@ -365,6 +384,9 @@ impl Queue { drop(low); range.start = end; } + Err(error) if error == ENOSPC => { + return Err(resource_exhausted!("client GPU VA for a render kick ring")) + } Err(error) => return Err(error), } }; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 206912ae3a6568..7f0bbcd8e4edbe 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -17,6 +17,26 @@ //! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state //! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. +/// A finite device resource refused an allocation: evaluates to `ENOSPC`. +/// The first refusal at each call site is reported once per module load, +/// naming the resource and site, so a client's ENOSPC can be attributed. +macro_rules! resource_exhausted { + ($what:literal) => {{ + static REPORTED: ::core::sync::atomic::AtomicBool = + ::core::sync::atomic::AtomicBool::new(false); + if !REPORTED.swap(true, ::core::sync::atomic::Ordering::Relaxed) { + let file = ::core::file!(); + ::kernel::pr_warn!( + "asahi: G17 {} exhausted ({}:{})\n", + $what, + file.rsplit("asahi/").next().unwrap_or(file), + ::core::line!() + ); + } + ::kernel::error::code::ENOSPC + }}; +} + mod buffer; mod channel; mod command; diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index eefa63692a7a15..d2612f50fea254 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -335,6 +335,13 @@ impl Allocator<'_> { prot, cpu, ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("firmware GPU VA") + } else { + error + } + }) } /// Allocates a canonical low mapping, which can be aliased at the same @@ -354,6 +361,13 @@ impl Allocator<'_> { prot, cpu, ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA") + } else { + error + } + }) } /// Place fresh backing at one address in the canonical and both alias VMs. diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 8b9fbd08d4c6fb..7ef89ed9b58ffb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -438,6 +438,17 @@ impl Pool { pub(crate) fn queues(&self) -> usize { self.queues.load(Ordering::Acquire) as usize } + /// Owned, released, exit-held and quarantined slots, for diagnostics. + pub(crate) fn census(&self) -> [usize; 4] { + let state = self.state.lock(); + let bits = |mask: u64| (mask & State::MASK).count_ones() as usize; + [ + state.owners[1..].iter().filter(|owner| **owner != 0).count(), + bits(state.released), + bits(state.kill_hold), + bits(state.quarantined), + ] + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 86daa2bf591bfd..451f7a113d4a5b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1806,7 +1806,7 @@ impl Queue { } let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); + return Err(resource_exhausted!("compute item ring")); } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index ba3d4b273cc830..63932f4f385d7e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -53,14 +53,22 @@ impl Aliases { loop { // Descriptor aliases are read-only to GPU consumers; the canonical // firmware mapping remains writable for command state updates. - let global = object.map_range( - alloc.uat.kernel_lower_vm(), - source.clone(), - range.clone(), - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - )?; + let global = object + .map_range( + alloc.uat.kernel_lower_vm(), + source.clone(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RO, + 0, + ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA (render descriptors)") + } else { + error + } + })?; let address = global.iova(); let end = address.checked_add(size).ok_or(EOVERFLOW)?; match object.map_range( @@ -81,6 +89,9 @@ impl Aliases { drop(global); range.start = end; } + Err(error) if error == ENOSPC => { + return Err(resource_exhausted!("client GPU VA for render descriptors")) + } Err(error) => return Err(error), } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 95f0ebe049de51..67320e626c9d28 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1332,12 +1332,15 @@ impl Backend { // second queue. Its flag is raised before its first decision. let taken = !self.binding.load(Ordering::Acquire) && firmware.queues.compute(self.owner).is_none() - && firmware.queues.rebind_or_take_over_compute( + && match firmware.queues.rebind_or_take_over_compute( self.owner, &self.context, required, contended, - )?; + ) { + Err(error) if error == ENOSPC => false, + result => result?, + }; if !taken { drop(state); drop(lease); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 0f569c0d03978b..9727477f87ee1b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -368,6 +368,60 @@ impl Registry { Ok(true) } + /// One line describing every render capacity pool, for attributing refusals. + pub(super) fn log_render_census(&self, reason: &str, render_pools: usize) { + let mut census = [0usize; 6]; + for entry in self.render.entries.iter().flatten() { + census[0] += 1; + census[1] += usize::from(entry.pair.is_some()); + census[2] += usize::from(entry.parked.is_some()); + census[3] += usize::from(entry.constructing || entry.growing); + census[4] += usize::from(entry.preparing.is_some()); + census[5] += usize::from(entry.closed); + } + let [owned, released, held, quarantined] = self.admission.census(); + let [deferred, draining, releasing] = self.teardown.phases(); + let (leased, pair_ids, compute_ids) = self.qids.census(); + let (metrics, metric_slots) = self.metrics.census(); + let compute = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .fold([0usize; 2], |[all, bound], queue| { + [all + 1, bound + usize::from(queue.owner().is_some())] + }); + pr_warn!( + "asahi: G17 render capacity {}: queues={} slots owned={} released={} held={} quarantined={}; entries={} pairs={} parked={} building={} preparing={} closed={}; retired={}/{}; teardown open={} draining={} releasing={}; qids leased={} pair={} compute={}; compute queues={} bound={}; pb ids={}; metrics={}/{}; render pools={}\n", + reason, + self.admission.queues(), + owned, + released, + held, + quarantined, + census[0], + census[1], + census[2], + census[3], + census[4], + census[5], + self.render.retired.len(), + self.render.retired.capacity(), + deferred, + draining, + releasing, + leased, + pair_ids, + compute_ids, + compute[0], + compute[1], + self.buffers.in_use(), + metrics, + metric_slots, + render_pools, + ); + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index a06e3c91f5b9de..9bce9d5dfc45b8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -482,7 +482,7 @@ impl super::super::Backend { if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(resource_exhausted!("retired render bindings")); } firmware.queues.teardown.displace( pair.context(), @@ -589,8 +589,11 @@ impl super::super::Backend { // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. + // Warming is optional: exhausted identities, VA or memory must not + // fail a command whose own pair is ready. Its failed construction + // leaves the reservation unchanged, as for the primary pair. match self.build_render_pair(sibling.slot(), packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + Err(error) if [EBUSY, EAGAIN, ENOSPC, ENOMEM].contains(&error) => {} result => result?, } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 292a3a5265eb03..48aad17dfbaad5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,9 +14,11 @@ use kernel::{ time::{delay::fsleep, Delta}, }; -/// Bound on waiting for render capacity held by other live owners. Capacity -/// frees when a warming graph finishes construction, when a sibling's published -/// work retires or times out, or when an owner closes. +/// Bound on waiting for render capacity held by other live owners: admission +/// slots, queue IDs, parameter-buffer and counter identities, retired-binding +/// room and shared GPU VA. Capacity frees when a warming graph is reclaimed or +/// finishes construction, when published work retires or times out, when a +/// displaced binding's scheduler release completes, or when an owner closes. const RENDER_CAPACITY_WAIT_NS: u64 = 4 * crate::g17::job::TIMEOUT_MS as u64 * 1_000_000; impl Backend { @@ -24,15 +26,48 @@ impl Backend { /// built or whose siblings are busy. Retry reclamation after their next /// release instead of failing the packet; fail only after the bound. fn wait_render_capacity(&self, since: &mut Option) -> Result { + use core::sync::atomic::AtomicBool; + static STARTED: AtomicBool = AtomicBool::new(false); + static EXPIRED: AtomicBool = AtomicBool::new(false); let now = super::super::now_ns(); + let first = since.is_none(); if now.saturating_sub(*since.get_or_insert(now)) >= RENDER_CAPACITY_WAIT_NS { - return Err(ENOSPC); + if !EXPIRED.swap(true, Ordering::Relaxed) { + self.log_render_census("wait expired"); + } + return Err(resource_exhausted!("render capacity held by open owners")); + } + if first && !STARTED.swap(true, Ordering::Relaxed) { + self.log_render_census("wait started"); } self.shared.queue_events(); fsleep(Delta::from_millis(1)); Ok(()) } + /// A pair is borrowed, still settling, or waiting for its previous + /// scheduler's release to be consumed and parked (the event worker does + /// both). Poll briefly; time out only after the capacity bound. + fn wait_render_busy(&self, since: &mut Option) -> Result { + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= RENDER_CAPACITY_WAIT_NS { + return Err(ETIMEDOUT); + } + self.shared.queue_events(); + fsleep(Delta::from_micros(200)); + Ok(()) + } + + /// Once-per-boot capacity snapshot taken with the device mutex. + fn log_render_census(&self, reason: &str) { + let state = self.shared.state.lock(); + if let Some(firmware) = (*state).as_deref() { + firmware + .queues + .log_render_census(reason, firmware.render_ids.in_use()); + } + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -150,8 +185,9 @@ impl Backend { packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; + let mut busy_since = None; let mut recovery_polls = 0; + let mut capacity_wait = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -190,11 +226,13 @@ impl Backend { match self.ensure_render(packet, slot) { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since)?; + continue; + } + // Graph or binding resources held by other owners: every failed + // build left its reservation unchanged, so retry after a release. + Err(error) if error == ENOSPC => { + self.wait_render_capacity(&mut capacity_wait)?; continue; } result => result?, @@ -215,13 +253,13 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since)?; } } + // Unpublished: the preparation was discarded and the pair returned. + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_render_capacity(&mut capacity_wait)?; + } result => return result, } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 53518a76e91a47..8e15340e0c3ba4 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -90,6 +90,19 @@ impl Pending { }) } + /// Entries by phase (displaced but open, draining, release published), for diagnostics. + pub(crate) fn phases(&self) -> [usize; 3] { + let mut phases = [0; 3]; + for entry in self.entries.iter() { + phases[match entry.phase { + Phase::Deferred => 0, + Phase::Draining => 1, + Phase::Released(_) => 2, + }] += 1; + } + phases + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() From 5b16bbdc26e2460e5626123eace43ea1649681be Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:53 -0400 Subject: [PATCH 13/22] drm/asahi: keep G17 render capacity waits from stopping the runtime A render packet that waits in publication for graph capacity already holds its admission slot, and the entry of a pair under construction has no installed pair. When such a wait outlived the two-second scheduler watchdog, timeout_render() looked the pair up for an unpublished packet, failed with EIO, and the timeout handler stopped the runtime for every client. A burst of new clients whose render free lists did not fit the shared lower GPU VA window did exactly that two seconds after the first wait began. Each retry of such a build also populated the full free-list backing, tens of MiB with ray-query scratch reservations, before the window was searched, and freed it again after the refusal. Sixteen concurrent builders reached several GiB of transient kernel allocations. - A render or compute watchdog expiry for a packet that is not published returns without action: its publisher owns the bounded wait and the result. If the collector cannot be allocated, the expiry is retried at the next watchdog period instead of stopping the runtime. - A refused free-list placement is not attempted again for the same or a larger size until free-list backing is freed, apart from one probe per 100 ms across all callers, so waiting builders allocate nothing. - While the window refuses free-list backing, idle retained compute pools are given back regardless of the warm cap. - The capacity census reports the installed render and compute free-list backing in MiB, and the largest render free list. The host resource tests check the placement gate and the watchdog guard placement. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/freelist.rs | 103 +++++++++++++++--- drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 + drivers/gpu/drm/asahi/g17/runtime.rs | 9 +- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 8 +- .../drm/asahi/g17/runtime/compute_control.rs | 9 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 26 ++++- .../drm/asahi/g17/runtime/render/settle.rs | 12 +- tools/asahi/g17-resource-tests.py | 59 ++++++++++ 8 files changed, 210 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index bcb221c1529de8..1d87cc3fd09455 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -74,21 +74,87 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; Self::from_backing(backing, blocks, layout) } + /// Size of the backing, for diagnostics. + pub(crate) fn bytes(&self) -> usize { + TABLE_SIZE + self.blocks * FREE_LIST_BLOCK_SIZE + } fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { backing.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); let base = backing.gpu_va(); @@ -153,13 +219,16 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let (backing, [global, client]) = alloc.lower_with_aliases( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - [render_global, client], - ) + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + }) .map_err(|error| { if error == ENOSPC { resource_exhausted!("shared lower GPU VA (render free list)") @@ -261,6 +330,10 @@ impl RenderPool { pub(crate) fn vacant(&self) -> bool { self.page_list.load(Ordering::Acquire) == 0 } + /// Bytes of installed backing in the shared lower window, for diagnostics. + pub(crate) fn backing_bytes(&self) -> usize { + self.backing.lock().as_ref().map_or(0, |backing| backing.pages.bytes()) + } /// Pressure reclaim evicted the client VM, which is idle. pub(crate) fn client_evicted(&self) -> bool { self.client_residency.as_ref().is_some_and(|gate| gate.evicted()) @@ -520,6 +593,10 @@ impl ComputePool { pub(crate) fn populated(&self) -> bool { self.pages.is_some() } + /// Bytes of installed backing in the shared lower window, for diagnostics. + pub(crate) fn backing_bytes(&self) -> usize { + self.pages.as_ref().map_or(0, Pages::bytes) + } /// The release record for this pool's buffer slot under the free-list /// generation the queue last published. pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 451f7a113d4a5b..c9596d604df773 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1213,6 +1213,10 @@ impl Queue { pub(crate) fn last_end(&self) -> u64 { self.last_end } + /// Bytes of USC pool backing in the shared lower window, for diagnostics. + pub(crate) fn pool_bytes(&self) -> usize { + self.pool.backing_bytes() + } /// The last owner's file and VM are gone: its execution root was released /// after the scheduler-state release and command retirement were witnessed /// (`Context::release_execution`), and this queue is idle without an owner. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 67320e626c9d28..63afae5a029c10 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1497,9 +1497,16 @@ impl super::Firmware { if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f1..de5e5c486f0c75 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -147,7 +147,13 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { - self.release_backlog(&packet); + // A packet still in publication keeps its place in the backlog: its + // publisher waits on, and a later dependency-free command must not be + // published from its ioctl ahead of it. Publication, cancellation and + // job drop release it. + if packet.is_published() { + self.release_backlog(&packet); + } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34aec..48867cab3ddd1f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -50,7 +50,14 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Unpublished packets wait in publication, which bounds them. Without a + // collector the watchdog expires again and retries. + if !packet.is_published() { + return Ok(()); + } + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 9727477f87ee1b..d1f0f84d1782dd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -388,11 +388,27 @@ impl Registry { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .fold([0usize; 2], |[all, bound], queue| { - [all + 1, bound + usize::from(queue.owner().is_some())] + .fold([0usize; 3], |[all, bound, bytes], queue| { + [all + 1, bound + usize::from(queue.owner().is_some()), bytes + queue.pool_bytes()] }); + // Render pools reachable from installed or prepared entries, each once. + let mut pools = [0usize; 3]; + for (index, entry) in self.render.entries.iter().enumerate() { + let Some(pool) = entry.as_ref().and_then(|entry| entry.pool.as_ref()) else { + continue; + }; + let first = !self.render.entries[..index].iter().flatten().any(|other| { + other.pool.as_ref().is_some_and(|other| Arc::ptr_eq(other, pool)) + }); + if first { + let bytes = pool.backing_bytes(); + pools[0] += 1; + pools[1] += bytes; + pools[2] = pools[2].max(bytes); + } + } pr_warn!( - "asahi: G17 render capacity {}: queues={} slots owned={} released={} held={} quarantined={}; entries={} pairs={} parked={} building={} preparing={} closed={}; retired={}/{}; teardown open={} draining={} releasing={}; qids leased={} pair={} compute={}; compute queues={} bound={}; pb ids={}; metrics={}/{}; render pools={}\n", + "asahi: G17 render capacity {}: queues={} slots owned={} released={} held={} quarantined={}; entries={} pairs={} parked={} building={} preparing={} closed={}; retired={}/{}; teardown open={} draining={} releasing={}; qids leased={} pair={} compute={}; compute queues={} bound={} pools={} MiB; pb ids={}; metrics={}/{}; render pools={} (with entries {} totalling {} MiB, largest {} MiB)\n", reason, self.admission.queues(), owned, @@ -415,10 +431,14 @@ impl Registry { compute_ids, compute[0], compute[1], + compute[2] >> 20, self.buffers.in_use(), metrics, metric_slots, render_pools, + pools[0], + pools[1] >> 20, + pools[2] >> 20, ); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2e88848b932161..b38d0d64aa5e13 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,17 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // A packet still in publication, for example waiting for capacity or + // for its pair's graph, has nothing visible to firmware. Its publisher + // owns its bounded wait and result; the pair may not even be installed. + if !packet.is_published() { + return Ok(()); + } + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 88931b1894e263..67d9888a40731a 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -377,6 +377,63 @@ def item(start): subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'compute-takeover-controls')],check=True) subprocess.run([str(root/'compute-takeover-controls')],check=True) +def lower_window_gate_tests(root, source): + s=(source/'g17/freelist.rs').read_text() + start=s.index('static LOWER_FREED');end=s.index('pub(crate) struct Pages {') + gate=s[start:end] + fixture=root/'lower-window-gate-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::sync::atomic::{AtomicU64, Ordering}; +type Result = std::result::Result; +const ENOSPC:i32=-28;const ENOMEM:i32=-12; +static CLOCK: AtomicU64 = AtomicU64::new(1_000_000_000); +fn now_ns()->u64 {CLOCK.load(Ordering::Relaxed)} +mod freelist { +use super::*; +'''+gate+''' +pub(crate) fn freed() {LOWER_FREED.fetch_add(1, Ordering::AcqRel);} +pub(crate) fn place(size: usize, f: impl FnOnce() -> Result) -> Result {place_lower(size, f)} +} +use freelist::*; +fn attempt(size:usize, result:Result, calls:&mut u32)->Result {place(size,||{*calls+=1;result})} +fn main(){ + let mut calls=0; + assert_eq!(attempt(100,Ok(1),&mut calls),Ok(1));assert_eq!(calls,1);assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,2);assert!(lower_pressure()); + for _ in 0..1000 {assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));} + assert_eq!(attempt(200,Ok(1),&mut calls),Err(ENOSPC)); + assert_eq!(calls,2,"a refused size is not placed again while nothing was freed"); + assert_eq!(attempt(50,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"a smaller size still probes"); + assert_eq!(attempt(60,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"the smaller refusal now gates larger sizes"); + freed();assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,4,"freed backing lets a waiter retry"); + // One probe per interval across all callers, then the gate closes again. + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Ok(7),&mut calls),Ok(7));assert_eq!(calls,6);assert!(!lower_pressure(),"a fitting placement clears pressure"); + assert_eq!(attempt(100,Err(ENOMEM),&mut calls),Err(ENOMEM));assert!(!lower_pressure(),"only window refusals gate"); + println!("lower window: refused free-list placements are not repeated until backing is freed or one probe per 100 ms"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'lower-window-gate-controls')],check=True) + subprocess.run([str(root/'lower-window-gate-controls')],check=True) + +def watchdog_publication_guard_tests(source): + # A packet still in publication (capacity or graph waits) must never reach + # pair lookups or a collector allocation from its watchdog: either error + # stops the runtime for every client. + for path, name in [('g17/runtime/render/settle.rs', 'pub(in crate::g17) fn timeout_render('), + ('g17/runtime/compute_control.rs', 'pub(super) fn timeout_compute(')]: + text = (source / path).read_text() + at = text.index(name) + body = text[at:text.index('\n }\n', at)] + guard = body.index('if !packet.is_published() {\n return Ok(());') + collector = body.index('let Ok(mut deferred) = DeferredBatch::worker() else {') + assert guard < collector and 'DeferredBatch::worker()?' not in body, name + print('watchdog: unpublished packets and collector allocation failure never stop the runtime') + def shared_alias_tests(root, source): s=(source/'g17/object.rs').read_text() at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 @@ -1057,4 +1114,6 @@ def item(s, start): queue_tests(Path(temporary), DRIVER) compute_takeover_tests(Path(temporary), DRIVER) compute_admission_tests(Path(temporary), DRIVER) + lower_window_gate_tests(Path(temporary), DRIVER) + watchdog_publication_guard_tests(DRIVER) shared_alias_tests(Path(temporary), DRIVER) From 0ce71e35d8563bab1cdd1a7fa9a3200558bb1812 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:53 -0400 Subject: [PATCH 14/22] drm/asahi: wait for G17 contexts of closing queues at queue creation Every logical queue leases one of the 58 application contexts of the TTBAT (5..62) when it is created. A closed queue keeps that lease until its scheduler release has been consumed and the reclaim worker returns it; warm retained compute queues and render pairs keep only storage, not the context. After heavy device churn, contexts of queues closed moments earlier can therefore still be leased while fewer than 58 queues are open, and a new queue failed QUEUE_CREATE with ENOSPC without any kernel message. Userspace creates its kernel queue at the first submission and reports that as a lost device. When the context allocation is refused and some leased contexts belong to closed queues, kick the event worker and retry for a bounded time with neither the device mutex nor the preparation lease held. Only open queues then count against the limit. Refusals that remain are reported once as exhausted logical queues or execution contexts. Add debugfs asahi-g17/capacity with one line of queue resource counts: open queues, leased contexts and those held by closing queues, admission slots, render entries, retired bindings, scheduler releases by phase, queue IDs, compute queues and pools, and parameter-buffer, counter and render free-list identities. The render capacity census prints the same line. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/mod.rs | 21 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 60 ++++++-- drivers/gpu/drm/asahi/g17/runtime/render.rs | 140 ++++++++++++------ .../drm/asahi/g17/runtime/render/submit.rs | 4 +- drivers/gpu/drm/asahi/mmu.rs | 6 + 5 files changed, 171 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 7f0bbcd8e4edbe..5ea909e791d348 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -852,9 +852,25 @@ pub(crate) struct Gpu { shared: Arc, _memory: Pin>>, _statistics: Pin>>, + _capacity: Pin>>, _telemetry: Option, } +/// One line of queue resource counts, taken with the device mutex. `contexts` +/// counts leased application contexts against the TTBAT limit; `closing` are +/// those still held by closed queues whose scheduler release has not completed. +struct CapacityView(Arc); + +impl kernel::debugfs::Writer for CapacityView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let state = self.0.state.lock(); + match (*state).as_deref() { + Some(firmware) => writeln!(f, "version=1 {}", firmware.capacity()), + None => writeln!(f, "version=1 stopped"), + } + } +} + struct StatisticsView(Arc); impl kernel::debugfs::Writer for StatisticsView { @@ -961,6 +977,10 @@ impl Gpu { directory.read_only_file(c"statistics", StatisticsView(report_statistics)), GFP_KERNEL, )?; + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", CapacityView(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -971,6 +991,7 @@ impl Gpu { shared, _memory: memory, _statistics: statistics, + _capacity: capacity, _telemetry: telemetry, }, GFP_KERNEL, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 63afae5a029c10..cf34a7d73d3085 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -509,24 +509,55 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed. When only such contexts block this one, wait + // for the teardown (bounded) instead of failing a live client; the + // preparation lease is not held while sleeping. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } - let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + let slot = firmware.queues.admission.reserve_queue().map_err(|error| { + if error == ENOSPC { + resource_exhausted!("logical queues") + } else { + error + } + })?; + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_in_use().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(resource_exhausted!("execution contexts")); + } + drop(slot); + drop(state); + drop(lease); + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -1048,6 +1079,11 @@ impl compute::Host for ComputeHost<'_> { } impl super::Firmware { + /// One snapshot of every finite queue resource. + pub(in crate::g17) fn capacity(&self) -> render::Capacity { + self.queues.capacity(self.render_ids.in_use(), self.uat.execution_contexts_in_use()) + } + fn publish_compute( &mut self, owner: u64, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index d1f0f84d1782dd..95a8ed8ec9f941 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -58,6 +58,79 @@ struct Entry { published: bool, } +/// Counts of every finite queue resource at one instant. Displayed on one line. +pub(in crate::g17) struct Capacity { + queues: usize, + contexts: usize, + slots: [usize; 4], + entries: [usize; 6], + retired: [usize; 2], + teardown: [usize; 3], + qids: (usize, usize, usize), + compute: [usize; 3], + buffers: usize, + metrics: (usize, usize), + render_pools: usize, + pools: [usize; 3], +} + +// Printed by `pr_warn!` (kernel formatting) and written to debugfs (core formatting). +impl kernel::fmt::Display for Capacity { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + core::fmt::Display::fmt(self, f) + } +} + +impl core::fmt::Display for Capacity { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let [owned, released, held, quarantined] = self.slots; + let [entries, pairs, parked, building, preparing, closed] = self.entries; + let [open, draining, releasing] = self.teardown; + let (leased, pair_ids, compute_ids) = self.qids; + write!( + f, + "queues={} contexts={}/{} (closing {}); slots owned={} released={} held={} \ + quarantined={}; entries={} pairs={} parked={} building={} preparing={} \ + closed={}; retired={}/{}; teardown open={} draining={} releasing={}; qids \ + leased={} pair={} compute={}; compute queues={} bound={} pools={} MiB; \ + pb ids={}; metrics={}/{}; render pools={} (with entries {} totalling {} MiB, \ + largest {} MiB)", + self.queues, + self.contexts, + crate::mmu::MAX_EXECUTION_CONTEXTS, + self.contexts.saturating_sub(self.queues), + owned, + released, + held, + quarantined, + entries, + pairs, + parked, + building, + preparing, + closed, + self.retired[0], + self.retired[1], + open, + draining, + releasing, + leased, + pair_ids, + compute_ids, + self.compute[0], + self.compute[1], + self.compute[2] >> 20, + self.buffers, + self.metrics.0, + self.metrics.1, + self.render_pools, + self.pools[0], + self.pools[1] >> 20, + self.pools[2] >> 20, + ) + } +} + /// Unpublished objects removed together, then destroyed outside the device lock. pub(super) struct UnusedRenderSpare { _entry: Option, @@ -368,21 +441,17 @@ impl Registry { Ok(true) } - /// One line describing every render capacity pool, for attributing refusals. - pub(super) fn log_render_census(&self, reason: &str, render_pools: usize) { - let mut census = [0usize; 6]; + /// A snapshot of every finite queue resource, for attributing refusals. + pub(in crate::g17) fn capacity(&self, render_pools: usize, contexts: usize) -> Capacity { + let mut entries = [0usize; 6]; for entry in self.render.entries.iter().flatten() { - census[0] += 1; - census[1] += usize::from(entry.pair.is_some()); - census[2] += usize::from(entry.parked.is_some()); - census[3] += usize::from(entry.constructing || entry.growing); - census[4] += usize::from(entry.preparing.is_some()); - census[5] += usize::from(entry.closed); + entries[0] += 1; + entries[1] += usize::from(entry.pair.is_some()); + entries[2] += usize::from(entry.parked.is_some()); + entries[3] += usize::from(entry.constructing || entry.growing); + entries[4] += usize::from(entry.preparing.is_some()); + entries[5] += usize::from(entry.closed); } - let [owned, released, held, quarantined] = self.admission.census(); - let [deferred, draining, releasing] = self.teardown.phases(); - let (leased, pair_ids, compute_ids) = self.qids.census(); - let (metrics, metric_slots) = self.metrics.census(); let compute = self .compute .iter() @@ -407,39 +476,20 @@ impl Registry { pools[2] = pools[2].max(bytes); } } - pr_warn!( - "asahi: G17 render capacity {}: queues={} slots owned={} released={} held={} quarantined={}; entries={} pairs={} parked={} building={} preparing={} closed={}; retired={}/{}; teardown open={} draining={} releasing={}; qids leased={} pair={} compute={}; compute queues={} bound={} pools={} MiB; pb ids={}; metrics={}/{}; render pools={} (with entries {} totalling {} MiB, largest {} MiB)\n", - reason, - self.admission.queues(), - owned, - released, - held, - quarantined, - census[0], - census[1], - census[2], - census[3], - census[4], - census[5], - self.render.retired.len(), - self.render.retired.capacity(), - deferred, - draining, - releasing, - leased, - pair_ids, - compute_ids, - compute[0], - compute[1], - compute[2] >> 20, - self.buffers.in_use(), - metrics, - metric_slots, + Capacity { + queues: self.admission.queues(), + contexts, + slots: self.admission.census(), + entries, + retired: [self.render.retired.len(), self.render.retired.capacity()], + teardown: self.teardown.phases(), + qids: self.qids.census(), + compute, + buffers: self.buffers.in_use(), + metrics: self.metrics.census(), render_pools, - pools[0], - pools[1] >> 20, - pools[2] >> 20, - ); + pools, + } } fn has_render_ids(&self, owner: u64) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 48aad17dfbaad5..ff9c89ddb596bc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -62,9 +62,7 @@ impl Backend { fn log_render_census(&self, reason: &str) { let state = self.shared.state.lock(); if let Some(firmware) = (*state).as_deref() { - firmware - .queues - .log_render_census(reason, firmware.render_ids.in_use()); + pr_warn!("asahi: G17 render capacity {}: {}\n", reason, firmware.capacity()); } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f4fb0b4bb50f8b..d1d9002298d703 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2529,6 +2529,12 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Application contexts currently leased, including those of closed queues + /// whose scheduler release has not completed. + pub(crate) fn execution_contexts_in_use(&self) -> usize { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() as usize + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { From ffca5762f05981ccd78dea688d514bc38f5fd0e0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:53 -0400 Subject: [PATCH 15/22] drm/asahi: keep G17 render kick ring storage for the life of the device A pair installs the client alias of each kick ring into its VM when the pair is built, and the VM keeps that alias until it closes: a client that gets the pair back later reuses it. Under identity pressure, an installed pair whose identities were never published is reclaimed and its rings are dropped. That frees the rings' shared lower addresses while the building client still aliases the old pages there. A later ring placed at the same address and handed to that client cannot be aliased: its preparation finds the client's mapping naming other pages and fails with EFAULT, and the client's render is reported as a lost device. Held GLES clients hit this every few seconds while fresh clients came and went. Keep the storage of a dropped ring and reuse it for later rings. An address then keeps naming the same pages, so every alias a client keeps stays valid; a reused ring is zeroed and keeps an existing alias. Storage of a ring that ever wrote an entry is retained and never reused. A new ring is counted only once its storage exists, reserving room to return it then, so a dropped ring never allocates and a failed creation leaves no reservation behind. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/kick.rs | 163 ++++++++++++++++-- drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/render.rs | 8 +- .../drm/asahi/g17/runtime/render/prepare.rs | 3 + 4 files changed, 161 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f288fa643a4eaa..6fed8ee6dc4ce7 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -271,10 +278,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -310,6 +401,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -344,14 +438,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -390,14 +515,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -407,19 +544,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -432,7 +569,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 5ea909e791d348..47efa034af0b98 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -502,6 +502,8 @@ struct Firmware { registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -1068,6 +1070,7 @@ impl Gpu { registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 1e28d2d37bb385..33ff9f9423fd96 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -183,6 +183,7 @@ impl Pair { pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -220,8 +221,9 @@ impl Pair { } let layout = target::Layout::new(pass, clusters)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -235,7 +237,7 @@ impl Pair { word, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 9bce9d5dfc45b8..ba1605c98215a5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,6 +29,7 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), @@ -251,6 +253,7 @@ impl super::super::Backend { Pair::new( &alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, From 9c4ffd7a2f5a485cee19c230ea48382788cdf847 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 17:57:54 -0400 Subject: [PATCH 16/22] drm/asahi: bound G17 publication waits by one deadline and keep them out of the submit ioctl A publication that meets contention waits in several phases, each with its own budget: the admission pick (8 s), two render capacity waits (8 s each), borrowed or settling pairs (8 s), up to 1000 work-state polls, and for compute 2 s each for binding and ring backpressure, 200 ms for a spared queue's witness and 2 s for pool growth. One packet could wait about 30 s before failing. Dependency-free renders are published from the submit ioctl, which holds the logical queue's mutex, so a client killed while its ioctl waited stalled for that long. Submission-time compute binding also slept for a spared queue's witness and failed the submit with EBUSY after 200 ms. Give each publication from the scheduler's worker a single deadline, four job timeouts after it starts, shared by every contention wait including the admission pick, which keeps its own shorter budgets within it. Expiry still fails the packet with the error of the phase that waited. The submit ioctl no longer sleeps for contention. An ioctl publication that would wait (capacity, a borrowed or settling pair, work states, a closed preparation epoch during recovery, an admission pick) returns its admission lease and its publication claim before anything became firmware-visible, and leaves the command to the scheduler like a command with dependencies; later commands of the queue then wait behind it in the backlog. Optional sibling warming treats this like any other skipped build. Submission-time compute binding is a single attempt and leaves binding to publication. Compute publication now also waits for a free work state, as render does, instead of failing the packet with EBUSY. A queue creation waiting for contexts of closing queues returns ERESTARTSYS when a signal is pending; nothing of the new queue is visible at that point. A publication that gives up at its deadline names what it waited for, the source line and the owner's pid in one warning, at most once a second with a count of the suppressed ones, so a lost device reported for contention can be attributed from the kernel log alone. For a busy render pair it also names the pair and the state that kept it busy. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 6 + drivers/gpu/drm/asahi/g17/job.rs | 7 + drivers/gpu/drm/asahi/g17/queue/admission.rs | 40 +++- drivers/gpu/drm/asahi/g17/runtime.rs | 201 ++++++++++++++---- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- .../drm/asahi/g17/runtime/render/prepare.rs | 27 ++- .../drm/asahi/g17/runtime/render/submit.rs | 173 +++++++++++---- 7 files changed, 357 insertions(+), 101 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a3888f98231dba..842f99e18a4dd9 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -401,6 +401,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b5a5b3759de6dd..9bc6f0e7df5a15 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -179,6 +179,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 7ef89ed9b58ffb..644ee02708be0e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -466,14 +466,26 @@ impl Pool { } } } + /// Waits at most until `deadline` (monotonic ns), the bound of the whole + /// publication, besides each wait's own budget. pub(crate) fn acquire_render( self: &Arc, owner: u64, request: Option>, + deadline: u64, ) -> Result { if owner == 0 { return Err(EINVAL); } + // A wait of at most `budget` jiffies, cut short at the deadline. + let capped = |budget: kernel::time::Jiffies| { + let left = deadline.saturating_sub(crate::g17::now_ns()) / 1_000_000; + budget.min(kernel::time::msecs_to_jiffies(left.min(u64::from(u32::MAX)) as u32)) + }; + // The budget left after waiting `timeout` jiffies of which `left` remained. + let spend = |budget: kernel::time::Jiffies, + timeout: kernel::time::Jiffies, + left: kernel::time::Jiffies| budget.saturating_sub(timeout.saturating_sub(left)); let mut state = self.state.lock(); let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); @@ -492,16 +504,19 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { + let timeout = capped(quarantine_budget); + if timeout == 0 { return Err(EIO); } quarantine_budget = match self .released - .wait_interruptible_timeout(&mut state, quarantine_budget) + .wait_interruptible_timeout(&mut state, timeout) { CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, + CondVarTimeoutResult::Woken { jiffies } => { + spend(quarantine_budget, timeout, jiffies) + } }; continue; } @@ -531,30 +546,39 @@ impl Pool { } } } + let timeout = capped(conflict_budget); + if timeout == 0 { + // Out of time for overlap: take any slot without waiting. + expired = true; + continue; + } match self .released - .wait_interruptible_timeout(&mut state, conflict_budget) + .wait_interruptible_timeout(&mut state, timeout) { CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), CondVarTimeoutResult::Timeout => { conflict_budget = 0; expired = true; } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + CondVarTimeoutResult::Woken { jiffies } => { + conflict_budget = spend(conflict_budget, timeout, jiffies) + } } } else { // Never wait unboundedly in publication: the scheduler watchdog // cannot fail a packet that has not claimed hardware yet. - if pick_budget == 0 { + let timeout = capped(pick_budget); + if timeout == 0 { return Err(EBUSY); } pick_budget = match self .released - .wait_interruptible_timeout(&mut state, pick_budget) + .wait_interruptible_timeout(&mut state, timeout) { CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, + CondVarTimeoutResult::Woken { jiffies } => spend(pick_budget, timeout, jiffies), }; } })(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index cf34a7d73d3085..ab55dda6013b2e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -32,6 +32,46 @@ const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Bound on all contention waits of one publication together: capacity and +/// identities held by other live owners, borrowed pairs, work states, queue +/// binding and ring backpressure. Each frees when published work retires or +/// times out, when warming finishes or is reclaimed, or when an owner closes. +const PUBLICATION_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + +/// How a publication may wait for contention. +#[derive(Clone, Copy)] +pub(super) enum Wait { + /// From the scheduler's worker, until one deadline for the whole publication. + Until(u64), + /// From the submitting ioctl, which holds the logical queue's mutex: never + /// sleep for contention, return `FALLBACK`. + Never, +} + +impl Wait { + pub(super) fn scheduled() -> Self { + Self::Until(now_ns().saturating_add(PUBLICATION_WAIT_NS)) + } + /// Before a contention wait: `FALLBACK` when it may not sleep at all, + /// otherwise whether the publication's deadline has passed. + pub(super) fn expired(self) -> Result { + match self { + Self::Never => Err(FALLBACK), + Self::Until(deadline) => Ok(now_ns() >= deadline), + } + } + pub(super) fn deadline(self) -> Option { + match self { + Self::Never => None, + Self::Until(deadline) => Some(deadline), + } + } +} /// Whether a publication with no queue may take another live owner's idle one: /// when neither a fresh QID nor surplus render capacity remains, or once its @@ -551,6 +591,10 @@ impl Backend { drop(slot); drop(state); drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } shared.queue_events(); kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); } @@ -590,9 +634,9 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, wait: Wait) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); @@ -615,6 +659,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } + if wait.expired()? { + return Err(self.wait_expired(EBUSY, "the shared compute aperture")); + } self.shared.changed.wait(&mut state); } }; @@ -703,7 +750,7 @@ impl Backend { result } - fn grow_compute(&self, required: usize) -> Result { + fn grow_compute(&self, required: usize, wait: Wait) -> Result { use kernel::time::{delay::fsleep, Delta}; { let state = self.shared.state.lock(); @@ -712,14 +759,13 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); loop { let result = (|| { let mut state = self.shared.state.lock(); @@ -738,11 +784,13 @@ impl Backend { deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + Err(error) if error == EAGAIN && lease.is_current() => { + if wait.expired()? { + return Err(self.wait_expired(error, "compute pool growth")); } + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } Err(error) => return Err(error), } } @@ -753,9 +801,9 @@ impl Backend { /// owner's idle queue happens only in publication's device-locked rebind, /// which publishes under the same mutex hold. Submission never waits for /// or fails on physical capacity: it leaves binding to publication. - fn build_compute(&self, required: usize) -> Result { + fn build_compute(&self, required: usize, wait: Wait) -> Result { loop { - match self.shared_compute() { + match self.shared_compute(wait) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -764,7 +812,7 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); @@ -778,7 +826,7 @@ impl Backend { } match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Grow => { drop(state); return self.grow_compute(required, wait); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { @@ -1258,11 +1306,61 @@ impl Backend { /// reports ENOSPC when no physical queue is free, leaving publication to bind. fn ensure_compute(&self, scs_pages: u16) -> Result { let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - self.bind_compute(required) + // The submit ioctl never sleeps for contention: publication binds. + match self.bind_compute(required, Wait::Never) { + Err(error) if error == FALLBACK => Err(EAGAIN), + result => result, + } + } + + /// A publication gave up waiting for `what` at its deadline and fails with + /// `error`. Names the wait, its source line and the owner, at most once a + /// second, so a lost device reported for contention is attributable from + /// the kernel log alone. Only reached on that failure path. + #[track_caller] + pub(super) fn wait_expired(&self, error: Error, what: impl kernel::fmt::Display) -> Error { + use core::sync::atomic::AtomicU64; + static NEXT: AtomicU64 = AtomicU64::new(0); + static SUPPRESSED: AtomicU64 = AtomicU64::new(0); + let now = now_ns(); + let next = NEXT.load(Ordering::Relaxed); + if now < next + || NEXT + .compare_exchange(next, now + 1_000_000_000, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + SUPPRESSED.fetch_add(1, Ordering::Relaxed); + return error; + } + let at = core::panic::Location::caller(); + dev_warn!( + self.shared.dev.as_ref(), + "G17 publication of pid {} gave up waiting for {} ({:?}, {}:{}); {} similar since the last report\n", + self.context.owner_pid(), + what, + error, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), + at.line(), + SUPPRESSED.swap(0, Ordering::Relaxed) + ); + error + } + + /// The preparation lease. From the ioctl, a recovery that has closed the + /// preparation epoch is contention. + fn enter_preparation(&self, wait: Wait) -> Result { + match wait { + Wait::Never => self + .shared + .preparations + .try_enter() + .map_err(|error| if error == EAGAIN { FALLBACK } else { error }), + Wait::Until(_) => self.shared.preparations.enter(false), + } } /// EAGAIN while another binder of this owner runs; it binds for both. - fn bind_compute(&self, required: usize) -> Result { + fn bind_compute(&self, required: usize, wait: Wait) -> Result { use kernel::time::{delay::fsleep, Delta}; if self .binding @@ -1272,9 +1370,8 @@ impl Backend { return Err(EAGAIN); } let _binder = Binder(&self.binding); - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1286,15 +1383,16 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + return self.build_compute(required, wait); + } + // Keep the spared queue and its retained work intact until a later + // admission observes its witness. Other owners continue submitting + // while this owner gives its queue-local drain witness time to arrive. + if wait.expired()? { + return Err(self.wait_expired(EBUSY, "a spared compute queue's drain witness")); } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. fsleep(Delta::from_millis(1)); } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - Err(EBUSY) } /// Called after taking this packet's publication claim. All waits and work @@ -1303,6 +1401,7 @@ impl Backend { &self, packet: &Arc, deferred: &mut DeferredBatch, + wait: Wait, ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { @@ -1311,15 +1410,29 @@ impl Backend { let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; let mut recovery_polls = 0; - let mut outer_wait = None; - let mut binding_wait = None; + let mut state_delay = 20i64; let mut contended = false; loop { if packet.completion.status().get() != 0 { return Err(EIO); } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; + let lease = self.enter_preparation(wait)?; + // Work states free as this context's earlier commands retire. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + if wait.expired()? { + return Err(self.wait_expired(error, "a free compute work state")); + } + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { @@ -1338,8 +1451,8 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); + if wait.expired()? || recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(self.wait_expired(EBUSY, "GPU recovery to settle")); } recovery_polls += 1; drop(state); @@ -1358,10 +1471,8 @@ impl Backend { // took this idle one. Bind again; a takeover publishes under // the same mutex hold. Capacity frees whenever any queue's // published work retires or times out, so the wait is bounded. - let now = now_ns(); - let since = *binding_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); + if wait.expired()? { + return Err(self.wait_expired(EBUSY, "a compute queue to bind")); } // An off-lock binder of this owner may hold a reservation or // a queue being re-armed: binding here could give the owner a @@ -1383,7 +1494,7 @@ impl Backend { // Replays collected above belong to other packets; do not // hold their fences while this one waits for a queue. deferred.finish(); - match self.bind_compute(required) { + match self.bind_compute(required, wait) { Ok(()) => {} Err(error) if [ENOSPC, EBUSY, EAGAIN].contains(&error) => { // No released queue, fresh QID or spare: the next @@ -1409,8 +1520,8 @@ impl Backend { && (firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); + if wait.expired()? || recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(self.wait_expired(EBUSY, "GPU recovery to settle")); } recovery_polls += 1; drop(state); @@ -1420,10 +1531,20 @@ impl Backend { continue; } if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); + if wait.expired()? { + let releasing = firmware.queues.compute(self.owner).is_some_and(|queue| { + queue + .pending_switch_from() + .is_some_and(|previous| firmware.queues.teardown.releasing(previous)) + }); + return Err(self.wait_expired( + EBUSY, + if releasing { + "the previous owner's scheduler release" + } else { + "compute ring room" + }, + )); } // Backpressure owns no firmware-visible attempt. Other queues // can submit and the event worker can retire while we sleep. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index de5e5c486f0c75..78d26ae6262cad 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -138,7 +138,9 @@ impl job::Backend for Backend { Err(EIO) } else { match packet.engine() { - Engine::Compute => self.compute_packet(&packet, &mut deferred), + Engine::Compute => { + self.compute_packet(&packet, &mut deferred, super::Wait::scheduled()) + } Engine::Render => self.render_packet(&packet, &mut deferred), } }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index ba1605c98215a5..9a42d14fbd6675 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -74,8 +74,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + wait: super::super::Wait, + ) -> Result { + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -576,8 +582,13 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render( + &self, + packet: &Arc, + slot: u8, + wait: super::super::Wait, + ) -> Result { + self.build_render_pair(slot, packet, false, wait)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -595,8 +606,9 @@ impl super::super::Backend { // Warming is optional: exhausted identities, VA or memory must not // fail a command whose own pair is ready. Its failed construction // leaves the reservation unchanged, as for the primary pair. - match self.build_render_pair(sibling.slot(), packet, true) { - Err(error) if [EBUSY, EAGAIN, ENOSPC, ENOMEM].contains(&error) => {} + match self.build_render_pair(sibling.slot(), packet, true, wait) { + Err(error) + if [EBUSY, EAGAIN, ENOSPC, ENOMEM, super::super::FALLBACK].contains(&error) => {} result => result?, } } @@ -608,8 +620,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + wait: super::super::Wait, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index ff9c89ddb596bc..a6dd3f5bd7fc8b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -2,7 +2,7 @@ //! Submission admission, preparation and the firmware visibility boundary. -use super::super::{Backend, DeferredBatch, Registry}; +use super::super::{Backend, DeferredBatch, Registry, Wait, FALLBACK}; use crate::g17::{ job::Packet, queue::admission::{RenderLease, Request}, @@ -14,30 +14,23 @@ use kernel::{ time::{delay::fsleep, Delta}, }; -/// Bound on waiting for render capacity held by other live owners: admission -/// slots, queue IDs, parameter-buffer and counter identities, retired-binding -/// room and shared GPU VA. Capacity frees when a warming graph is reclaimed or -/// finishes construction, when published work retires or times out, when a -/// displaced binding's scheduler release completes, or when an owner closes. -const RENDER_CAPACITY_WAIT_NS: u64 = 4 * crate::g17::job::TIMEOUT_MS as u64 * 1_000_000; - impl Backend { - /// Every slot or QID is held by live owners whose spares are still being - /// built or whose siblings are busy. Retry reclamation after their next - /// release instead of failing the packet; fail only after the bound. - fn wait_render_capacity(&self, since: &mut Option) -> Result { + /// Render capacity is held by other live owners: admission slots, queue + /// IDs, parameter-buffer and counter identities, retired-binding room or + /// shared GPU VA, whose spares are still being built or whose siblings are + /// busy. Retry reclamation after their next release instead of failing + /// the packet; fail only once the publication's deadline has passed. + fn wait_render_capacity(&self, wait: Wait, started: &mut bool) -> Result { use core::sync::atomic::AtomicBool; static STARTED: AtomicBool = AtomicBool::new(false); static EXPIRED: AtomicBool = AtomicBool::new(false); - let now = super::super::now_ns(); - let first = since.is_none(); - if now.saturating_sub(*since.get_or_insert(now)) >= RENDER_CAPACITY_WAIT_NS { + if wait.expired()? { if !EXPIRED.swap(true, Ordering::Relaxed) { self.log_render_census("wait expired"); } return Err(resource_exhausted!("render capacity held by open owners")); } - if first && !STARTED.swap(true, Ordering::Relaxed) { + if !core::mem::replace(started, true) && !STARTED.swap(true, Ordering::Relaxed) { self.log_render_census("wait started"); } self.shared.queue_events(); @@ -47,11 +40,20 @@ impl Backend { /// A pair is borrowed, still settling, or waiting for its previous /// scheduler's release to be consumed and parked (the event worker does - /// both). Poll briefly; time out only after the capacity bound. - fn wait_render_busy(&self, since: &mut Option) -> Result { - let now = super::super::now_ns(); - if now.saturating_sub(*since.get_or_insert(now)) >= RENDER_CAPACITY_WAIT_NS { - return Err(ETIMEDOUT); + /// both). Poll briefly until the publication's deadline, then name the + /// state that kept pair `slot` busy. + fn wait_render_busy(&self, slot: u8, wait: Wait) -> Result { + if wait.expired()? { + let cause = { + let state = self.shared.state.lock(); + (*state).as_deref().map_or("the device stopped", |firmware| { + firmware.render_busy_cause(slot, self.owner, &self.context) + }) + }; + return Err(self.wait_expired( + ETIMEDOUT, + format_args!("a busy render pair (slot {}: {})", slot, cause), + )); } self.shared.queue_events(); fsleep(Delta::from_micros(200)); @@ -71,7 +73,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - self.render_with_lease(packet, None, deferred) + self.render_with_lease(packet, None, Wait::scheduled(), deferred) } /// Reserve identities before attaching a lease to the packet. An empty @@ -81,12 +83,13 @@ impl Backend { &self, packet: &Arc, mut next: Option, + wait: Wait, ) -> Result { let pool = { let state = self.shared.state.lock(); (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() }; - let mut capacity_wait = None; + let mut capacity_wait = false; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -97,8 +100,15 @@ impl Backend { let lease = if let Some(lease) = next.take() { lease } else { - match pool.acquire_render(self.owner, Request::of(packet)) { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + let Some(deadline) = wait.deadline() else { + return Err(FALLBACK); + }; + match pool.acquire_render(self.owner, Request::of(packet), deadline) { Ok(lease) => lease, + Err(error) if error == EBUSY => { + return Err(self.wait_expired(error, "a render admission slot")) + } Err(error) if error == ENOSPC => { // Live owners hold every slot. Prefer unused warming, // then an idle sibling of an owner keeping another pair. @@ -112,7 +122,7 @@ impl Backend { (reclaimed, forfeited) }; if reclaimed.is_none() && !forfeited { - self.wait_render_capacity(&mut capacity_wait)?; + self.wait_render_capacity(wait, &mut capacity_wait)?; continue; } drop(reclaimed); @@ -140,7 +150,7 @@ impl Backend { } if !lease.cancel_empty(self.owner) { drop(state); - self.wait_render_capacity(&mut capacity_wait)?; + self.wait_render_capacity(wait, &mut capacity_wait)?; continue; } let reclaimed = firmware @@ -154,7 +164,7 @@ impl Backend { && firmware.queues.forfeit_idle_render_sibling(self.owner)?; drop(state); if reclaimed.is_none() && !ready && !forfeited { - self.wait_render_capacity(&mut capacity_wait)?; + self.wait_render_capacity(wait, &mut capacity_wait)?; continue; } drop(reclaimed); @@ -168,6 +178,7 @@ impl Backend { &self, packet: &Arc, early: Option, + wait: Wait, deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; @@ -177,15 +188,12 @@ impl Backend { if packet.completion.status().get() != 0 { return Err(EIO); } - let interruptible = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early)?; + let admission = self.acquire_render_with_ids(packet, early, wait)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; let mut state_delay = 20i64; - let mut busy_since = None; let mut recovery_polls = 0; - let mut capacity_wait = None; + let mut capacity_wait = false; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -198,7 +206,7 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = self.enter_preparation(wait)?; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -209,10 +217,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if state_polls == 1000 { - return Err(ETIMEDOUT); + if wait.expired()? { + return Err(self.wait_expired(ETIMEDOUT, "a free render work state")); } - state_polls += 1; fsleep(Delta::from_micros(state_delay)); state_delay = (state_delay * 2).min(1000); continue; @@ -221,21 +228,21 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, wait) { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since)?; + self.wait_render_busy(slot, wait)?; continue; } // Graph or binding resources held by other owners: every failed // build left its reservation unchanged, so retry after a release. Err(error) if error == ENOSPC => { - self.wait_render_capacity(&mut capacity_wait)?; + self.wait_render_capacity(wait, &mut capacity_wait)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, wait) { Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { let recovering = { let state = self.shared.state.lock(); @@ -244,19 +251,21 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); + if wait.expired()? || recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(self.wait_expired(ETIMEDOUT, "GPU recovery to settle")); } recovery_polls += 1; self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - self.wait_render_busy(&mut busy_since)?; + self.wait_render_busy(slot, wait)?; + } else if wait.expired()? { + return Err(self.wait_expired(ETIMEDOUT, "render preparation")); } } // Unpublished: the preparation was discarded and the pair returned. Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(&mut capacity_wait)?; + self.wait_render_capacity(wait, &mut capacity_wait)?; } result => return result, } @@ -297,7 +306,17 @@ impl Backend { return true; } }; - let result = self.render_with_lease(packet, Some(admission), &mut deferred); + let result = self.render_with_lease(packet, Some(admission), Wait::Never, &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -348,6 +367,70 @@ impl Registry { } impl crate::g17::Firmware { + /// The state of pair `slot` that refuses a publication of `owner` with + /// EBUSY, for the report of a busy wait that reached its deadline. Checks + /// follow the order of the refusals in construction and preparation. + fn render_busy_cause( + &self, + slot: u8, + owner: u64, + context: &Arc, + ) -> &'static str { + use crate::g17::fw::queue::DataMaster; + let Ok(entry) = self.queues.render.entry(slot) else { + return "no queue identities"; + }; + if entry.pending_error.is_some() || entry.pending_timeout { + return "a deferred failure or timeout awaits the event worker"; + } + if entry.constructing { + return "its graph is being constructed"; + } + if entry.growing { + return "its parameter buffer is growing"; + } + if entry.preparing.is_some() { + return if entry.pending_owner.is_some_and(|other| other != owner) { + "another owner is preparing it" + } else { + "a preparation has borrowed it" + }; + } + let Some(pair) = entry.pair.as_ref() else { + return if entry.parked.is_some() { + "parked" + } else { + "no installed pair" + }; + }; + if pair.matches(owner, context).unwrap_or(false) { + let priority = context.policy().priority(); + return if pair + .owner_switch_from(context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + "the release of the scheduler it last ran is not consumed yet" + } else if !self.init.work_ready(priority, DataMaster::Fragment).unwrap_or(false) + || !self.init.work_ready(priority, DataMaster::Tiling).unwrap_or(false) + { + "the outer work ring is full" + } else if !pair.room() { + "its command ring is full" + } else { + "installed for this owner" + }; + } + if self.queues.teardown.releasing(pair.context()) { + "the release of its previous owner's scheduler is in progress" + } else if pair.owner().is_some() { + "another owner holds it" + } else if !pair.reusable().unwrap_or(false) { + "its previous owner's work or displaced graph has not settled" + } else { + "reusable" + } + } + pub(super) fn apply_render_deferred( &mut self, slot: u8, From 98a5ccbeefdb5521d99c7a7597a533b4d3070d73 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 15:34:01 -0400 Subject: [PATCH 17/22] drm/asahi: rerun the G17 event pass for completions missed while a pair is borrowed Event passes observe only render pairs that are installed in their entry. A pair that preparation, growth or an observation has borrowed is skipped, and nothing requested another pass when it came back: the completion of work published on that pair waited for an unrelated later firmware event, or for the one-millisecond rescan when no further event arrived. Record a completion event that arrives for a borrowed pair and have the borrower request an event pass when it returns the pair. Growth needs no request of its own; its work item always queues one when it finishes. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime/render.rs | 24 ++++++++++++++++++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 2 ++ .../drm/asahi/g17/runtime/render/observe.rs | 2 +- .../drm/asahi/g17/runtime/render/prepare.rs | 9 +++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 95a8ed8ec9f941..a532dadb09494d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -50,6 +50,9 @@ struct Entry { /// nothing, so the snapshot can only be stale towards an older switch. borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -206,6 +209,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -532,6 +543,7 @@ impl Registry { borrowed_dependencies: None, borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -1062,9 +1074,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 9e229855012e53..1090349072e150 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -273,6 +273,8 @@ impl crate::g17::Shared { entry.borrowed_dependencies = None; entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 51fe6428fae481..8bc8dad5d6bb51 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -165,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 9a42d14fbd6675..9e965fa870ce61 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -415,6 +415,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -525,6 +528,9 @@ impl super::super::Backend { entry.pair = Some(pair); entry.borrowed_dependencies = None; entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -698,6 +704,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues From dfef788d3a573d4642cf6cd37fba1f557b3e9954 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 15:34:01 -0400 Subject: [PATCH 18/22] drm/asahi: wait for a borrowed G17 render pair to return, not 200 us Publication retries with a fixed 200 us sleep when the pair it needs is busy. The usual cause is a borrow: an event pass observing the pair, or a preparation or growth of it. Each returns the pair under the device mutex and then notifies `changed`, typically within microseconds, so a submission that collided with an observation slept for ~200 us for no reason. When the pair is borrowed, wait on `changed` (bounded by one tick) under the device mutex, so the return wakes the submission at once. Other busy causes keep the 200 us poll. Both stay within the publication's deadline. A publication from the submit ioctl does not wait at all: it hands the command to the scheduler, as for any other contention. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime/render.rs | 8 ++++++++ .../gpu/drm/asahi/g17/runtime/render/submit.rs | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a532dadb09494d..77ca6e85026037 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -209,6 +209,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether the pair in `slot` is out of its entry, borrowed by an event + /// pass, a preparation or a growth that returns it under the device mutex. + fn borrowed(&self, slot: u8) -> bool { + self.entry(slot).is_ok_and(|entry| { + entry.pair.is_none() && (entry.preparing.is_some() || entry.growing) + }) + } + /// Whether a completion event arrived while the pair in `slot` was /// borrowed; clears the record. The returned pair is installed again, so /// the next event pass observes it. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index a6dd3f5bd7fc8b..c80d8f3695de23 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -42,6 +42,11 @@ impl Backend { /// scheduler's release to be consumed and parked (the event worker does /// both). Poll briefly until the publication's deadline, then name the /// state that kept pair `slot` busy. + /// + /// A borrower (an event pass observing the pair, a preparation or a + /// growth) returns the pair under the device mutex and then notifies + /// `changed`, usually within microseconds: wait for that notification, + /// bounded by one tick, instead of the fixed sleep. fn wait_render_busy(&self, slot: u8, wait: Wait) -> Result { if wait.expired()? { let cause = { @@ -56,6 +61,16 @@ impl Backend { )); } self.shared.queue_events(); + { + let mut state = self.shared.state.lock(); + let borrowed = (*state) + .as_deref() + .is_some_and(|firmware| firmware.queues.render.borrowed(slot)); + if borrowed { + let _ = self.shared.changed.wait_timeout(&mut state, 1); + return Ok(()); + } + } fsleep(Delta::from_micros(200)); Ok(()) } From 3082345824dd9db5af53a39483a4abd34160beac Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 15:34:02 -0400 Subject: [PATCH 19/22] drm/asahi: queue dependent G17 renders behind their predecessor's pair Admission prefers an idle render pair of the owner over stacking onto a busy one, so a stream of renders that each depend on the previous one alternates between the owner's two pairs. Each render's fragment kick then waits for the previous fragment kick on the other pair through an explicit cross-queue dependency, which the firmware resolves only after completing that kick, and the next fragment work always runs on another queue. When the command's queue prefix already covers the owner's newest live render, the sibling pair cannot run it any earlier: its barrier waits for that render either way. Queue it on that render's pair instead, where the queue's own order provides the dependency, as long as the pair is healthy and has room. Independent renders keep preferring an idle pair, so they can still overlap on two pairs; attachment hazards keep their planned pair. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/admission.rs | 52 +++++++++++++++++--- tools/asahi/g17-resource-tests.py | 47 +++++++++--------- 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 644ee02708be0e..1838462da3553f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -10,7 +10,7 @@ use crate::g17::{ hazard::{self, Footprint, Plan}, - job::Packet, + job::{Order, Packet}, }; use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; use core::sync::atomic::{AtomicU32, Ordering}; @@ -121,10 +121,20 @@ impl State { self.advance(slot); slot as u8 } - fn acquire(&mut self, owner: u64) -> Option { + fn acquire(&mut self, owner: u64, follow: Option) -> Option { if owner == 0 { return None; } + // A command ordered after the owner's newest live render gains no + // concurrency from the sibling: its barrier waits for that render + // anyway. Queue it behind it on the same pair, where the queue's own + // order replaces a cross-queue firmware dependency. + if let Some(slot) = follow { + let index = usize::from(slot); + self.refs[index] += 1; + self.advance(index); + return Some(slot); + } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. let idle = (0..cfg::RENDER_SLOTS) @@ -352,9 +362,37 @@ impl State { ) } - fn pick(&mut self, owner: u64, plan: Plan) -> Option { + /// The pair holding the owner's newest live render when `request` is + /// ordered after that render by its queue prefix, and the pair can take + /// another command. + fn follow(&self, owner: u64, request: Option>) -> Option { + let request = request?; + let newest = self + .footprints + .iter() + .filter(|entry| entry.owner == owner && entry.slot.is_some()) + .max_by_key(|entry| entry.sequence)?; + if !Order::contains(request.prefix, newest.sequence) { + return None; + } + let slot = newest.slot?; + let index = usize::from(slot); + let bit = 1u64 << slot; + if index >= cfg::RENDER_SLOTS + || self.owners[index] != owner + || (self.quarantined | self.kill_hold) & bit != 0 + || self.active & bit == 0 + || self.refs[index] == 0 + || self.refs[index] >= cfg::RENDER_DEPTH + { + return None; + } + Some(slot) + } + + fn pick(&mut self, owner: u64, plan: Plan, follow: Option) -> Option { match plan { - Plan::Any => self.acquire(owner), + Plan::Any => self.acquire(owner, follow), Plan::Wait => None, Plan::Pair(slot) => { let index = usize::from(slot); @@ -526,7 +564,8 @@ impl Pool { } else { state.plan(owner, request, ticket) }; - if let Some(slot) = state.pick(owner, plan) { + let follow = state.follow(owner, request); + if let Some(slot) = state.pick(owner, plan, follow) { state.forget_waiter(ticket.take()); let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -601,7 +640,8 @@ impl Pool { return Ok(None); } let plan = state.plan(owner, request, None); - let Some(slot) = state.pick(owner, plan) else { + let follow = state.follow(owner, request); + let Some(slot) = state.pick(owner, plan, follow) else { return Ok(None); }; let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 67d9888a40731a..f3b3518d15fa22 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -34,25 +34,25 @@ def item(start): let mut state=State::new(); for owner in 1..=32 { assert!(state.refusal(owner).is_ok()); - let slot=state.acquire(owner).expect("existing owner fits"); + let slot=state.acquire(owner,None).expect("existing owner fits"); state.release(slot); } assert_eq!(state.active,0,"all submitted work retired"); let retained=state.owners.iter().filter(|o|**o!=0).count(); assert_eq!(retained,63); assert_eq!(state.refusal(33),Err(ENOSPC)); - assert!(state.acquire(33).is_none()); + assert!(state.acquire(33,None).is_none()); println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); // Yield one unused spare for each additional logical owner. The runtime // separately proves that these physical graphs have never been published. for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); - assert_eq!(state.acquire(owner),Some(slot as u8));state.release(slot as u8); + assert_eq!(state.acquire(owner,None),Some(slot as u8));state.release(slot as u8); } - for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner).unwrap();state.release(slot);} + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner,None).unwrap();state.release(slot);} assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); - let mut guarded=State::new();let first=guarded.acquire(1).unwrap();guarded.release(first); + let mut guarded=State::new();let first=guarded.acquire(1,None).unwrap();guarded.release(first); let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; guarded.take_idle(spare,1); assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); @@ -67,10 +67,10 @@ def item(start): assert!(!guarded.reassign_unused_spare(64,None,Some(3))); guarded.released |= 1u64<Vec<(u8,u64)> {(1..cfg::RENDER_SLOTS).filter(|slot|s.owners[*slot]!=0).map(|slot|(slot as u8,s.owners[slot])).collect()}; let mut live=State::new(); - for owner in 1..=32 {let slot=live.acquire(owner).unwrap();live.release(slot);} + for owner in 1..=32 {let slot=live.acquire(owner,None).unwrap();live.release(slot);} assert_eq!(live.refusal(33),Err(ENOSPC)); let mut gifts=Vec::new(); for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { @@ -112,11 +115,11 @@ def item(start): let (slot,donor)=live.forfeit_sibling(&candidates(&live),0,owner).expect("an owner with two pairs donates one"); assert_eq!(live.owners[slot as usize],owner);assert_eq!(live.epochs[slot as usize],0); assert_eq!(live.owners.iter().filter(|o|**o==donor).count(),1,"donor keeps exactly one pair"); - assert_eq!(live.acquire(owner),Some(slot));live.release(slot); + assert_eq!(live.acquire(owner,None),Some(slot));live.release(slot); gifts.push((slot as usize,donor)); } for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - assert!(live.refusal(owner).is_ok());let slot=live.acquire(owner).unwrap();live.release(slot); + assert!(live.refusal(owner).is_ok());let slot=live.acquire(owner,None).unwrap();live.release(slot); } println!("forfeit: all{} live logical owners render while idle siblings move; every donor keeps a pair",MAX_EXECUTION_CONTEXTS); // A donor never receives another pair while open: neither free capacity @@ -126,18 +129,18 @@ def item(start): let closed=live.owners.iter().position(|o|*o==u64::from(MAX_EXECUTION_CONTEXTS)).unwrap(); assert_ne!(given,closed,"control: the released pair was not this donor's"); live.released|=1u64<Vec {(0..cfg::RENDER_SLOTS).filter(|s|g.owners[*s]==o).collect()}; let (a,b)=(pairs(&g,1)[0],pairs(&g,1)[1]); g.refs[b]=1;assert_eq!(g.forfeit_sibling(&[(b as u8,1)],0,9),None,"borrowed");g.refs[b]=0; @@ -158,7 +161,7 @@ def item(start): assert_eq!(pairs(&g,2),vec![b]);assert_eq!(g.owners[held],0); assert_eq!(g.forfeit_sibling(&[(a as u8,1)],0,3),None,"never take a donor's last pair"); let mut lru=State::new(); - for owner in 1..=2 {let slot=lru.acquire(owner).unwrap();lru.release(slot);} + for owner in 1..=2 {let slot=lru.acquire(owner,None).unwrap();lru.release(slot);} let one=pairs(&lru,1);let two=pairs(&lru,2); lru.epochs[one[0]]=7;lru.epochs[one[1]]=9;lru.epochs[two[0]]=8;lru.epochs[two[1]]=3; let all=candidates(&lru); @@ -773,7 +776,7 @@ def body(s, start): } // One command: admission, then construction and publication on its pair. fn render(s:&mut State,m:&mut Model,owner:u64)->u8 { - let slot=s.acquire(owner).unwrap_or_else(||panic!("owner {} found no pair",owner)); + let slot=s.acquire(owner,None).unwrap_or_else(||panic!("owner {} found no pair",owner)); m.publish(owner,slot as usize); slot } From 41ef5e9c4d45dd05af23876c6b0bfb8a071c0381 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 15:34:02 -0400 Subject: [PATCH 20/22] drm/asahi: reuse G17 compute timestamp pages within a context Every compute command got a fresh page for its firmware start/end words: a GEM object, its write-combined kernel mapping and a GPU mapping were set up in the submit ioctl, and torn down again when the command's completion was dropped. On Neo a lone dispatch spends ~95 us from its fence signal to the next command's ioctl, against ~17 us for a render, which has no such per-command allocation. Keep up to eight retired pages per context, mapped in its VM, and hand them out again with both words cleared. A page returns only when its completion is dropped, which follows exact retirement or a kill witness, so the firmware no longer writes it. Commands beyond the retained count still allocate and free their page. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 88 +++++++++++++++++++++++-- drivers/gpu/drm/asahi/g17/context.rs | 6 ++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 842f99e18a4dd9..34c7cdca3c4294 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,81 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +249,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -216,7 +292,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -278,14 +354,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -335,7 +411,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 333a6a3fcf54d8..e1c890ca16f91c 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -195,12 +197,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } From 46a16d94e388be6d9c0d1087d71984e6f99b601d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 15:34:02 -0400 Subject: [PATCH 21/22] drm/asahi: publish dependency-free G17 compute commands from the submit ioctl Renders without dependencies are published from the submitting ioctl; compute commands always went through the scheduler's worker, which costs a worker wakeup per command: ~30 us from ioctl to doorbell for a lone dispatch on Neo, against ~12 us for a render. Publish a compute command from the ioctl under the same conditions as a render (no dependencies, a shallow queue) when its queue is already bound, fits it and has room, and no earlier compute command of the owner still waits in the scheduler. Count scheduled compute commands separately from renders, so the existing render condition is unchanged. The ioctl makes one attempt under one device mutex hold and never sleeps. Binding, takeover, pool growth, replays of recovered commands, a missing work state, ring backpressure and recovery all return the publication claim before anything became visible and leave the command to the scheduler, whose path is unchanged. The ioctl's call chain therefore does not include queue binding. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/job.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/runtime.rs | 123 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 15 ++- 4 files changed, 137 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9bc6f0e7df5a15..a7dc2aa327c6d5 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -403,7 +403,7 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + if !(early && self.backend.try_early(&packet)) { self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 7b8d401cf8dd71..2e40b7e7dde07c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -280,8 +280,7 @@ impl Queue { checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() + let early = dependencies.is_empty() && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( id, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ab55dda6013b2e..f3bc478442f82f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -528,7 +528,10 @@ pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, + /// Render packets queued to the scheduler and not yet published. backlog: core::sync::atomic::AtomicU32, + /// Compute packets queued to the scheduler and not yet published. + compute_backlog: core::sync::atomic::AtomicU32, /// One compute binder per owner: submission and publication can both bind, /// and two concurrent binders could leave the owner with two queues. binding: core::sync::atomic::AtomicBool, @@ -624,6 +627,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), binding: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, @@ -1395,6 +1399,125 @@ impl Backend { } } + /// Publishes a dependency-free compute command from the submitting ioctl, + /// as renders are, instead of waking the scheduler's worker for it. Only a + /// queue already bound to this owner that fits the command and has room + /// qualifies, and only while no earlier compute command of this owner + /// still waits in the scheduler, so publication order stays submission + /// order. The ioctl never binds, grows, replays or sleeps: any of those + /// returns false before anything became visible and leaves the command to + /// the scheduler. + pub(in crate::g17) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return false; + }; + let Ok(required) = + super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) + else { + return false; + }; + { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + if firmware.recovery.pending() + || self.binding.load(Ordering::Acquire) + || !firmware + .queues + .compute(self.owner) + .is_some_and(|queue| queue.pool_fits(required) && queue.room()) + { + return false; + } + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.publish_compute_early(packet, required, &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + /// The ioctl's single publication attempt, under one device mutex hold on + /// the owner's bound queue. Contention returns `FALLBACK` unpublished. + fn publish_compute_early( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Result { + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = self.enter_preparation(Wait::Never)?; + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => return Err(FALLBACK), + result => { + result?; + } + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + || self.binding.load(Ordering::Acquire) + // Replays of recovered commands publish first, from the scheduler. + || firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.replay_commands() != 0) + || !firmware + .queues + .compute(self.owner) + .is_some_and(|queue| queue.pool_fits(required) && queue.room()) + { + return Err(FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) + { + Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { + Err(FALLBACK) + } + result => result, + } + } + /// Called after taking this packet's publication claim. All waits and work /// state allocation precede the device mutex; only the checked epoch commits. fn compute_packet( diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 78d26ae6262cad..305c12b1b4a23e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,10 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + match packet.engine() { + Engine::Render => self.backlog.fetch_sub(1, Ordering::AcqRel), + Engine::Compute => self.compute_backlog.fetch_sub(1, Ordering::AcqRel), + }; } } @@ -192,11 +195,17 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + match packet.engine() { + Engine::Render => self.backlog.fetch_add(1, Ordering::AcqRel), + Engine::Compute => self.compute_backlog.fetch_add(1, Ordering::AcqRel), + }; } } From 639fef4ccc095ee09a3112e3fc95a0d4d48e4db5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 15:34:02 -0400 Subject: [PATCH 22/22] drm/asahi: publish G17 compute from the submit ioctl up to four commands deep A dependency-free command is published from its ioctl only while fewer than two commands of its engine are still in flight; deeper ones go through the scheduler's worker, a hop of ~100 us per command on Neo. With four dispatches in flight a dependency-free compute stream spends that hop on every command: on Neo it takes 71.7 us per dispatch with the limit of two and 56.9 us when those publish inline. Allow inline publication up to four commands deep for compute. Renders keep the limit of two, so render publication is unchanged. Ordering is unchanged: inline publication still requires that no earlier command of the engine waits in the scheduler, and it still makes one attempt that never binds or sleeps. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 2e40b7e7dde07c..02913513fdbbd8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -27,7 +27,12 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; +/// Commands of one engine (render, compute) still in flight below which a +/// dependency-free command is published from its ioctl rather than from the +/// scheduler's worker, whose hop costs ~100 us per command on Neo. Compute +/// streams of three or four dispatches publish inline; renders keep the +/// shallow limit. +const EARLY_DEPTH: [usize; 2] = [2, 4]; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -281,7 +286,7 @@ impl Queue { } } let early = dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + && self.frontiers[index].len() < EARLY_DEPTH[index]; let packet = Packet::new( id, self.context.clone(),