diff --git a/Documentation/ABI/testing/sysfs-driver-asahi b/Documentation/ABI/testing/sysfs-driver-asahi new file mode 100644 index 00000000000000..0c6442c88667a7 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-asahi @@ -0,0 +1,54 @@ +What: /sys/devices/platform/.../*.gpu/agx_stats +Date: October 2026 +Contact: Aurora Silicon developers +Description: + Read-only, per-device firmware GPU utilization on T8140 G17P. + With asahi.stats_export=0, the file reports unsupported. + The default value 1 enables the version 2 export described here. + Earlier GPU generations use the separate legacy agx_stats format. + Version 2 is a whitespace-delimited key/value format. Readers + must check the version and load_valid before using percentages. + + version: 2 + source: firmware + load_valid: 1 if the current utilization is available, else 0 + busy_percent: firmware aggregate utilization, percent (0..100) + util1..util3: firmware engine percentages (0..100); engine labels + are not assigned without independent validation + util4: same aggregate as busy_percent; do not sum engine values + gpu_power_known: whether a timestamped power state was received + gpu_powered: 0 when firmware reports the GPU powered off, else 1 + samples: accepted primary utilization reports since probe + sample_received_ns: host CLOCK_MONOTONIC receive timestamp + sample_age_ns: time since the retained utilization report + sample_timestamp_ticks: 64-bit firmware utilization timestamp + power_timestamp_ticks: newest secondary power-state timestamp + clock_hz: 24000000 (timestamp and residency ticks) + powered_ticks: completed powered intervals, accumulated from + firmware power-off records since probe + unpowered_ticks: completed unpowered intervals, accumulated from + firmware power-on records since probe + invalid_samples: malformed utilization/power records rejected + + A timestamped power-off witness overrides retained utilization + with zero. After wake, an old utilization report is unavailable + until a report for the new interval arrives. Device loss or + shutdown invalidates load until a new device is probed. Receive timestamps + do not order events across firmware rings: firmware timestamps + determine which state is newer. Suspend can separate the host + and firmware clocks; do not subtract values from those domains. + + Power residency is not GPU busy time. Residency counters omit + the currently open power interval and saturate at U64_MAX. + Neither these counters nor the percentages are per-client DRM + engine execution time. No per-client execution counter is + synthesized from queue admission, wall time or job spans. + + Reading debugfs asahi-g17/statistics requests two seconds of full + raw-payload capture, including undecoded record kinds. Repeat + reads to keep diagnostic capture active. It is disabled by + default and is not needed for normal utilization reporting. + Trace records and uncaptured statistics only read required words. + Captured counts exclude records outside the capture window. That diagnostic format is not a + stable userspace ABI. Optional telemetry registration failure + does not prevent GPU initialization and is logged by the driver. diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 11a964e6f17d81..1db3281eb64c4a 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -2,6 +2,9 @@ What: /sys/class/drm/card*/device/agx_stats Date: October 2026 Contact: Asahi Linux GPU driver maintainers Description: + Legacy GPU format. For T8140 G17P, see sysfs-driver-asahi + for the version 2 format and power-state validity rules. + Read-only dump of the AGX firmware statistics decoded by the drm/asahi driver. The driver already polls the firmware statistics ring (see `drivers/gpu/drm/asahi/channel.rs::StatsChannel::poll`, diff --git a/drivers/base/base.h b/drivers/base/base.h index a5b7abc10ff02c..d16026fdb2ef00 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -222,6 +222,8 @@ typedef void (*dr_node_free_t)(struct devres_node *node); struct devres_node { struct list_head entry; + /* Device-list membership, cleared under devres_lock before detachment. */ + struct device *owner; dr_node_release_t release; dr_node_free_t free_node; const char *name; diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 9d9842fc5a19f9..26e5518b6482b3 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -41,6 +41,7 @@ void devres_node_init(struct devres_node *node, dr_node_free_t free_node) { INIT_LIST_HEAD(&node->entry); + WRITE_ONCE(node->owner, NULL); node->release = release; node->free_node = free_node; } @@ -156,6 +157,7 @@ static void add_dr(struct device *dev, struct devres_node *node) devres_log(dev, node, "ADD"); BUG_ON(!list_empty(&node->entry)); list_add_tail(&node->entry, &dev->devres_head); + WRITE_ONCE(node->owner, dev); } static void replace_dr(struct device *dev, @@ -164,6 +166,8 @@ static void replace_dr(struct device *dev, devres_log(dev, old, "REPLACE"); BUG_ON(!list_empty(&new->entry)); list_replace(&old->entry, &new->entry); + WRITE_ONCE(old->owner, NULL); + WRITE_ONCE(new->owner, dev); } /** @@ -367,18 +371,14 @@ EXPORT_SYMBOL_GPL(devres_get); bool devres_node_remove(struct device *dev, struct devres_node *node) { - struct devres_node *__node; - guard(spinlock_irqsave)(&dev->devres_lock); - list_for_each_entry_reverse(__node, &dev->devres_head, entry) { - if (__node == node) { - list_del_init(&node->entry); - devres_log(dev, node, "REM"); - return true; - } - } - - return false; + /* A release-all/group pass may own a detached node on its private list. */ + if (READ_ONCE(node->owner) != dev) + return false; + list_del_init(&node->entry); + WRITE_ONCE(node->owner, NULL); + devres_log(dev, node, "REM"); + return true; } /** @@ -405,6 +405,7 @@ void *devres_remove(struct device *dev, dr_release_t release, dr = find_dr(dev, release, match, match_data); if (dr) { list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr->data; } @@ -500,6 +501,7 @@ static int remove_nodes(struct device *dev, if (&node->entry == first) first = first->next; list_move_tail(&node->entry, todo); + WRITE_ONCE(node->owner, NULL); cnt++; } } @@ -530,7 +532,9 @@ static int remove_nodes(struct device *dev, * nodes are always before both. */ list_move_tail(&grp->node[0].entry, todo); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); } } @@ -685,7 +689,9 @@ void devres_remove_group(struct device *dev, void *id) grp = find_group(dev, id); if (grp) { list_del_init(&grp->node[0].entry); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); devres_log(dev, &grp->node[0], "REM"); } else WARN_ON(1); @@ -849,6 +855,7 @@ static struct devres_action *remove_action(struct device *dev, return ERR_PTR(-ENOENT); list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr; diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index a3aff32eea1608..7d8eb6c4376f0d 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o +asahi-y := asahi_core.o sysfs.o asahi_telemetry.o diff --git a/drivers/gpu/drm/asahi/agx_memory_stats.rs b/drivers/gpu/drm/asahi/agx_memory_stats.rs new file mode 100644 index 00000000000000..568157432298cd --- /dev/null +++ b/drivers/gpu/drm/asahi/agx_memory_stats.rs @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded host accounting for driver-owned GEM extents and M3 coherent backing. +//! +//! These are allocation extents, not resident/pinned pages or unique imported +//! physical memory. GEM construction/unwind is included. Counters are per +//! module instance and each read is observational, not a transactional snapshot. +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +struct Counters { + objects: AtomicU64, + bytes: AtomicU64, + peak_objects: AtomicU64, + peak_bytes: AtomicU64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backing_owner_drop_releases_extent_once_and_keeps_peak() { + static COUNTERS: Counters = Counters::new(); + let first = Allocation::new(&COUNTERS, 0x4000).unwrap(); + let second = Allocation::new(&COUNTERS, 0x8000).unwrap(); + assert_eq!(COUNTERS.snapshot(), [2, 0xc000, 2, 0xc000]); + let shared_owner = std::sync::Arc::new(first); + let retained = shared_owner.clone(); + drop(shared_owner); + drop(second); + assert_eq!(COUNTERS.snapshot(), [1, 0x4000, 2, 0xc000]); + drop(retained); + assert_eq!(COUNTERS.snapshot(), [0, 0, 2, 0xc000]); + } + + #[test] + fn refused_extent_does_not_change_live_object_count() { + static COUNTERS: Counters = Counters::new(); + assert!(matches!(Allocation::new(&COUNTERS, 0), Err(EINVAL))); + assert_eq!(COUNTERS.snapshot(), [0; 4]); + COUNTERS.bytes.store(u64::MAX, Ordering::Relaxed); + assert!(matches!(Allocation::new(&COUNTERS, 1), Err(EOVERFLOW))); + assert_eq!(COUNTERS.objects.load(Ordering::Relaxed), 0); + assert_eq!(COUNTERS.bytes.load(Ordering::Relaxed), u64::MAX); + } + + #[test] + fn concurrent_aliases_classify_one_backing_and_release_it_once() { + let allocation = std::sync::Arc::new(Allocation::gem(0x4000, true).unwrap()); + std::thread::scope(|scope| { + for category in [ + KernelCategory::Firmware, + KernelCategory::Descriptor, + KernelCategory::QueueConfig, + KernelCategory::FreeList, + ] { + let alias = allocation.clone(); + scope.spawn(move || alias.classify_kernel(category)); + } + }); + let class_bytes: u64 = KERNEL_CATEGORIES.iter().map(|c| c.snapshot()[1]).sum(); + let class_objects: u64 = KERNEL_CATEGORIES.iter().map(|c| c.snapshot()[0]).sum(); + assert_eq!((class_objects, class_bytes), (1, 0x4000)); + assert_eq!(KERNEL_CATEGORIES[0].snapshot()[0], 0); + let retained = allocation.clone(); + drop(allocation); + assert_eq!(KERNEL_GEM.snapshot()[1], 0x4000); + drop(retained); + assert_eq!(KERNEL_GEM.snapshot()[1], 0); + assert!(KERNEL_CATEGORIES + .iter() + .all(|c| c.snapshot()[0] == 0 && c.snapshot()[1] == 0)); + } +} +impl Counters { + const fn new() -> Self { + Self { + objects: AtomicU64::new(0), + bytes: AtomicU64::new(0), + peak_objects: AtomicU64::new(0), + peak_bytes: AtomicU64::new(0), + } + } + fn snapshot(&self) -> [u64; 4] { + let objects = self.objects.load(Ordering::Relaxed); + let bytes = self.bytes.load(Ordering::Relaxed); + [ + objects, + bytes, + self.peak_objects.load(Ordering::Relaxed).max(objects), + self.peak_bytes.load(Ordering::Relaxed).max(bytes), + ] + } +} +static USER_GEM: Counters = Counters::new(); +static KERNEL_GEM: Counters = Counters::new(); +// Core-created objects may be local dumb buffers or wrappers around imported backing. +static CORE_GEM: Counters = Counters::new(); +static COHERENT: Counters = Counters::new(); + +/// A breakdown of kernel GEM extents, not an additional memory total. +#[derive(Copy, Clone)] +#[repr(u32)] +pub(crate) enum KernelCategory { + Other = 0, + Firmware = 1, + Descriptor = 2, + WorkState = 3, + QueueState = 4, + QueueConfig = 5, + FreeList = 6, + RenderGeneration = 7, + RenderTarget = 8, + TileBuffer = 9, + QueueSupport = 10, + Scratch = 11, + Scheduler = 12, +} +const CATEGORY_NAMES: [&str; 13] = [ + "other", + "firmware", + "descriptor", + "work_state", + "queue_state", + "queue_config", + "free_list", + "render_generation", + "render_target", + "tile_buffer", + "queue_support", + "scratch", + "scheduler", +]; +static KERNEL_CATEGORIES: [Counters; 13] = [const { Counters::new() }; 13]; +const NO_CATEGORY: u32 = u32::MAX; + +/// Drop with the actual backing owner, never with an individual GEM handle or +/// its mapping. Retained fault owners therefore remain included after exit. +pub(crate) struct Allocation { + counters: &'static Counters, + bytes: u64, + kernel_category: AtomicU32, +} +impl Allocation { + fn new(counters: &'static Counters, size: usize) -> Result { + let bytes = size as u64; + if bytes == 0 { + return Err(EINVAL); + } + let previous = counters + .bytes + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(bytes) + }) + .map_err(|_| EOVERFLOW)?; + let objects = counters.objects.fetch_add(1, Ordering::Relaxed) + 1; + counters + .peak_bytes + .fetch_max(previous + bytes, Ordering::Relaxed); + counters.peak_objects.fetch_max(objects, Ordering::Relaxed); + Ok(Self { + counters, + bytes, + kernel_category: AtomicU32::new(NO_CATEGORY), + }) + } + pub(crate) fn gem(size: usize, kernel: bool) -> Result { + let allocation = Self::new(if kernel { &KERNEL_GEM } else { &USER_GEM }, size)?; + if kernel { + Self::add_category(0, allocation.bytes); + allocation.kernel_category.store(0, Ordering::Relaxed); + } + Ok(allocation) + } + pub(crate) fn coherent(size: usize) -> Result { + Self::new(&COHERENT, size) + } + pub(crate) fn core(size: usize) -> Result { + Self::new(&CORE_GEM, size) + } + + fn add_category(index: usize, bytes: u64) { + let counters = &KERNEL_CATEGORIES[index]; + let previous = counters.bytes.fetch_add(bytes, Ordering::Relaxed); + let objects = counters.objects.fetch_add(1, Ordering::Relaxed) + 1; + counters + .peak_bytes + .fetch_max(previous + bytes, Ordering::Relaxed); + counters.peak_objects.fetch_max(objects, Ordering::Relaxed); + } + + /// Refine an unclassified backing once. Aliases do not create another allocation. + pub(crate) fn classify_kernel(&self, category: KernelCategory) { + let index = category as u32; + if index != 0 + && self + .kernel_category + .compare_exchange(0, index, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + Self::add_category(index as usize, self.bytes); + KERNEL_CATEGORIES[0] + .bytes + .fetch_sub(self.bytes, Ordering::Relaxed); + KERNEL_CATEGORIES[0].objects.fetch_sub(1, Ordering::Relaxed); + } + } +} +impl Drop for Allocation { + fn drop(&mut self) { + let category = self.kernel_category.load(Ordering::Relaxed); + if category != NO_CATEGORY { + KERNEL_CATEGORIES[category as usize] + .bytes + .fetch_sub(self.bytes, Ordering::Relaxed); + KERNEL_CATEGORIES[category as usize] + .objects + .fetch_sub(1, Ordering::Relaxed); + } + self.counters.bytes.fetch_sub(self.bytes, Ordering::Relaxed); + self.counters.objects.fetch_sub(1, Ordering::Relaxed); + } +} + +pub(crate) struct View; +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 unit=allocation_extent_bytes")?; + for (name, counters) in [ + ("user_gem", &USER_GEM), + ("kernel_gem", &KERNEL_GEM), + ("core_gem_may_alias_imports", &CORE_GEM), + ("m3_coherent", &COHERENT), + ] { + let [objects, bytes, peak_objects, peak_bytes] = counters.snapshot(); + writeln!( + f, + "{} objects={} bytes={} peak_objects={} peak_bytes={}", + name, objects, bytes, peak_objects, peak_bytes + )?; + } + writeln!(f, "kernel_categories=subset_of_kernel_gem")?; + for (name, counters) in CATEGORY_NAMES.iter().zip(&KERNEL_CATEGORIES) { + let [objects, bytes, peak_objects, peak_bytes] = counters.snapshot(); + writeln!( + f, + "kernel_{} objects={} bytes={} peak_objects={} peak_bytes={}", + name, objects, bytes, peak_objects, peak_bytes + )?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 4dca72fb85dc07..8d32b97da1d24b 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -3,6 +3,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. +mod agx_memory_stats; mod alloc; mod buffer; mod channel; diff --git a/drivers/gpu/drm/asahi/asahi_telemetry.c b/drivers/gpu/drm/asahi/asahi_telemetry.c new file mode 100644 index 00000000000000..bf46d02ddb678c --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi_telemetry.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* A device attribute with an owned Rust snapshot callback. */ +#include +#include +#include +#include +#include +#include + +struct asahi_load_export { + u32 valid, power_known, powered, utilization[4], clock_hz; + u64 samples, received_ns, timestamp_ticks, power_timestamp_ticks; + u64 powered_ticks, unpowered_ticks, invalid, age_ns; +}; + +typedef int (*asahi_snapshot_fn)(const void *, void *); +struct asahi_telemetry_attribute { + struct device_attribute attr; + const void *data; + asahi_snapshot_fn snapshot; +}; + +static ssize_t agx_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_telemetry_attribute *stats = + container_of(attr, struct asahi_telemetry_attribute, attr); + struct asahi_load_export value; + int ret = stats->snapshot(stats->data, &value); + + if (ret) + return ret; + return sysfs_emit(buf, + "version 2\nsource firmware\nload_valid %u\n" + "gpu_power_known %u\ngpu_powered %u\n" + "util1 %u\nutil2 %u\nutil3 %u\nutil4 %u\nbusy_percent %u\n" + "samples %llu\nsample_received_ns %llu\nsample_age_ns %llu\n" + "sample_timestamp_ticks %llu\npower_timestamp_ticks %llu\n" + "clock_hz %u\npowered_ticks %llu\nunpowered_ticks %llu\n" + "invalid_samples %llu\n", + value.valid, value.power_known, value.powered, + value.utilization[0], value.utilization[1], value.utilization[2], + value.utilization[3], value.utilization[3], + (unsigned long long)value.samples, + (unsigned long long)value.received_ns, + (unsigned long long)value.age_ns, + (unsigned long long)value.timestamp_ticks, + (unsigned long long)value.power_timestamp_ticks, + value.clock_hz, + (unsigned long long)value.powered_ticks, + (unsigned long long)value.unpowered_ticks, + (unsigned long long)value.invalid); +} + +int asahi_telemetry_register(struct device *dev, const void *data, + asahi_snapshot_fn snapshot, void **handle); +void asahi_telemetry_unregister(struct device *dev, void *handle); + +int asahi_telemetry_register(struct device *dev, const void *data, + asahi_snapshot_fn snapshot, void **handle) +{ + struct asahi_telemetry_attribute *stats; + int ret; + + if (!dev || !data || !snapshot || !handle) + return -EINVAL; + BUILD_BUG_ON(sizeof(struct asahi_load_export) != 96); + BUILD_BUG_ON(offsetof(struct asahi_load_export, samples) != 32); + stats = kzalloc(sizeof(*stats), GFP_KERNEL); + if (!stats) + return -ENOMEM; + sysfs_attr_init(&stats->attr.attr); + stats->attr.attr.name = "agx_stats"; + stats->attr.attr.mode = 0444; + stats->attr.show = agx_stats_show; + stats->data = data; + stats->snapshot = snapshot; + ret = device_create_file(dev, &stats->attr); + if (ret) { + kfree(stats); + return ret; + } + *handle = stats; + return 0; +} +EXPORT_SYMBOL_GPL(asahi_telemetry_register); + +void asahi_telemetry_unregister(struct device *dev, void *handle) +{ + struct asahi_telemetry_attribute *stats = handle; + + if (!stats) + return; + device_remove_file(dev, &stats->attr); + kfree(stats); +} +EXPORT_SYMBOL_GPL(asahi_telemetry_unregister); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Asahi firmware telemetry sysfs registration"); diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 510a70ba4f0bd9..3dddb7fb5e62ae 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -100,6 +100,9 @@ impl drm::driver::Driver for AsahiDrmDriver { const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; + const SHOW_FDINFO: Option< + unsafe extern "C" fn(*mut bindings::drm_printer, *mut bindings::drm_file), + > = Some(bindings::drm_show_memory_stats); kernel::declare_drm_ioctls! { (ASAHI_GET_PARAMS, drm_asahi_get_params, @@ -329,14 +332,19 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + // Enabled G17 telemetry owns agx_stats through Gpu's registration. + // Only the disabled-export placeholder belongs to the platform wrapper. let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { - Ok(handle) => handle, - Err(error) => { - gpu.shutdown(); - return Err(error); + let raw_stats = if *module_parameters::stats_export.value() == 0 { + match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + Ok(handle) => handle, + Err(error) => { + gpu.shutdown(); + return Err(error); + } } + } else { + core::ptr::null_mut() }; Ok(AsahiPlatformDriverData { _device: reg.device().into(), diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea02..c82bc579d451e4 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -70,6 +70,10 @@ impl BufferIds { self.next_token = token + 1; token } + /// Identities held by an owner or a reference, for diagnostics. + pub(crate) fn in_use(&self) -> usize { + self.entries.iter().filter(|entry| !entry.free()).count() + } pub(crate) fn allocate_render(&mut self) -> Result<(u8, u64)> { let id = if self.returned_count != 0 { self.returned_count -= 1; @@ -78,7 +82,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| resource_exhausted!("parameter-buffer IDs"))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -111,7 +115,10 @@ impl BufferIds { if !entry.owner { return Err(EINVAL); } - entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; + entry.references = entry + .references + .checked_add(1) + .ok_or_else(|| resource_exhausted!("parameter-buffer references"))?; Ok((id, entry.token)) } /// Preflight shared completion bookkeeping before any of its counters change. @@ -168,7 +175,7 @@ impl Scenes { (0..fw::SCENES) .map(|offset| (self.next + offset) % fw::SCENES) .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) - .ok_or(ENOSPC) + .ok_or_else(|| resource_exhausted!("render scene slots")) } pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { if self.select(ordinal)? != scene { @@ -240,7 +247,7 @@ impl Growth { let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { - return Err(ENOSPC); + return Err(resource_exhausted!("tiler heap block slots")); } let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; Ok(Self { @@ -320,7 +327,7 @@ pub(crate) fn compact_alias( Err(error) => return Err(error), } } - Err(ENOSPC) + Err(resource_exhausted!("client compact GPU VA")) } /// Allocated aliases remain private until the owner installs them in its VM @@ -379,6 +386,7 @@ impl Tvb { return Err(ERANGE); } let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::TileBuffer); let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; for index in 0..blocks { @@ -430,6 +438,7 @@ impl Tvb { self.extensions.reserve(1, GFP_KERNEL)?; self.addresses.reserve(added, GFP_KERNEL)?; let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::TileBuffer); let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; for relative in 0..added { @@ -463,6 +472,13 @@ impl MetricsIds { ], } } + /// Leased counter slots and their total, for diagnostics. + pub(crate) fn census(&self) -> (usize, usize) { + let used = self.used[0].load(Ordering::Relaxed).count_ones() + + (self.used[1].load(Ordering::Relaxed) & ((1u64 << (fw::METRICS_SLOTS - 64)) - 1)) + .count_ones(); + (used as usize, fw::METRICS_SLOTS) + } } pub(crate) struct MetricsLease { @@ -503,7 +519,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| resource_exhausted!("performance-counter slots"))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c935..99142c3dd6bd83 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -114,6 +114,7 @@ struct Object { impl Object { fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize) -> Result { let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; + backing.classify_memory(crate::agx_memory_stats::KernelCategory::RenderTarget); let prot = mmu::PROT_GPU_FW_SHARED_RW; let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; Ok(Self { diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a1661..1468bc7b3f46dd 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -56,20 +56,26 @@ impl Rings { } pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { - self.drain_report(dev, init, role, ReportRing::Trace); + self.drain_report(dev, init, role, ReportRing::Trace, None); } - pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role) { - self.drain_report(dev, init, role, ReportRing::Statistics); + pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role, + statistics: &super::statistics::Statistics) { + self.drain_report(dev, init, role, ReportRing::Statistics, Some(statistics)); } - fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { + fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing, + statistics: Option<&super::statistics::Statistics>) { let quarantined = &mut self.quarantined[role as usize][ring as usize]; if *quarantined { return; } - if let Err(error) = Self::consume_report(init, role, ring) { + if let Err(error) = Self::consume_report(init, role, ring, statistics) { *quarantined = true; + if let Some(statistics) = statistics { + // Without both report streams, retained load is no longer current. + statistics.load.invalidate(); + } dev_err!( dev.as_ref(), "G17 {:?} {:?} ring stopped: {:?}\n", @@ -80,7 +86,8 @@ impl Rings { } } - fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { + fn consume_report(init: &InitData, role: Role, ring: ReportRing, + statistics: Option<&super::statistics::Statistics>) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; @@ -94,14 +101,42 @@ impl Rings { } while consumer != producer { let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - let tag = match ring { - // Copy the entire trace entry before returning its slot. - ReportRing::Trace => init.status_read_words::(role, offset)?[0], - ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], - }; + let tag = init.status_word(role, offset)?.load(Ordering::Relaxed); if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { return Err(EIO); } + if let Some(statistics) = statistics { + const { assert!(super::statistics::WORDS == EVENT_WORDS) }; + let pending = statistics.capture_pending(); + let now_ns = if pending || (role == Role::Primary && tag == 0x13) { + super::now_ns() + } else { 0 }; + if pending && statistics.capture_enabled(now_ns) { + // Diagnostic capture retains every word before returning the ring slot. + let words = init.status_read_words::(role, offset)?; + if words[0] != tag { return Err(EIO); } + statistics.record(role as usize, now_ns, &words); + } else { + // The production decoder reads only fields it consumes. Trace records + // and unsupported statistics need no payload copy or receive timestamp. + let mut words = [0; EVENT_WORDS]; + words[0] = tag; + match (role, tag) { + (Role::Primary, 0x13) => { + words[1..7].copy_from_slice(&init.status_read_words::<6>(role, offset + 4)?); + } + (Role::Secondary, 0x19) => { + words[1..3].copy_from_slice(&init.status_read_words::<2>(role, offset + 4)?); + words[6] = init.status_word(role, offset + 24)?.load(Ordering::Relaxed); + } + (Role::Secondary, 0x11 | 0x12) => { + words[1..3].copy_from_slice(&init.status_read_words::<2>(role, offset + 4)?); + } + _ => {} + } + statistics.load.record(role as usize, now_ns, &words); + } + } consumer = (consumer + 1) & (capacity - 1); fence(Ordering::SeqCst); consumer_word.store(consumer, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a3888f98231dba..34c7cdca3c4294 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,81 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +249,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -216,7 +292,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -278,14 +354,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -335,7 +411,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -401,6 +477,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44beef..e1c890ca16f91c 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(resource_exhausted!("QoS IDs")); } *used |= 1u128 << id; Ok(QosLease { @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -142,6 +144,7 @@ impl Context { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + scheduler.classify_memory(crate::agx_memory_stats::KernelCategory::Scheduler); scheduler.initialize::(0, |value| { value.init(); Ok(()) @@ -194,12 +197,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -387,7 +394,12 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context + .work + .lock() + .render + .get(pair) + .ok_or_else(|| resource_exhausted!("context render channels")) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc0..1d87cc3fd09455 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -29,6 +29,11 @@ impl RenderIds { used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], } } + /// Allocated render pool IDs, for diagnostics. + pub(crate) fn in_use(&self) -> usize { + (self.used[0].load(Ordering::Relaxed).count_ones() + + (self.used[1].load(Ordering::Relaxed) & !(1u64 << 63)).count_ones()) as usize + } } struct RenderId { slot: u16, @@ -59,7 +64,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(resource_exhausted!("render free-list IDs")) } } impl Drop for RenderId { @@ -69,19 +74,89 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; + Self::from_backing(backing, blocks, layout) + } + /// Size of the backing, for diagnostics. + pub(crate) fn bytes(&self) -> usize { + TABLE_SIZE + self.blocks * FREE_LIST_BLOCK_SIZE + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { + backing.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -144,14 +219,24 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + }) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA (render free list)") + } else { + error + } + })?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, @@ -245,6 +330,10 @@ impl RenderPool { pub(crate) fn vacant(&self) -> bool { self.page_list.load(Ordering::Acquire) == 0 } + /// Bytes of installed backing in the shared lower window, for diagnostics. + pub(crate) fn backing_bytes(&self) -> usize { + self.backing.lock().as_ref().map_or(0, |backing| backing.pages.bytes()) + } /// Pressure reclaim evicted the client VM, which is idle. pub(crate) fn client_evicted(&self) -> bool { self.client_residency.as_ref().is_some_and(|gate| gate.evicted()) @@ -504,6 +593,10 @@ impl ComputePool { pub(crate) fn populated(&self) -> bool { self.pages.is_some() } + /// Bytes of installed backing in the shared lower window, for diagnostics. + pub(crate) fn backing_bytes(&self) -> usize { + self.pages.as_ref().map_or(0, Pages::bytes) + } /// The release record for this pool's buffer slot under the free-list /// generation the queue last published. pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs index e1e6a10949fb44..a53a7d993f822f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -9,11 +9,18 @@ use kernel::prelude::*; pub(crate) const SUPPORT_SIZE: usize = 0x40000; pub(crate) const EVENT_SLOTS: usize = 0x4000; pub(crate) const EVENT_COUNT: usize = 35; -pub(crate) const STATE_SIZE: usize = 0x18000; -pub(crate) const DEFLAKE: usize = 0x4000; -pub(crate) const TA_STATUS: usize = 0x8000; -pub(crate) const AUXILIARY: usize = 0x10000; +// Host-private packing: consumers receive separate GPU pointers to these ranges. +// Keep every accessed range and all generation slots, without unused backing pages. +pub(crate) const STATE_SIZE: usize = 0x10000; +pub(crate) const DEFLAKE: usize = 0; +pub(crate) const TA_STATUS: usize = 0x4000; +pub(crate) const AUXILIARY: usize = 0x8000; pub(crate) const AUXILIARY_SIZE: usize = 0x8000; +const _: () = { + assert!(DEFLAKE + 0x4000 <= TA_STATUS); + assert!(TA_STATUS + 0x4000 <= AUXILIARY); + assert!(AUXILIARY + AUXILIARY_SIZE == STATE_SIZE); +}; #[repr(C)] pub(crate) struct Event { diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09eb..f53ef73e10bd73 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -64,12 +64,14 @@ impl ComputeShared { KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; let runs = KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; + pages.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); + runs.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_MAPPED, GFP_KERNEL)?; for _ in 0..COMPUTE_BLOCKS_MAPPED { - blocks.push( - KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, - GFP_KERNEL, - )?; + let block = KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, + CpuMap::WriteCombined)?; + block.classify_memory(crate::agx_memory_stats::KernelCategory::FreeList); + blocks.push(block, GFP_KERNEL)?; } let id = u64::from(super::ids::OBJECTS.object()?); Ok(Self { @@ -95,6 +97,7 @@ impl ComputeScheduler { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + object.classify_memory(crate::agx_memory_stats::KernelCategory::Scheduler); object.initialize::(0, |state| { state.init(); Ok(()) @@ -675,6 +678,7 @@ impl InitData { } fn add(&mut self, object: KernelObject) -> Result { + object.classify_memory(crate::agx_memory_stats::KernelCategory::Firmware); let index = self.objects.len(); self.objects.push(object, GFP_KERNEL)?; Ok(index) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b5a5b3759de6dd..a7dc2aa327c6d5 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -179,6 +179,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -396,7 +403,7 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + if !(early && self.backend.try_early(&packet)) { self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86de..6fed8ee6dc4ce7 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -17,8 +24,18 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. -const COMPUTE_RENDER_RESERVE: usize = 16; +/// Minimum render identities protected from compute-only allocation. +const MIN_RENDER_IDENTITIES: usize = 16; + +/// Render identities compute leaves free while `queues` logical queues are +/// open: one pair for each of them, existing pairs included. A published QID +/// never changes engine (no firmware operation uninstalls one) and a live owner +/// keeps its last render pair, so render capacity handed to compute could not +/// be recovered while those owners live. Compute takes any QID beyond this and +/// changes owners only when none remains. +fn render_floor(queues: usize) -> usize { + MIN_RENDER_IDENTITIES.max(queues.saturating_mul(2)).min(QID_COUNT - 2) +} /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -47,6 +64,7 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], + initial_render: [Option; 2], compute_publication: u64, leased: usize, } @@ -55,18 +73,57 @@ impl QueueIds { pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), + initial_render: [None; 2], compute_publication: 0, leased: 0, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for - /// bootstrapping; it does not reserve a partition for an engine. + /// bootstrapping; it does not reserve a partition for an engine. Compute + /// reservations use `reserve_compute`, which knows the open queues. pub(crate) fn reserve( &mut self, owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, 0) + } + + /// A compute reservation while `queues` logical queues are open. + pub(crate) fn reserve_compute( + &mut self, + owner: u64, + preferred: Option, + queues: usize, + ) -> Result { + self.reserve_for(owner, DataMaster::Compute, preferred, queues) + } + + fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -83,16 +140,38 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); - } - let index = preferred - .map(usize::from) - .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + // Existing render pairs already count towards the reserve. The initial + // reservations do not: no render pair can use them, so compute may + // replace an unpublished one instead of taking reserved capacity. + // Published identities always retain their original lifetime rules. + let initial = if engine == DataMaster::Compute { + let need = render_floor(queues).saturating_sub(self.pair_identities()); + if QID_COUNT - self.leased > need { + None + } else { + Some( + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + .ok_or(ENOSPC)?, + ) + } + } else { + None + }; + let index = match initial { + Some((_, id)) => usize::from(id.qid), + None => preferred + .map(usize::from) + .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?, + }; let id = Id { qid: index as u8, owner, @@ -102,10 +181,55 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } + /// Leased IDs, render pair identities and compute identities, for diagnostics. + pub(crate) fn census(&self) -> (usize, usize, usize) { + let compute = self + .entries + .iter() + .flatten() + .filter(|entry| entry.id.engine == DataMaster::Compute) + .count(); + (self.leased, self.pair_identities(), compute) + } + + /// Tiling and fragment identities of render pairs, excluding the initial + /// reservations, which no pair uses. + fn pair_identities(&self) -> usize { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count(); + render.saturating_sub(self.initial_render.iter().flatten().count()) + } + + /// Render identities beyond the reserve for `queues` open queues. Only + /// these become compute capacity when an unused render reservation is + /// cancelled. + pub(crate) fn render_surplus(&self, queues: usize) -> usize { + self.pair_identities().saturating_sub(render_floor(queues)) + } + + /// A compute reservation would succeed now: a fresh QID beyond the render + /// reserve, or a replaceable initial reservation. + pub(crate) fn compute_available(&self, queues: usize) -> bool { + QID_COUNT - self.leased > render_floor(queues).saturating_sub(self.pair_identities()) + || self + .initial_render + .iter() + .flatten() + .any(|id| self.can_cancel_unpublished(*id)) + } + /// Ordered under the device mutex; no atomic operation on the submission path. pub(crate) fn next_compute_publication(&mut self) -> Result { self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; @@ -127,6 +251,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -147,10 +278,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -186,6 +401,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -220,14 +438,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -236,12 +485,20 @@ impl Queue { )?; let mut range = t8140::dynamic::LOWER; let (low, client) = loop { - let low = high.alias_in( - alloc.uat.kernel_lower_vm(), - range.clone(), - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - )?; + let low = high + .alias_in( + alloc.uat.kernel_lower_vm(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA (render kick ring)") + } else { + error + } + })?; let end = low .iova() .checked_add(KICK_RING_SIZE as u64) @@ -252,17 +509,32 @@ impl Queue { drop(low); range.start = end; } + Err(error) if error == ENOSPC => { + return Err(resource_exhausted!("client GPU VA for a render kick ring")) + } Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -272,19 +544,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -297,7 +569,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } diff --git a/drivers/gpu/drm/asahi/g17/load.rs b/drivers/gpu/drm/asahi/g17/load.rs new file mode 100644 index 00000000000000..9ed2f3a1312369 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/load.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide firmware utilization. This is not per-client execution time. +//! +//! The primary utilization record contains four independently reported percentages; +//! the aggregate is not the sum of the engine values. Power state comes from the +//! secondary's timestamped power record, never its coprocessor busy flag. Only +//! explicitly written fields are read: unused record tails retain old ring contents. + +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; + +pub(crate) const CLOCK_HZ: u64 = 24_000_000; +const UTILIZATION: u32 = 0x13; +const POWER_STATE: u32 = 0x19; +const POWER_ON: u32 = 0x11; +const POWER_OFF: u32 = 0x12; + +pub(crate) struct Load { + invalidated: AtomicBool, + sequence: AtomicU64, + samples: AtomicU64, + received_ns: AtomicU64, + ticks: AtomicU64, + utilization: [AtomicU32; 4], + power_known: AtomicU32, + powered: AtomicU32, + power_ticks: AtomicU64, + on_ticks: AtomicU64, + powered_ticks: AtomicU64, + unpowered_ticks: AtomicU64, + invalid: AtomicU64, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Report { + pub(crate) invalidated: bool, + pub(crate) samples: u64, + pub(crate) received_ns: u64, + pub(crate) ticks: u64, + pub(crate) utilization: [u32; 4], + pub(crate) power_known: bool, + pub(crate) powered: bool, + pub(crate) power_ticks: u64, + pub(crate) on_ticks: u64, + pub(crate) powered_ticks: u64, + pub(crate) unpowered_ticks: u64, + pub(crate) invalid: u64, +} + +impl Report { + /// An off witness at or after the sample overrides its retained percentages. + /// After wake, an old sample is unavailable until the new interval is sampled. + pub(crate) fn current(&self) -> Option<[u32; 4]> { + if self.invalidated { + return None; + } + if self.power_known && !self.powered && self.power_ticks >= self.ticks { + return Some([0; 4]); + } + if self.samples == 0 || self.ticks < self.on_ticks { + return None; + } + Some(self.utilization) + } +} + +fn ticks(words: &[u32; 18]) -> u64 { + u64::from(words[1]) | (u64::from(words[2]) << 32) +} + +impl Load { + pub(crate) const fn new() -> Self { + Self { + invalidated: AtomicBool::new(false), + sequence: AtomicU64::new(0), + samples: AtomicU64::new(0), + received_ns: AtomicU64::new(0), + ticks: AtomicU64::new(0), + utilization: [const { AtomicU32::new(0) }; 4], + power_known: AtomicU32::new(0), + powered: AtomicU32::new(0), + power_ticks: AtomicU64::new(0), + on_ticks: AtomicU64::new(0), + powered_ticks: AtomicU64::new(0), + unpowered_ticks: AtomicU64::new(0), + invalid: AtomicU64::new(0), + } + } + + /// The firmware receive worker is the sole writer, serialized by the device lock. + pub(crate) fn record(&self, role: usize, now_ns: u64, words: &[u32; 18]) { + let tag = words[0]; + if !matches!( + (role, tag), + (0, UTILIZATION) | (1, POWER_STATE | POWER_ON | POWER_OFF) + ) { + return; + } + if role == 1 && tag == POWER_STATE && words[6] <= 1 { + let timestamp = ticks(words); + let powered = u32::from(words[6] == 0); + if timestamp != 0 + && self.power_known.load(Ordering::Relaxed) != 0 + && self.powered.load(Ordering::Relaxed) == powered + { + // The state and wake epoch are unchanged. Only this atomic timestamp + // advances; a concurrent transition still changes the sequence below. + // Avoid rewriting the complete snapshot for every repeated report. + if timestamp > self.power_ticks.load(Ordering::Relaxed) { + self.power_ticks.store(timestamp, Ordering::Relaxed); + } + return; + } + } + self.sequence.fetch_add(1, Ordering::AcqRel); + match (role, tag) { + (0, UTILIZATION) => { + let timestamp = ticks(words); + if timestamp == 0 || words[3..7].iter().any(|&percent| percent > 100) { + self.invalid.fetch_add(1, Ordering::Relaxed); + } else if timestamp > self.ticks.load(Ordering::Relaxed) { + for (out, value) in self.utilization.iter().zip(&words[3..7]) { + out.store(*value, Ordering::Relaxed); + } + self.received_ns.store(now_ns, Ordering::Relaxed); + self.ticks.store(timestamp, Ordering::Relaxed); + self.samples.fetch_add(1, Ordering::Relaxed); + } + } + (1, POWER_STATE) => { + let timestamp = ticks(words); + if timestamp == 0 || words[6] > 1 { + self.invalid.fetch_add(1, Ordering::Relaxed); + } else if timestamp > self.power_ticks.load(Ordering::Relaxed) { + let powered = u32::from(words[6] == 0); + if powered != 0 + && (self.power_known.load(Ordering::Relaxed) == 0 + || self.powered.load(Ordering::Relaxed) == 0) + { + self.on_ticks.store(timestamp, Ordering::Relaxed); + } + self.powered.store(powered, Ordering::Relaxed); + self.power_known.store(1, Ordering::Relaxed); + self.power_ticks.store(timestamp, Ordering::Relaxed); + } + } + (1, POWER_ON | POWER_OFF) => { + let counter = if tag == POWER_OFF { + &self.powered_ticks + } else { + &self.unpowered_ticks + }; + counter.store( + counter.load(Ordering::Relaxed).saturating_add(ticks(words)), + Ordering::Relaxed, + ); + } + _ => {} + } + self.sequence.fetch_add(1, Ordering::Release); + } + + /// Fatal device loss and shutdown cannot leave the last busy value current. + pub(crate) fn invalidate(&self) { + self.invalidated.store(true, Ordering::Release); + } + + pub(crate) fn snapshot(&self) -> Option { + for _ in 0..8 { + let sequence = self.sequence.load(Ordering::Acquire); + if sequence & 1 != 0 { + continue; + } + let report = Report { + invalidated: self.invalidated.load(Ordering::Acquire), + samples: self.samples.load(Ordering::Relaxed), + received_ns: self.received_ns.load(Ordering::Relaxed), + ticks: self.ticks.load(Ordering::Relaxed), + utilization: core::array::from_fn(|i| self.utilization[i].load(Ordering::Relaxed)), + power_known: self.power_known.load(Ordering::Relaxed) != 0, + powered: self.powered.load(Ordering::Relaxed) != 0, + power_ticks: self.power_ticks.load(Ordering::Relaxed), + on_ticks: self.on_ticks.load(Ordering::Relaxed), + powered_ticks: self.powered_ticks.load(Ordering::Relaxed), + unpowered_ticks: self.unpowered_ticks.load(Ordering::Relaxed), + invalid: self.invalid.load(Ordering::Relaxed), + }; + fence(Ordering::Acquire); + if sequence == self.sequence.load(Ordering::Relaxed) { + return Some(report); + } + } + None + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6ca0398abd2e27..47efa034af0b98 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -17,6 +17,26 @@ //! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state //! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. +/// A finite device resource refused an allocation: evaluates to `ENOSPC`. +/// The first refusal at each call site is reported once per module load, +/// naming the resource and site, so a client's ENOSPC can be attributed. +macro_rules! resource_exhausted { + ($what:literal) => {{ + static REPORTED: ::core::sync::atomic::AtomicBool = + ::core::sync::atomic::AtomicBool::new(false); + if !REPORTED.swap(true, ::core::sync::atomic::Ordering::Relaxed) { + let file = ::core::file!(); + ::kernel::pr_warn!( + "asahi: G17 {} exhausted ({}:{})\n", + $what, + file.rsplit("asahi/").next().unwrap_or(file), + ::core::line!() + ); + } + ::kernel::error::code::ENOSPC + }}; +} + mod buffer; mod channel; mod command; @@ -42,6 +62,9 @@ mod queue; mod recovery; mod runtime; pub(crate) mod status; +mod statistics; +mod load; +mod telemetry; mod teardown; mod timeout; mod validation; @@ -479,6 +502,8 @@ struct Firmware { registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -611,6 +636,7 @@ struct Shared { descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, + statistics: Arc, #[pin] feed_work: Work, preparations: Arc, @@ -791,6 +817,7 @@ impl Shared { /// Marks the device lost without a crash log. fn lose_device_quietly(self: &Arc) { + self.statistics.load.invalidate(); self.preparations.fail(); if !self.crashed.swap(true, Ordering::AcqRel) { dev_err!(self.dev.as_ref(), "GPU runtime stopped, device lost\n"); @@ -802,6 +829,7 @@ impl Shared { /// /// The firmware cannot be restarted: outstanding and future work fails. fn lose_device(self: &Arc, role: Role, crashlog: Option<&[u8]>) { + self.statistics.load.invalidate(); self.preparations.fail(); self.crashed.store(true, Ordering::Release); dev_err!( @@ -824,6 +852,52 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _memory: Pin>>, + _statistics: Pin>>, + _capacity: Pin>>, + _telemetry: Option, +} + +/// One line of queue resource counts, taken with the device mutex. `contexts` +/// counts leased application contexts against the TTBAT limit; `closing` are +/// those still held by closed queues whose scheduler release has not completed. +struct CapacityView(Arc); + +impl kernel::debugfs::Writer for CapacityView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let state = self.0.state.lock(); + match (*state).as_deref() { + Some(firmware) => writeln!(f, "version=1 {}", firmware.capacity()), + None => writeln!(f, "version=1 stopped"), + } + } +} + +struct StatisticsView(Arc); + +impl kernel::debugfs::Writer for StatisticsView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.0.request_capture(now_ns()); + writeln!(f, "version=2 payload=raw_firmware_words received_clock=monotonic_ns capture_window_ns={}", statistics::CAPTURE_NS)?; + for role in 0..2 { + for tag in 16..=30 { + let Some(report) = self.0.snapshot(role, tag) else { + writeln!(f, "role={} tag={:#x} busy=1", role, tag)?; + continue; + }; + if report.count == 0 { + continue; + } + write!(f, "role={} tag={:#x} count={} received_ns={} words=", + role, tag, report.count, report.received_ns)?; + for word in report.words { + write!(f, "{:08x} ", word)?; + } + writeln!(f)?; + } + } + Ok(()) + } } impl Gpu { @@ -857,6 +931,19 @@ impl Gpu { let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; + let statistics = Arc::new(statistics::Statistics::new(), GFP_KERNEL)?; + let report_statistics = statistics.clone(); + let telemetry = if *crate::module_parameters::stats_export.value() == 0 { + None + } else { + match telemetry::Telemetry::new(pdev.as_ref(), statistics.clone()) { + Ok(telemetry) => Some(telemetry), + Err(error) => { + dev_warn!(pdev.as_ref(), "GPU telemetry registration failed: {:?}\n", error); + None + } + } + }; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let shared = Arc::pin_init( pin_init!(Shared { @@ -867,6 +954,7 @@ impl Gpu { descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, + statistics, feed_work <- new_work!("g17::Shared::feed_work"), preparations, work <- new_work!("g17::Shared::work"), @@ -882,6 +970,19 @@ impl Gpu { GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let memory = KBox::pin_init( + directory.read_only_file(c"memory", crate::agx_memory_stats::View), + GFP_KERNEL, + )?; + let statistics = KBox::pin_init( + directory.read_only_file(c"statistics", StatisticsView(report_statistics)), + GFP_KERNEL, + )?; + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", CapacityView(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -890,6 +991,10 @@ impl Gpu { geometry, ids: Default::default(), shared, + _memory: memory, + _statistics: statistics, + _capacity: capacity, + _telemetry: telemetry, }, GFP_KERNEL, )?; @@ -965,6 +1070,7 @@ impl Gpu { registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -1029,6 +1135,7 @@ impl Gpu { /// Stops the firmware and releases everything it used. pub(crate) fn shutdown(&self) { + self.shared.statistics.load.invalidate(); self.shared.feed.begin_shutdown(); self.shared.preparations.remove(); self.shared.stop_queues(); diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae4..d2612f50fea254 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -66,6 +66,9 @@ unsafe impl Send for KernelObject {} unsafe impl Sync for KernelObject {} impl KernelObject { + pub(crate) fn classify_memory(&self, category: crate::agx_memory_stats::KernelCategory) { + self.obj.gem.classify_memory(category); + } /// Allocates a zeroed object of `size` bytes and maps it into `vm`. pub(crate) fn new( dev: &AsahiDrmDevice, @@ -161,6 +164,17 @@ impl KernelObject { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) } + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) + } + /// Maps an exact backing subrange, retaining an explicit unmapped guard. pub(crate) fn map_range( &self, @@ -321,6 +335,13 @@ impl Allocator<'_> { prot, cpu, ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("firmware GPU VA") + } else { + error + } + }) } /// Allocates a canonical low mapping, which can be aliased at the same @@ -340,6 +361,79 @@ impl Allocator<'_> { prot, cpu, ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA") + } else { + error + } + }) + } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + )? + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + )? + }; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + )?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } } } @@ -478,6 +572,13 @@ impl Pool { }); } let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; + use crate::agx_memory_stats::KernelCategory; + object.classify_memory(match family { + Family::Descriptor => KernelCategory::Descriptor, + Family::WorkState => KernelCategory::WorkState, + Family::QueueState => KernelCategory::QueueState, + Family::QueueConfig => KernelCategory::QueueConfig, + }); let reserved = { let mut inner = self.inner.lock(); if inner.stopped { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f6..1838462da3553f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -10,7 +10,7 @@ use crate::g17::{ hazard::{self, Footprint, Plan}, - job::Packet, + job::{Order, Packet}, }; use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; use core::sync::atomic::{AtomicU32, Ordering}; @@ -46,6 +46,10 @@ struct InFlight { } const QUARANTINE_WAIT_MS: u32 = 1000; +/// Bound on waiting for a lease of this owner's own pairs (all at depth) or a +/// transiently borrowed free slot. Every awaited lease belongs to published +/// work, which retires or is failed by its watchdog well within this time. +const PICK_WAIT_MS: u32 = 4 * crate::g17::job::TIMEOUT_MS; struct State { active: u64, @@ -58,6 +62,13 @@ struct State { epoch: u64, footprints: KVec, next_ticket: u64, + /// Live owners that gave a sibling pair to a refused owner. They keep + /// their remaining pair and never receive another while they are open, + /// not even the one they gave away. Their work-state channel ordinals stay + /// bound to the pairs they published on, and the displaced graph of the + /// given pair stays retained for their scheduler until its own release: + /// their scheduler must not get a second graph on that pair before then. + forfeited: [u64; cfg::RENDER_SLOTS], } impl State { const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; @@ -73,6 +84,7 @@ impl State { epoch: 0, footprints: KVec::new(), next_ticket: 1, + forfeited: [0; cfg::RENDER_SLOTS], } } fn exhausted(&self) -> bool { @@ -109,24 +121,36 @@ impl State { self.advance(slot); slot as u8 } - fn acquire(&mut self, owner: u64) -> Option { + fn acquire(&mut self, owner: u64, follow: Option) -> Option { if owner == 0 { return None; } + // A command ordered after the owner's newest live render gains no + // concurrency from the sibling: its barrier waits for that render + // anyway. Queue it behind it on the same pair, where the queue's own + // order replaces a cross-queue firmware dependency. + if let Some(slot) = follow { + let index = usize::from(slot); + self.refs[index] += 1; + self.advance(index); + return Some(slot); + } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -157,7 +181,9 @@ impl State { Some(first) } fn claim_sibling(&mut self, owner: u64) { - if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER + || self.forfeited.contains(&owner) + { return; } let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; @@ -175,8 +201,109 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; + } + } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true + } + /// A refused owner without any pair takes the least recently used idle + /// pair of a live owner which keeps another usable pair. `candidates` are + /// the slots whose physical pair the device lock proved idle, with the + /// owner bound to that pair; `empty` are the slots without any physical + /// identities. The recipient gives back its empty, unborrowed slots and ends + /// with exactly the received pair. Returns the slot and the donor. + fn forfeit_sibling( + &mut self, + candidates: &[(u8, u64)], + empty: u64, + owner: u64, + ) -> Option<(u8, u64)> { + if owner == 0 + || (0..cfg::RENDER_SLOTS).any(|slot| { + self.owners[slot] == owner + && (empty & (1u64 << slot) == 0 + || self.refs[slot] != 0 + || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0) + }) + { + return None; + } + let record = self.forfeited.iter().position(|value| *value == 0)?; + let blocked = self.active | self.quarantined | self.kill_hold | self.released; + let (slot, donor) = candidates + .iter() + .copied() + .filter(|(slot, donor)| { + let slot = usize::from(*slot); + let bit = 1u64 << slot; + slot != 0 + && slot < cfg::RENDER_SLOTS + && *donor != 0 + && *donor != owner + && self.owners[slot] == *donor + && self.refs[slot] == 0 + && blocked & bit == 0 + && self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == *donor + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + }) + }) + .min_by_key(|(slot, _)| (self.epochs[usize::from(*slot)], *slot))?; + if !self.forfeited.contains(&donor) { + self.forfeited[record] = donor; + } + for other in 0..cfg::RENDER_SLOTS { + if self.owners[other] == owner { + self.owners[other] = 0; + self.released &= !(1u64 << other); + self.epochs[other] = 0; + } + } + self.owners[usize::from(slot)] = owner; + self.epochs[usize::from(slot)] = 0; + Some((slot, donor)) } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { @@ -188,9 +315,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -213,9 +362,37 @@ impl State { ) } - fn pick(&mut self, owner: u64, plan: Plan) -> Option { + /// The pair holding the owner's newest live render when `request` is + /// ordered after that render by its queue prefix, and the pair can take + /// another command. + fn follow(&self, owner: u64, request: Option>) -> Option { + let request = request?; + let newest = self + .footprints + .iter() + .filter(|entry| entry.owner == owner && entry.slot.is_some()) + .max_by_key(|entry| entry.sequence)?; + if !Order::contains(request.prefix, newest.sequence) { + return None; + } + let slot = newest.slot?; + let index = usize::from(slot); + let bit = 1u64 << slot; + if index >= cfg::RENDER_SLOTS + || self.owners[index] != owner + || (self.quarantined | self.kill_hold) & bit != 0 + || self.active & bit == 0 + || self.refs[index] == 0 + || self.refs[index] >= cfg::RENDER_DEPTH + { + return None; + } + Some(slot) + } + + fn pick(&mut self, owner: u64, plan: Plan, follow: Option) -> Option { match plan { - Plan::Any => self.acquire(owner), + Plan::Any => self.acquire(owner, follow), Plan::Wait => None, Plan::Pair(slot) => { let index = usize::from(slot); @@ -295,6 +472,21 @@ impl Pool { GFP_KERNEL, ) } + /// Open logical queues, each of which may need a render pair. + pub(crate) fn queues(&self) -> usize { + self.queues.load(Ordering::Acquire) as usize + } + /// Owned, released, exit-held and quarantined slots, for diagnostics. + pub(crate) fn census(&self) -> [usize; 4] { + let state = self.state.lock(); + let bits = |mask: u64| (mask & State::MASK).count_ones() as usize; + [ + state.owners[1..].iter().filter(|owner| **owner != 0).count(), + bits(state.released), + bits(state.kill_hold), + bits(state.quarantined), + ] + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { @@ -312,17 +504,30 @@ impl Pool { } } } + /// Waits at most until `deadline` (monotonic ns), the bound of the whole + /// publication, besides each wait's own budget. pub(crate) fn acquire_render( self: &Arc, owner: u64, request: Option>, + deadline: u64, ) -> Result { if owner == 0 { return Err(EINVAL); } + // A wait of at most `budget` jiffies, cut short at the deadline. + let capped = |budget: kernel::time::Jiffies| { + let left = deadline.saturating_sub(crate::g17::now_ns()) / 1_000_000; + budget.min(kernel::time::msecs_to_jiffies(left.min(u64::from(u32::MAX)) as u32)) + }; + // The budget left after waiting `timeout` jiffies of which `left` remained. + let spend = |budget: kernel::time::Jiffies, + timeout: kernel::time::Jiffies, + left: kernel::time::Jiffies| budget.saturating_sub(timeout.saturating_sub(left)); let mut state = self.state.lock(); let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let mut pick_budget = kernel::time::msecs_to_jiffies(PICK_WAIT_MS); let mut expired = false; let mut ticket = None; let result = (|| loop { @@ -337,16 +542,19 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { + let timeout = capped(quarantine_budget); + if timeout == 0 { return Err(EIO); } quarantine_budget = match self .released - .wait_interruptible_timeout(&mut state, quarantine_budget) + .wait_interruptible_timeout(&mut state, timeout) { CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, + CondVarTimeoutResult::Woken { jiffies } => { + spend(quarantine_budget, timeout, jiffies) + } }; continue; } @@ -356,7 +564,8 @@ impl Pool { } else { state.plan(owner, request, ticket) }; - if let Some(slot) = state.pick(owner, plan) { + let follow = state.follow(owner, request); + if let Some(slot) = state.pick(owner, plan, follow) { state.forget_waiter(ticket.take()); let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -376,19 +585,40 @@ impl Pool { } } } + let timeout = capped(conflict_budget); + if timeout == 0 { + // Out of time for overlap: take any slot without waiting. + expired = true; + continue; + } match self .released - .wait_interruptible_timeout(&mut state, conflict_budget) + .wait_interruptible_timeout(&mut state, timeout) { CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), CondVarTimeoutResult::Timeout => { conflict_budget = 0; expired = true; } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + CondVarTimeoutResult::Woken { jiffies } => { + conflict_budget = spend(conflict_budget, timeout, jiffies) + } + } + } else { + // Never wait unboundedly in publication: the scheduler watchdog + // cannot fail a packet that has not claimed hardware yet. + let timeout = capped(pick_budget); + if timeout == 0 { + return Err(EBUSY); } - } else if self.released.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); + pick_budget = match self + .released + .wait_interruptible_timeout(&mut state, timeout) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => spend(pick_budget, timeout, jiffies), + }; } })(); state.forget_waiter(ticket); @@ -410,7 +640,8 @@ impl Pool { return Ok(None); } let plan = state.plan(owner, request, None); - let Some(slot) = state.pick(owner, plan) else { + let follow = state.follow(owner, request); + let Some(slot) = state.pick(owner, plan, follow) else { return Ok(None); }; let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -420,8 +651,42 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) + } + /// The caller holds the device mutex, which proved every candidate pair idle. + pub(crate) fn forfeit_sibling( + &self, + candidates: &[(u8, u64)], + empty: u64, + owner: u64, + ) -> Option<(u8, u64)> { + self.state.lock().forfeit_sibling(candidates, empty, owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -430,6 +695,9 @@ impl Pool { state.released |= 1u64 << slot; } } + for value in state.forfeited.iter_mut().filter(|value| **value == owner) { + *value = 0; + } drop(state); self.released.notify_all(); } @@ -468,6 +736,17 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb7609..c9596d604df773 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -566,6 +566,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -626,6 +641,12 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -1134,6 +1155,17 @@ impl Queue { } self.retirement_ready = true; } + // An owner that rebound a detached queue and closed without + // publishing left no command for a witness to retire. Without + // this the queue keeps that owner, its QID and its VM aliases + // until processor stop. + if self.previous.is_none() + && self.replays.len == 0 + && self.exit.is_none() + && !self.growing + { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1141,6 +1173,50 @@ impl Queue { } Ok(false) } + /// Installed, populated and idle under a live owner: no retained, replayed, + /// quarantined, exiting, growing or superseded state. Publication on this + /// queue holds the device mutex, which the caller also holds, so no command + /// can become visible between this check and a takeover. + pub(crate) fn idle_owned(&self) -> bool { + self.owner.is_some() + && !self.released + && self.installed + && !self.growing + && self.pool_phase == PoolPhase::Populated + && !self.needs_fresh_backing + && self.exit.is_none() + && !self.quarantined + && !self.awaiting_witness + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.binding.is_some() + } + /// Releases an idle live owner exactly as its logical close would, so that + /// `bind_owner` can move the queue. The owner keeps its context and takes a + /// queue again before its next publication. + pub(crate) fn yield_idle_owner(&mut self) -> Result { + if !self.idle_owned() { + return Err(EBUSY); + } + self.released = true; + self.retirement_ready = true; + self.owner = None; + self.released_at = now_ns(); + Ok(()) + } + /// The bound owner's VM status, for preferring a same-VM takeover. + pub(crate) fn bound_status(&self) -> Option<&Arc> { + self.binding.as_ref().map(|binding| binding.context.status()) + } + /// GPU end time of the last retired command, for least-recent selection. + pub(crate) fn last_end(&self) -> u64 { + self.last_end + } + /// Bytes of USC pool backing in the shared lower window, for diagnostics. + pub(crate) fn pool_bytes(&self) -> usize { + self.pool.backing_bytes() + } /// The last owner's file and VM are gone: its execution root was released /// after the scheduler-state release and command retirement were witnessed /// (`Context::release_execution`), and this queue is idle without an owner. @@ -1734,7 +1810,7 @@ impl Queue { } let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); + return Err(resource_exhausted!("compute item ring")); } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b4d4bcef24421f..f5b68c67cf9354 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -156,10 +156,11 @@ pub(crate) struct Batch { } impl Batch { - pub(crate) fn new() -> Result { - Ok(Self { + /// Initialized in place: the slot table is too large to stage on the probe stack. + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, - slots: [(0, 0); QIDS], + slots <- pin_init::init_zeroed(), open: None, }) } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 569b5e289e7d8e..02913513fdbbd8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -27,7 +27,12 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; +/// Commands of one engine (render, compute) still in flight below which a +/// dependency-free command is published from its ioctl rather than from the +/// scheduler's worker, whose hop costs ~100 us per command on Neo. Compute +/// streams of three or four dispatches publish inline; renders keep the +/// shallow limit. +const EARLY_DEPTH: [usize; 2] = [2, 4]; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -129,7 +134,13 @@ impl Queue { fn ensure_compute(&mut self, scs_pages: u16) -> Result { // Admission is checked even when this queue already owns a scheduler. - self.backend.ensure_compute(scs_pages)?; + // Physical queues held by other live owners (ENOSPC) and a concurrent + // binder of this owner (EAGAIN) are contention, not client errors: + // publication binds before the command becomes visible, bounded. + match self.backend.ensure_compute(scs_pages) { + Err(error) if error == ENOSPC || error == EAGAIN => {} + result => result?, + } if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -274,9 +285,8 @@ impl Queue { checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + let early = dependencies.is_empty() + && self.frontiers[index].len() < EARLY_DEPTH[index]; let packet = Packet::new( id, self.context.clone(), diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e402..33ff9f9423fd96 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -152,6 +152,7 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, descriptor_flags: [u32; 2], installed: [bool; 2], @@ -170,12 +171,19 @@ pub(crate) struct Pair { context_update: bool, /// A retired, never-executed group awaiting republication on this pair. replay: Option>, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -213,8 +221,9 @@ impl Pair { } let layout = target::Layout::new(pass, clusters)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -228,7 +237,7 @@ impl Pair { word, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, @@ -239,6 +248,7 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, descriptor_flags, installed: [false; 2], @@ -253,6 +263,8 @@ impl Pair { displaced: None, replay: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -503,9 +515,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -730,9 +750,6 @@ impl Pair { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -781,6 +798,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -1578,6 +1600,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.is_some() || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -1616,6 +1650,19 @@ impl Pair { && previous_ready && self.idle_for_context(self.context())?) } + /// Idle under a live owner and reusable as soon as that owner is released: + /// no prepared, active, replayed, quarantined, displaced or unsettled + /// superseded state, so a successor needs no firmware witness first. + pub(crate) fn forfeitable(&self) -> Result { + let previous_ready = self + .previous + .as_ref() + .is_none_or(|p| p.memory.is_none() && self.context_update && !p.ready); + Ok(self.owner.is_some() + && self.displaced.is_none() + && previous_ready + && self.idle_for_context(self.context())?) + } /// Logical close leaves the physical channel identity and backing retained. pub(crate) fn release_owner(&mut self) { self.owner = None; @@ -1736,6 +1783,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 85319585b3d135..63932f4f385d7e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -13,6 +13,7 @@ use crate::g17::{ }, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; +use crate::agx_memory_stats::KernelCategory; use crate::{ hw::t8140::{dynamic, kernel_window, low, queues, resources as cfg}, mmu, @@ -52,14 +53,22 @@ impl Aliases { loop { // Descriptor aliases are read-only to GPU consumers; the canonical // firmware mapping remains writable for command state updates. - let global = object.map_range( - alloc.uat.kernel_lower_vm(), - source.clone(), - range.clone(), - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - )?; + let global = object + .map_range( + alloc.uat.kernel_lower_vm(), + source.clone(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RO, + 0, + ) + .map_err(|error| { + if error == ENOSPC { + resource_exhausted!("shared lower GPU VA (render descriptors)") + } else { + error + } + })?; let address = global.iova(); let end = address.checked_add(size).ok_or(EOVERFLOW)?; match object.map_range( @@ -80,6 +89,9 @@ impl Aliases { drop(global); range.start = end; } + Err(error) if error == ENOSPC => { + return Err(resource_exhausted!("client GPU VA for render descriptors")) + } Err(error) => return Err(error), } } @@ -140,6 +152,7 @@ impl Graph { prot, CpuMap::WriteCombined, )?; + support.classify_memory(KernelCategory::QueueSupport); let address = queues.gpu_va(); for stage in 0..2 { queues.write( @@ -241,6 +254,7 @@ struct Compact { impl Compact { fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + backing.classify_memory(KernelCategory::Scratch); let mapping = buffer::compact_alias( &backing, vm, @@ -266,14 +280,16 @@ pub(super) struct Generation { } impl Generation { fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { - KernelObject::new( + let state = KernelObject::new( alloc.dev, vm, crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, cfg::TVB_PAGE), storage::STATE_SIZE, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, - ) + )?; + state.classify_memory(KernelCategory::RenderGeneration); + Ok(state) } fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { let deflake = buffer::compact_alias( @@ -290,6 +306,7 @@ impl Generation { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + fragment.classify_memory(KernelCategory::RenderGeneration); let fragment_client = buffer::compact_alias( &fragment, vm, @@ -347,7 +364,6 @@ impl Generation { } pub(super) struct Memory { - _fragment_view: mmu::KernelMapping, pub(super) graph: Graph, scratch: Compact, discard: Compact, @@ -357,9 +373,6 @@ pub(super) struct Memory { pub(super) timestamps: KernelObject, pub(super) manager: buffer::Manager, pub(super) lifecycle: [fw::ObjectIds; 2], - // The extra client views retain their allocation positions even though - // kick entries consume the register arrays embedded in the descriptors. - registers: KernelObject, pub(super) context: Arc, pub(super) installed_pool: Option, } @@ -430,22 +443,6 @@ impl Memory { word, ) })?; - let registers = KernelObject::new( - alloc.dev, - vm, - crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), - mmu::UAT_PGSZ, - mmu::PROT_GPU_SHARED_RO, - CpuMap::WriteCombined, - )?; - let fragment_view = graph.descriptors.map_range( - vm, - TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, - dynamic::CLIENT_LOWER, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - )?; Ok(( storage.write_init(kernel::try_init!(Self { graph, @@ -457,8 +454,6 @@ impl Memory { timestamps, manager, lifecycle: [tiling, fragment], - _fragment_view: fragment_view, - registers: registers, context, installed_pool: None, }))?, @@ -542,21 +537,9 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick records use the arrays embedded in this descriptor directly. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, @@ -609,15 +592,13 @@ impl Memory { ) -> Result> { if self.scratch.mapping.is_none() || self.discard.mapping.is_none() - || self.registers.gpu_va() == 0 { return Err(EIO); } let mut aliases = KVec::new(); - aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.reserve(tvb.len().checked_add(2).ok_or(EOVERFLOW)?, GFP_KERNEL)?; aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; for mapping in core::mem::take(tvb) { aliases.push(mapping, GFP_KERNEL)?; } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 39941de815e077..c2ad99cb32523e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -31,6 +31,11 @@ pub(crate) struct ParkedPair { registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -48,11 +53,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -82,6 +96,7 @@ impl Pair { registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -96,6 +111,13 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } @@ -190,6 +212,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, @@ -215,6 +238,8 @@ impl ParkedPair { displaced: None, replay: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9f..f3bc478442f82f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -32,6 +32,55 @@ const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Bound on all contention waits of one publication together: capacity and +/// identities held by other live owners, borrowed pairs, work states, queue +/// binding and ring backpressure. Each frees when published work retires or +/// times out, when warming finishes or is reclaimed, or when an owner closes. +const PUBLICATION_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + +/// How a publication may wait for contention. +#[derive(Clone, Copy)] +pub(super) enum Wait { + /// From the scheduler's worker, until one deadline for the whole publication. + Until(u64), + /// From the submitting ioctl, which holds the logical queue's mutex: never + /// sleep for contention, return `FALLBACK`. + Never, +} + +impl Wait { + pub(super) fn scheduled() -> Self { + Self::Until(now_ns().saturating_add(PUBLICATION_WAIT_NS)) + } + /// Before a contention wait: `FALLBACK` when it may not sleep at all, + /// otherwise whether the publication's deadline has passed. + pub(super) fn expired(self) -> Result { + match self { + Self::Never => Err(FALLBACK), + Self::Until(deadline) => Ok(now_ns() >= deadline), + } + } + pub(super) fn deadline(self) -> Option { + match self { + Self::Never => None, + Self::Until(deadline) => Some(deadline), + } + } +} + +/// Whether a publication with no queue may take another live owner's idle one: +/// when neither a fresh QID nor surplus render capacity remains, or once its +/// own off-lock attempt (`contended`) found no released queue, fresh QID or +/// reclaimable render spare. Surplus held by installed render pairs never +/// becomes compute capacity, so it alone must not keep a publication waiting. +fn takeover_allowed(compute_available: bool, render_surplus: usize, contended: bool) -> bool { + contended || (!compute_available && render_surplus < 2) +} /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. @@ -74,9 +123,7 @@ impl Registry { ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), @@ -95,7 +142,7 @@ impl Registry { compute_building: false, compute_scheduler_building: false, replay_outer_wait: None, - observations: compute::retirement::Batch::new()?, + observations <- compute::retirement::Batch::new(), worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) @@ -130,6 +177,7 @@ impl Registry { }); } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -137,6 +185,12 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -148,6 +202,81 @@ impl Registry { Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) } + /// No fresh QID and no released queue remain: move the least recently used + /// idle queue of another live owner to `owner`, preferring one bound to the + /// same VM. Only a pool that already fits is taken; growth would have to + /// wait for a publication of the new owner. The previous owner keeps its + /// context and takes a queue again at its next publication. + pub(super) fn take_over_idle_compute( + &mut self, + owner: u64, + context: &Arc, + required: usize, + ) -> Result { + if !context.is_current() { + return Err(EFAULT); + } + let mut selected = None::<(bool, u64, usize)>; + for (index, entry) in self.compute.iter().enumerate() { + let Some(queue) = entry.as_ref().and_then(|entry| entry.queue.as_deref()) else { + continue; + }; + if queue.owner() == Some(owner) + || !queue.idle_owned() + || !queue.pool_fits(required) + || queue.context().is_some_and(|bound| self.teardown.releasing(bound)) + { + continue; + } + let foreign = !queue + .bound_status() + .is_some_and(|status| Arc::ptr_eq(status, context.status())); + let key = (foreign, queue.last_end(), index); + if selected.is_none_or(|best| key < best) { + selected = Some(key); + } + } + let Some((_, _, index)) = selected else { + return Ok(false); + }; + let queue = self.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)?; + queue.yield_idle_owner()?; + queue.bind_owner(owner, context.clone())?; + Ok(true) + } + + /// Publication's device-locked binding: a released queue that fits, or a + /// takeover when neither a fresh QID nor surplus render capacity remains, + /// or once an off-lock attempt of this publication (`contended`) found no + /// released queue, fresh QID or reclaimable render spare. Render surplus + /// held by installed pairs cannot become compute capacity, so it must not + /// keep a publication from taking an idle queue. False leaves + /// construction, re-arming or growth to the off-lock path. + pub(super) fn rebind_or_take_over_compute( + &mut self, + owner: u64, + context: &Arc, + required: usize, + contended: bool, + ) -> Result { + match self.rebind_compute(owner, context, required)? { + Rebind::Done => Ok(true), + Rebind::Fresh + if takeover_allowed( + self.qids.compute_available(self.admission.queues()), + self.qids.render_surplus(self.admission.queues()), + contended, + ) => + { + self.take_over_idle_compute(owner, context, required) + } + _ => Ok(false), + } + } + /// Installs fresh USC backing into the vacant queue `qid`; the caller writes /// the returned descriptor row before binding. EAGAIN if the queue was taken. pub(super) fn rearm_compute( @@ -167,6 +296,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -174,6 +304,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -275,6 +409,20 @@ impl Registry { Ok(freed) } + /// A retained queue in a pool-release transaction temporarily cannot bind. + /// Distinguish that state from permanent physical-identity exhaustion. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC + && self.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_pending()) + { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { @@ -296,7 +444,8 @@ impl Registry { let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve_compute(identity, Some(FIRST_COMPUTE_QID), self.admission.queues()) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, @@ -379,7 +528,21 @@ pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, + /// Render packets queued to the scheduler and not yet published. backlog: core::sync::atomic::AtomicU32, + /// Compute packets queued to the scheduler and not yet published. + compute_backlog: core::sync::atomic::AtomicU32, + /// One compute binder per owner: submission and publication can both bind, + /// and two concurrent binders could leave the owner with two queues. + binding: core::sync::atomic::AtomicBool, +} + +/// Clears the owner's compute-binder flag on every exit. +struct Binder<'a>(&'a core::sync::atomic::AtomicBool); +impl Drop for Binder<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } } impl Backend { @@ -389,24 +552,59 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed. When only such contexts block this one, wait + // for the teardown (bounded) instead of failing a live client; the + // preparation lease is not held while sleeping. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } - let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + let slot = firmware.queues.admission.reserve_queue().map_err(|error| { + if error == ENOSPC { + resource_exhausted!("logical queues") + } else { + error + } + })?; + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_in_use().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(resource_exhausted!("execution contexts")); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -429,6 +627,8 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), + binding: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, )?, @@ -438,9 +638,9 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, wait: Wait) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); @@ -463,6 +663,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } + if wait.expired()? { + return Err(self.wait_expired(EBUSY, "the shared compute aperture")); + } self.shared.changed.wait(&mut state); } }; @@ -551,7 +754,7 @@ impl Backend { result } - fn grow_compute(&self, required: usize) -> Result { + fn grow_compute(&self, required: usize, wait: Wait) -> Result { use kernel::time::{delay::fsleep, Delta}; { let state = self.shared.state.lock(); @@ -560,14 +763,13 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); loop { let result = (|| { let mut state = self.shared.state.lock(); @@ -586,19 +788,26 @@ impl Backend { deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + Err(error) if error == EAGAIN && lease.is_current() => { + if wait.expired()? { + return Err(self.wait_expired(error, "compute pool growth")); } + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } Err(error) => return Err(error), } } } - fn build_compute(&self, required: usize) -> Result { + /// Off-lock binding: a released queue, a re-armed vacant one, growth, or a + /// fresh QID. ENOSPC leaves the owner unbound. Taking over another live + /// owner's idle queue happens only in publication's device-locked rebind, + /// which publishes under the same mutex hold. Submission never waits for + /// or fails on physical capacity: it leaves binding to publication. + fn build_compute(&self, required: usize, wait: Wait) -> Result { loop { - match self.shared_compute() { + match self.shared_compute(wait) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -607,7 +816,7 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); @@ -621,10 +830,40 @@ impl Backend { } match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Grow => { drop(state); return self.grow_compute(required, wait); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + // A pool release is still in flight: the event worker + // observes it and drops the vacated backing off this + // mutex. The caller retries after it. + Err(error) if error == EAGAIN => { + drop(state); + self.shared.queue_events(); + return Err(ENOSPC); + } + Err(error) if error == ENOSPC => { + // Cancelling an unused render reservation frees + // compute capacity only beyond the render reserve. + let surplus = firmware + .queues + .qids + .render_surplus(firmware.queues.admission.queues()); + let reclaimed = if surplus >= 2 { + firmware.queues.reclaim_unused_render_spare(None, true)? + } else { + None + }; + if reclaimed.is_none() { + return Err(error); + } + drop(state); + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; (firmware.uat.clone(), Ok(reservation)) } } @@ -892,6 +1131,11 @@ impl compute::Host for ComputeHost<'_> { } impl super::Firmware { + /// One snapshot of every finite queue resource. + pub(in crate::g17) fn capacity(&self) -> render::Capacity { + self.queues.capacity(self.render_ids.in_use(), self.uat.execution_contexts_in_use()) + } + fn publish_compute( &mut self, owner: u64, @@ -918,6 +1162,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, @@ -1053,12 +1306,76 @@ impl super::Firmware { } impl Backend { + /// Submission-time binding: never takes over another owner's queue and + /// reports ENOSPC when no physical queue is free, leaving publication to bind. fn ensure_compute(&self, scs_pages: u16) -> Result { let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; + // The submit ioctl never sleeps for contention: publication binds. + match self.bind_compute(required, Wait::Never) { + Err(error) if error == FALLBACK => Err(EAGAIN), + result => result, + } + } + + /// A publication gave up waiting for `what` at its deadline and fails with + /// `error`. Names the wait, its source line and the owner, at most once a + /// second, so a lost device reported for contention is attributable from + /// the kernel log alone. Only reached on that failure path. + #[track_caller] + pub(super) fn wait_expired(&self, error: Error, what: impl kernel::fmt::Display) -> Error { + use core::sync::atomic::AtomicU64; + static NEXT: AtomicU64 = AtomicU64::new(0); + static SUPPRESSED: AtomicU64 = AtomicU64::new(0); + let now = now_ns(); + let next = NEXT.load(Ordering::Relaxed); + if now < next + || NEXT + .compare_exchange(next, now + 1_000_000_000, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + SUPPRESSED.fetch_add(1, Ordering::Relaxed); + return error; + } + let at = core::panic::Location::caller(); + dev_warn!( + self.shared.dev.as_ref(), + "G17 publication of pid {} gave up waiting for {} ({:?}, {}:{}); {} similar since the last report\n", + self.context.owner_pid(), + what, + error, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), + at.line(), + SUPPRESSED.swap(0, Ordering::Relaxed) + ); + error + } + + /// The preparation lease. From the ioctl, a recovery that has closed the + /// preparation epoch is contention. + fn enter_preparation(&self, wait: Wait) -> Result { + match wait { + Wait::Never => self + .shared + .preparations + .try_enter() + .map_err(|error| if error == EAGAIN { FALLBACK } else { error }), + Wait::Until(_) => self.shared.preparations.enter(false), + } + } + + /// EAGAIN while another binder of this owner runs; it binds for both. + fn bind_compute(&self, required: usize, wait: Wait) -> Result { use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; + if self + .binding + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(EAGAIN); + } + let _binder = Binder(&self.binding); let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1070,15 +1387,135 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + return self.build_compute(required, wait); + } + // Keep the spared queue and its retained work intact until a later + // admission observes its witness. Other owners continue submitting + // while this owner gives its queue-local drain witness time to arrive. + if wait.expired()? { + return Err(self.wait_expired(EBUSY, "a spared compute queue's drain witness")); } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. fsleep(Delta::from_millis(1)); } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - Err(EBUSY) + } + + /// Publishes a dependency-free compute command from the submitting ioctl, + /// as renders are, instead of waking the scheduler's worker for it. Only a + /// queue already bound to this owner that fits the command and has room + /// qualifies, and only while no earlier compute command of this owner + /// still waits in the scheduler, so publication order stays submission + /// order. The ioctl never binds, grows, replays or sleeps: any of those + /// returns false before anything became visible and leaves the command to + /// the scheduler. + pub(in crate::g17) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return false; + }; + let Ok(required) = + super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) + else { + return false; + }; + { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + if firmware.recovery.pending() + || self.binding.load(Ordering::Acquire) + || !firmware + .queues + .compute(self.owner) + .is_some_and(|queue| queue.pool_fits(required) && queue.room()) + { + return false; + } + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.publish_compute_early(packet, required, &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + /// The ioctl's single publication attempt, under one device mutex hold on + /// the owner's bound queue. Contention returns `FALLBACK` unpublished. + fn publish_compute_early( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Result { + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = self.enter_preparation(Wait::Never)?; + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => return Err(FALLBACK), + result => { + result?; + } + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + || self.binding.load(Ordering::Acquire) + // Replays of recovered commands publish first, from the scheduler. + || firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.replay_commands() != 0) + || !firmware + .queues + .compute(self.owner) + .is_some_and(|queue| queue.pool_fits(required) && queue.room()) + { + return Err(FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) + { + Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { + Err(FALLBACK) + } + result => result, + } } /// Called after taking this packet's publication claim. All waits and work @@ -1087,16 +1524,38 @@ impl Backend { &self, packet: &Arc, deferred: &mut DeferredBatch, + wait: Wait, ) -> Result { packet.check_dependencies()?; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Err(EINVAL); + }; + let required = + super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; let mut recovery_polls = 0; - let mut outer_wait = None; + let mut state_delay = 20i64; + let mut contended = false; loop { if packet.completion.status().get() != 0 { return Err(EIO); } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; + let lease = self.enter_preparation(wait)?; + // Work states free as this context's earlier commands retire. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + if wait.expired()? { + return Err(self.wait_expired(error, "a free compute work state")); + } + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { @@ -1115,8 +1574,8 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); + if wait.expired()? || recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(self.wait_expired(EBUSY, "GPU recovery to settle")); } recovery_polls += 1; drop(state); @@ -1126,6 +1585,52 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; + if !firmware + .queues + .compute(self.owner) + .is_some_and(|queue| queue.pool_fits(required)) + { + // Submission found no free physical queue, or another owner + // took this idle one. Bind again; a takeover publishes under + // the same mutex hold. Capacity frees whenever any queue's + // published work retires or times out, so the wait is bounded. + if wait.expired()? { + return Err(self.wait_expired(EBUSY, "a compute queue to bind")); + } + // An off-lock binder of this owner may hold a reservation or + // a queue being re-armed: binding here could give the owner a + // second queue. Its flag is raised before its first decision. + let taken = !self.binding.load(Ordering::Acquire) + && firmware.queues.compute(self.owner).is_none() + && match firmware.queues.rebind_or_take_over_compute( + self.owner, + &self.context, + required, + contended, + ) { + Err(error) if error == ENOSPC => false, + result => result?, + }; + if !taken { + drop(state); + drop(lease); + // Replays collected above belong to other packets; do not + // hold their fences while this one waits for a queue. + deferred.finish(); + match self.bind_compute(required, wait) { + Ok(()) => {} + Err(error) if [ENOSPC, EBUSY, EAGAIN].contains(&error) => { + // No released queue, fresh QID or spare: the next + // locked attempt may take an idle queue over. + contended |= error != EAGAIN; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } + continue; + } + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; let result = firmware.publish_compute( self.owner, @@ -1138,8 +1643,8 @@ impl Backend { && (firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); + if wait.expired()? || recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(self.wait_expired(EBUSY, "GPU recovery to settle")); } recovery_polls += 1; drop(state); @@ -1149,10 +1654,20 @@ impl Backend { continue; } if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); + if wait.expired()? { + let releasing = firmware.queues.compute(self.owner).is_some_and(|queue| { + queue + .pending_switch_from() + .is_some_and(|previous| firmware.queues.teardown.releasing(previous)) + }); + return Err(self.wait_expired( + EBUSY, + if releasing { + "the previous owner's scheduler release" + } else { + "compute ring room" + }, + )); } // Backpressure owns no firmware-visible attempt. Other queues // can submit and the event worker can retire while we sleep. @@ -1262,9 +1777,16 @@ impl super::Firmware { if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f1..305c12b1b4a23e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,10 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + match packet.engine() { + Engine::Render => self.backlog.fetch_sub(1, Ordering::AcqRel), + Engine::Compute => self.compute_backlog.fetch_sub(1, Ordering::AcqRel), + }; } } @@ -138,7 +141,9 @@ impl job::Backend for Backend { Err(EIO) } else { match packet.engine() { - Engine::Compute => self.compute_packet(&packet, &mut deferred), + Engine::Compute => { + self.compute_packet(&packet, &mut deferred, super::Wait::scheduled()) + } Engine::Render => self.render_packet(&packet, &mut deferred), } }; @@ -147,7 +152,13 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { - self.release_backlog(&packet); + // A packet still in publication keeps its place in the backlog: its + // publisher waits on, and a later dependency-free command must not be + // published from its ioctl ahead of it. Publication, cancellation and + // job drop release it. + if packet.is_published() { + self.release_backlog(&packet); + } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -184,11 +195,17 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + match packet.engine() { + Engine::Render => self.backlog.fetch_add(1, Ordering::AcqRel), + Engine::Compute => self.compute_backlog.fetch_add(1, Ordering::AcqRel), + }; } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34aec..48867cab3ddd1f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -50,7 +50,14 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Unpublished packets wait in publication, which bounds them. Without a + // collector the watchdog expires again and retries. + if !packet.is_published() { + return Ok(()); + } + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3ae1d0a609a335..fef9463d699cc6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -34,7 +34,8 @@ impl event::Transport for Service<'_> { let firmware = &mut self.firmware; firmware .rings - .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role, + &firmware.primary.state.shared.statistics); } fn drain( &mut self, @@ -419,7 +420,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf20455261..77ca6e85026037 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -46,7 +46,13 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -55,6 +61,85 @@ struct Entry { published: bool, } +/// Counts of every finite queue resource at one instant. Displayed on one line. +pub(in crate::g17) struct Capacity { + queues: usize, + contexts: usize, + slots: [usize; 4], + entries: [usize; 6], + retired: [usize; 2], + teardown: [usize; 3], + qids: (usize, usize, usize), + compute: [usize; 3], + buffers: usize, + metrics: (usize, usize), + render_pools: usize, + pools: [usize; 3], +} + +// Printed by `pr_warn!` (kernel formatting) and written to debugfs (core formatting). +impl kernel::fmt::Display for Capacity { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + core::fmt::Display::fmt(self, f) + } +} + +impl core::fmt::Display for Capacity { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let [owned, released, held, quarantined] = self.slots; + let [entries, pairs, parked, building, preparing, closed] = self.entries; + let [open, draining, releasing] = self.teardown; + let (leased, pair_ids, compute_ids) = self.qids; + write!( + f, + "queues={} contexts={}/{} (closing {}); slots owned={} released={} held={} \ + quarantined={}; entries={} pairs={} parked={} building={} preparing={} \ + closed={}; retired={}/{}; teardown open={} draining={} releasing={}; qids \ + leased={} pair={} compute={}; compute queues={} bound={} pools={} MiB; \ + pb ids={}; metrics={}/{}; render pools={} (with entries {} totalling {} MiB, \ + largest {} MiB)", + self.queues, + self.contexts, + crate::mmu::MAX_EXECUTION_CONTEXTS, + self.contexts.saturating_sub(self.queues), + owned, + released, + held, + quarantined, + entries, + pairs, + parked, + building, + preparing, + closed, + self.retired[0], + self.retired[1], + open, + draining, + releasing, + leased, + pair_ids, + compute_ids, + self.compute[0], + self.compute[1], + self.compute[2] >> 20, + self.buffers, + self.metrics.0, + self.metrics.1, + self.render_pools, + self.pools[0], + self.pools[1] >> 20, + self.pools[2] >> 20, + ) + } +} + +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -124,6 +209,22 @@ impl RenderState { .ok_or(EIO) } + /// Whether the pair in `slot` is out of its entry, borrowed by an event + /// pass, a preparation or a growth that returns it under the device mutex. + fn borrowed(&self, slot: u8) -> bool { + self.entry(slot).is_ok_and(|entry| { + entry.pair.is_none() && (entry.preparing.is_some() || entry.growing) + }) + } + + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -157,6 +258,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -176,6 +278,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -198,6 +301,224 @@ impl RenderState { } impl Registry { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + /// Every admission slot is held by a live owner and no unused graph can + /// yield: move an idle installed pair from a live owner which keeps another + /// installed pair to `owner`. Only the owner's logical binding is released; + /// QIDs, backing and the old context's retained graph follow the existing + /// rebinding rules, and its scheduler is displaced as still live. + pub(super) fn forfeit_idle_render_sibling(&mut self, owner: u64) -> Result { + let mut candidates: KVec<(u8, u64)> = KVec::with_capacity(RENDER_SLOTS, GFP_KERNEL)?; + let mut empty = 0u64; + for slot in 1..RENDER_SLOTS { + let Some(entry) = self.render.entries[slot].as_ref() else { + empty |= 1u64 << slot; + continue; + }; + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + || entry.closed + || entry.parked.is_some() + { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + let Some(donor) = pair.owner() else { + continue; + }; + if donor == owner || donor != entry.reservation.owner || !pair.forfeitable()? { + continue; + } + let installed = self + .render + .entries + .iter() + .flatten() + .filter(|other| { + other.pair.as_ref().is_some_and(|pair| pair.owner() == Some(donor)) + }) + .count(); + if installed < 2 { + continue; + } + candidates + .push_within_capacity((slot as u8, donor)) + .map_err(|_| EIO)?; + } + let Some((slot, _)) = self.admission.forfeit_sibling(&candidates, empty, owner) else { + return Ok(false); + }; + self.render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .release_owner(); + Ok(true) + } + + /// A snapshot of every finite queue resource, for attributing refusals. + pub(in crate::g17) fn capacity(&self, render_pools: usize, contexts: usize) -> Capacity { + let mut entries = [0usize; 6]; + for entry in self.render.entries.iter().flatten() { + entries[0] += 1; + entries[1] += usize::from(entry.pair.is_some()); + entries[2] += usize::from(entry.parked.is_some()); + entries[3] += usize::from(entry.constructing || entry.growing); + entries[4] += usize::from(entry.preparing.is_some()); + entries[5] += usize::from(entry.closed); + } + let compute = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .fold([0usize; 3], |[all, bound, bytes], queue| { + [all + 1, bound + usize::from(queue.owner().is_some()), bytes + queue.pool_bytes()] + }); + // Render pools reachable from installed or prepared entries, each once. + let mut pools = [0usize; 3]; + for (index, entry) in self.render.entries.iter().enumerate() { + let Some(pool) = entry.as_ref().and_then(|entry| entry.pool.as_ref()) else { + continue; + }; + let first = !self.render.entries[..index].iter().flatten().any(|other| { + other.pool.as_ref().is_some_and(|other| Arc::ptr_eq(other, pool)) + }); + if first { + let bytes = pool.backing_bytes(); + pools[0] += 1; + pools[1] += bytes; + pools[2] = pools[2].max(bytes); + } + } + Capacity { + queues: self.admission.queues(), + contexts, + slots: self.admission.census(), + entries, + retired: [self.render.retired.len(), self.render.retired.capacity()], + teardown: self.teardown.phases(), + qids: self.qids.census(), + compute, + buffers: self.buffers.in_use(), + metrics: self.metrics.census(), + render_pools, + pools, + } + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); @@ -228,7 +549,9 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -262,12 +585,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -763,9 +1082,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index b283fb8e7d55ca..1090349072e150 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,7 +271,10 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c80018..8bc8dad5d6bb51 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -163,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 06bea3c84fa7ad..9e965fa870ce61 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,6 +29,7 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), @@ -72,8 +74,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + wait: super::super::Wait, + ) -> Result { + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -133,6 +141,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -164,6 +179,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -242,6 +259,7 @@ impl super::super::Backend { Pair::new( &alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, @@ -397,6 +415,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -463,14 +484,21 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // A pair forfeited by a live owner still names that owner's + // scheduler; only its logical close may arm the release. + let owner_closed = entry.closed; + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(resource_exhausted!("retired render bindings")); } firmware.queues.teardown.displace( pair.context(), - true, + owner_closed, super::super::now_ns(), )?; pair.bind_owner( @@ -499,6 +527,10 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -556,8 +588,13 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render( + &self, + packet: &Arc, + slot: u8, + wait: super::super::Wait, + ) -> Result { + self.build_render_pair(slot, packet, false, wait)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -568,10 +605,16 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + // Warming is optional: exhausted identities, VA or memory must not + // fail a command whose own pair is ready. Its failed construction + // leaves the reservation unchanged, as for the primary pair. + match self.build_render_pair(sibling.slot(), packet, true, wait) { + Err(error) + if [EBUSY, EAGAIN, ENOSPC, ENOMEM, super::super::FALLBACK].contains(&error) => {} result => result?, } } @@ -583,8 +626,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + wait: super::super::Wait, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(wait)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -660,6 +704,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2e88848b932161..b38d0d64aa5e13 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,17 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // A packet still in publication, for example waiting for capacity or + // for its pair's graph, has nothing visible to firmware. Its publisher + // owns its bounded wait and result; the pair may not even be installed. + if !packet.is_published() { + return Ok(()); + } + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c5..c80d8f3695de23 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -2,7 +2,7 @@ //! Submission admission, preparation and the firmware visibility boundary. -use super::super::{Backend, DeferredBatch, Registry}; +use super::super::{Backend, DeferredBatch, Registry, Wait, FALLBACK}; use crate::g17::{ job::Packet, queue::admission::{RenderLease, Request}, @@ -15,18 +15,185 @@ use kernel::{ }; impl Backend { + /// Render capacity is held by other live owners: admission slots, queue + /// IDs, parameter-buffer and counter identities, retired-binding room or + /// shared GPU VA, whose spares are still being built or whose siblings are + /// busy. Retry reclamation after their next release instead of failing + /// the packet; fail only once the publication's deadline has passed. + fn wait_render_capacity(&self, wait: Wait, started: &mut bool) -> Result { + use core::sync::atomic::AtomicBool; + static STARTED: AtomicBool = AtomicBool::new(false); + static EXPIRED: AtomicBool = AtomicBool::new(false); + if wait.expired()? { + if !EXPIRED.swap(true, Ordering::Relaxed) { + self.log_render_census("wait expired"); + } + return Err(resource_exhausted!("render capacity held by open owners")); + } + if !core::mem::replace(started, true) && !STARTED.swap(true, Ordering::Relaxed) { + self.log_render_census("wait started"); + } + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + Ok(()) + } + + /// A pair is borrowed, still settling, or waiting for its previous + /// scheduler's release to be consumed and parked (the event worker does + /// both). Poll briefly until the publication's deadline, then name the + /// state that kept pair `slot` busy. + /// + /// A borrower (an event pass observing the pair, a preparation or a + /// growth) returns the pair under the device mutex and then notifies + /// `changed`, usually within microseconds: wait for that notification, + /// bounded by one tick, instead of the fixed sleep. + fn wait_render_busy(&self, slot: u8, wait: Wait) -> Result { + if wait.expired()? { + let cause = { + let state = self.shared.state.lock(); + (*state).as_deref().map_or("the device stopped", |firmware| { + firmware.render_busy_cause(slot, self.owner, &self.context) + }) + }; + return Err(self.wait_expired( + ETIMEDOUT, + format_args!("a busy render pair (slot {}: {})", slot, cause), + )); + } + self.shared.queue_events(); + { + let mut state = self.shared.state.lock(); + let borrowed = (*state) + .as_deref() + .is_some_and(|firmware| firmware.queues.render.borrowed(slot)); + if borrowed { + let _ = self.shared.changed.wait_timeout(&mut state, 1); + return Ok(()); + } + } + fsleep(Delta::from_micros(200)); + Ok(()) + } + + /// Once-per-boot capacity snapshot taken with the device mutex. + fn log_render_census(&self, reason: &str) { + let state = self.shared.state.lock(); + if let Some(firmware) = (*state).as_deref() { + pr_warn!("asahi: G17 render capacity {}: {}\n", reason, firmware.capacity()); + } + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - self.render_with_lease(packet, None, deferred) + self.render_with_lease(packet, None, Wait::scheduled(), deferred) + } + + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + wait: Wait, + ) -> Result { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + let mut capacity_wait = false; + loop { + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + let Some(deadline) = wait.deadline() else { + return Err(FALLBACK); + }; + match pool.acquire_render(self.owner, Request::of(packet), deadline) { + Ok(lease) => lease, + Err(error) if error == EBUSY => { + return Err(self.wait_expired(error, "a render admission slot")) + } + Err(error) if error == ENOSPC => { + // Live owners hold every slot. Prefer unused warming, + // then an idle sibling of an owner keeping another pair. + let (reclaimed, forfeited) = { + let mut state = self.shared.state.lock(); + let queues = &mut (*state).as_deref_mut().ok_or(ENODEV)?.queues; + let reclaimed = + queues.reclaim_unused_render_spare(Some(self.owner), false)?; + let forfeited = reclaimed.is_none() + && queues.forfeit_idle_render_sibling(self.owner)?; + (reclaimed, forfeited) + }; + if reclaimed.is_none() && !forfeited { + self.wait_render_capacity(wait, &mut capacity_wait)?; + continue; + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => return Ok(lease), + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + drop(state); + self.wait_render_capacity(wait, &mut capacity_wait)?; + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + // Every remaining QID belongs to an installed graph: take an + // idle installed sibling, whose identities move with it. + let forfeited = reclaimed.is_none() + && !ready + && firmware.queues.forfeit_idle_render_sibling(self.owner)?; + drop(state); + if reclaimed.is_none() && !ready && !forfeited { + self.wait_render_capacity(wait, &mut capacity_wait)?; + continue; + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } } fn render_with_lease( &self, packet: &Arc, early: Option, + wait: Wait, deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; @@ -36,23 +203,12 @@ impl Backend { if packet.completion.status().get() != 0 { return Err(EIO); } - let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; + let admission = self.acquire_render_with_ids(packet, early, wait)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; let mut recovery_polls = 0; + let mut capacity_wait = false; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -65,7 +221,7 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = self.enter_preparation(wait)?; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -76,10 +232,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if state_polls == 1000 { - return Err(ETIMEDOUT); + if wait.expired()? { + return Err(self.wait_expired(ETIMEDOUT, "a free render work state")); } - state_polls += 1; fsleep(Delta::from_micros(state_delay)); state_delay = (state_delay * 2).min(1000); continue; @@ -88,19 +243,21 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, wait) { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(slot, wait)?; + continue; + } + // Graph or binding resources held by other owners: every failed + // build left its reservation unchanged, so retry after a release. + Err(error) if error == ENOSPC => { + self.wait_render_capacity(wait, &mut capacity_wait)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, wait) { Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { let recovering = { let state = self.shared.state.lock(); @@ -109,20 +266,22 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); + if wait.expired()? || recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(self.wait_expired(ETIMEDOUT, "GPU recovery to settle")); } recovery_polls += 1; self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(slot, wait)?; + } else if wait.expired()? { + return Err(self.wait_expired(ETIMEDOUT, "render preparation")); } } + // Unpublished: the preparation was discarded and the pair returned. + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_render_capacity(wait, &mut capacity_wait)?; + } result => return result, } } @@ -162,7 +321,17 @@ impl Backend { return true; } }; - let result = self.render_with_lease(packet, Some(admission), &mut deferred); + let result = self.render_with_lease(packet, Some(admission), Wait::Never, &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -213,6 +382,70 @@ impl Registry { } impl crate::g17::Firmware { + /// The state of pair `slot` that refuses a publication of `owner` with + /// EBUSY, for the report of a busy wait that reached its deadline. Checks + /// follow the order of the refusals in construction and preparation. + fn render_busy_cause( + &self, + slot: u8, + owner: u64, + context: &Arc, + ) -> &'static str { + use crate::g17::fw::queue::DataMaster; + let Ok(entry) = self.queues.render.entry(slot) else { + return "no queue identities"; + }; + if entry.pending_error.is_some() || entry.pending_timeout { + return "a deferred failure or timeout awaits the event worker"; + } + if entry.constructing { + return "its graph is being constructed"; + } + if entry.growing { + return "its parameter buffer is growing"; + } + if entry.preparing.is_some() { + return if entry.pending_owner.is_some_and(|other| other != owner) { + "another owner is preparing it" + } else { + "a preparation has borrowed it" + }; + } + let Some(pair) = entry.pair.as_ref() else { + return if entry.parked.is_some() { + "parked" + } else { + "no installed pair" + }; + }; + if pair.matches(owner, context).unwrap_or(false) { + let priority = context.policy().priority(); + return if pair + .owner_switch_from(context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + "the release of the scheduler it last ran is not consumed yet" + } else if !self.init.work_ready(priority, DataMaster::Fragment).unwrap_or(false) + || !self.init.work_ready(priority, DataMaster::Tiling).unwrap_or(false) + { + "the outer work ring is full" + } else if !pair.room() { + "its command ring is full" + } else { + "installed for this owner" + }; + } + if self.queues.teardown.releasing(pair.context()) { + "the release of its previous owner's scheduler is in progress" + } else if pair.owner().is_some() { + "another owner holds it" + } else if !pair.reusable().unwrap_or(false) { + "its previous owner's work or displaced graph has not settled" + } else { + "reusable" + } + } + pub(super) fn apply_render_deferred( &mut self, slot: u8, @@ -296,6 +529,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0c..eb04f803619558 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01c..f8020e8962f849 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/statistics.rs b/drivers/gpu/drm/asahi/g17/statistics.rs new file mode 100644 index 00000000000000..e5d5106e99499d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/statistics.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Observational copies of firmware statistics, independent of admission and power policy. +//! +//! The payload is deliberately unscaled until its units have been established for this +//! firmware. Readers never touch the firmware rings or take the device mutex. + +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; + +pub(crate) const WORDS: usize = 18; +const FIRST_TAG: u32 = 16; +const TAGS: usize = 15; +const ROLES: usize = 2; +pub(crate) const CAPTURE_NS: u64 = 2_000_000_000; + +struct Slot { + sequence: AtomicU64, + count: AtomicU64, + received_ns: AtomicU64, + words: [AtomicU32; WORDS], +} + +impl Slot { + const fn new() -> Self { + Self { + sequence: AtomicU64::new(0), + count: AtomicU64::new(0), + received_ns: AtomicU64::new(0), + words: [const { AtomicU32::new(0) }; WORDS], + } + } +} + +pub(crate) struct Statistics { + capture_until_ns: AtomicU64, + pub(crate) load: super::load::Load, + slots: [Slot; ROLES * TAGS], +} + +pub(crate) struct Report { + pub(crate) count: u64, + pub(crate) received_ns: u64, + pub(crate) words: [u32; WORDS], +} + +impl Statistics { + pub(crate) const fn new() -> Self { + Self { + capture_until_ns: AtomicU64::new(0), + load: super::load::Load::new(), + slots: [const { Slot::new() }; ROLES * TAGS], + } + } + + /// A diagnostic read requests a bounded full-payload capture; normal load polling does not. + pub(crate) fn request_capture(&self, now_ns: u64) { + self.capture_until_ns + .fetch_max(now_ns.saturating_add(CAPTURE_NS), Ordering::Relaxed); + } + + pub(crate) fn capture_pending(&self) -> bool { + self.capture_until_ns.load(Ordering::Relaxed) != 0 + } + + pub(crate) fn capture_enabled(&self, now_ns: u64) -> bool { + let deadline = self.capture_until_ns.load(Ordering::Relaxed); + if deadline == 0 { + return false; + } + if now_ns < deadline { + return true; + } + // A reader can renew the request concurrently with expiry. + match self.capture_until_ns.compare_exchange( + deadline, + 0, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => false, + Err(renewed) => now_ns < renewed, + } + } + + /// Called only by the serialized firmware event worker, before returning the ring slot. + pub(crate) fn record(&self, role: usize, received_ns: u64, words: &[u32; WORDS]) { + let Some(tag) = words[0].checked_sub(FIRST_TAG) else { + return; + }; + if role >= ROLES || tag as usize >= TAGS { + return; + } + self.load.record(role, received_ns, words); + let slot = &self.slots[role * TAGS + tag as usize]; + slot.sequence.fetch_add(1, Ordering::AcqRel); + for (out, value) in slot.words.iter().zip(words) { + out.store(*value, Ordering::Relaxed); + } + slot.received_ns.store(received_ns, Ordering::Relaxed); + slot.count.fetch_add(1, Ordering::Relaxed); + slot.sequence.fetch_add(1, Ordering::Release); + } + + /// A bounded, nonblocking snapshot. Contention is distinct from an unseen record. + pub(crate) fn snapshot(&self, role: usize, tag: u32) -> Option { + let tag = tag.checked_sub(FIRST_TAG)? as usize; + if role >= ROLES || tag >= TAGS { + return None; + } + let slot = &self.slots[role * TAGS + tag]; + for _ in 0..8 { + let sequence = slot.sequence.load(Ordering::Acquire); + if sequence & 1 != 0 { + continue; + } + let report = Report { + count: slot.count.load(Ordering::Relaxed), + received_ns: slot.received_ns.load(Ordering::Relaxed), + words: core::array::from_fn(|i| slot.words[i].load(Ordering::Relaxed)), + }; + fence(Ordering::Acquire); + if sequence == slot.sequence.load(Ordering::Relaxed) { + return Some(report); + } + } + None + } +} diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b1..8e15340e0c3ba4 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,34 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + + /// Entries by phase (displaced but open, draining, release published), for diagnostics. + pub(crate) fn phases(&self) -> [usize; 3] { + let mut phases = [0; 3]; + for entry in self.entries.iter() { + phases[match entry.phase { + Phase::Deferred => 0, + Phase::Draining => 1, + Phase::Released(_) => 2, + }] += 1; + } + phases + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/telemetry.rs b/drivers/gpu/drm/asahi/g17/telemetry.rs new file mode 100644 index 00000000000000..8b3d470911abb0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/telemetry.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-device sysfs lifetime. Readers retain only the atomic statistics allocation. + +use super::statistics::Statistics; +use core::ffi::{c_int, c_void}; +use kernel::{device::Device, prelude::*, sync::Arc}; + +/// Private callback contract shared with `asahi_telemetry.c`, not a userspace ABI. +#[repr(C)] +struct Export { + valid: u32, + power_known: u32, + powered: u32, + utilization: [u32; 4], + clock_hz: u32, + samples: u64, + received_ns: u64, + timestamp_ticks: u64, + power_timestamp_ticks: u64, + powered_ticks: u64, + unpowered_ticks: u64, + invalid: u64, + age_ns: u64, +} + +const _: () = { + assert!(core::mem::size_of::() == 96); + assert!(core::mem::offset_of!(Export, samples) == 32); +}; + +unsafe extern "C" { + fn asahi_telemetry_register( + dev: *mut c_void, + data: *const c_void, + snapshot: unsafe extern "C" fn(*const c_void, *mut c_void) -> c_int, + handle: *mut *mut c_void, + ) -> c_int; + fn asahi_telemetry_unregister(dev: *mut c_void, handle: *mut c_void); +} + +unsafe extern "C" fn snapshot(data: *const c_void, out: *mut c_void) -> c_int { + // SAFETY: registration retains this exact allocation until removal drains readers. + let stats = unsafe { &*data.cast::() }; + let Some(report) = stats.load.snapshot() else { + return EAGAIN.to_errno(); + }; + let current = report.current(); + // SAFETY: the C callback provides exclusive, properly aligned stack output storage. + unsafe { + out.cast::().write(Export { + valid: u32::from(current.is_some()), + power_known: u32::from(report.power_known), + powered: u32::from(report.powered), + utilization: current.unwrap_or([0; 4]), + clock_hz: super::load::CLOCK_HZ as u32, + samples: report.samples, + received_ns: report.received_ns, + timestamp_ticks: report.ticks, + power_timestamp_ticks: report.power_ticks, + powered_ticks: report.powered_ticks, + unpowered_ticks: report.unpowered_ticks, + invalid: report.invalid, + age_ns: super::now_ns().saturating_sub(report.received_ns), + }) + }; + 0 +} + +pub(super) struct Telemetry { + dev: *mut c_void, + handle: *mut c_void, + _statistics: Arc, +} + +// SAFETY: raw pointers are used only for attribute removal; readers access atomics. +unsafe impl Send for Telemetry {} +// SAFETY: registration state is immutable until its exclusive drop removes the attribute. +unsafe impl Sync for Telemetry {} + +impl Telemetry { + pub(super) fn new(dev: &Device, statistics: Arc) -> Result { + let raw = dev.as_raw().cast(); + let mut handle = core::ptr::null_mut(); + // SAFETY: the bound platform device outlives this registration; the Arc owns data. + let ret = unsafe { + asahi_telemetry_register( + raw, + (&*statistics as *const Statistics).cast(), + snapshot, + &mut handle, + ) + }; + kernel::error::to_result(ret)?; + Ok(Self { + dev: raw, + handle, + _statistics: statistics, + }) + } +} + +impl Drop for Telemetry { + fn drop(&mut self) { + // SAFETY: removal drains callback users before the Arc and its allocation are dropped. + unsafe { asahi_telemetry_unregister(self.dev, self.handle) }; + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 2266d18e9c85f2..911246738abc80 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -58,6 +58,7 @@ pub(crate) struct AsahiObject { exportable: bool, /// Whether this is a kernel-created object. kernel: bool, + _allocation: Option, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, } @@ -65,6 +66,14 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +impl AsahiObject { + pub(crate) fn classify_memory(&self, category: crate::agx_memory_stats::KernelCategory) { + if let Some(allocation) = &self._allocation { + allocation.classify_kernel(category); + } + } +} + /// Marks whether the pages of `gem` are held for the device. /// /// Populated asahi objects keep their pages referenced (device SG table, kernel vmap or a PFN @@ -211,6 +220,8 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + /// Distinguishes direct driver allocation from DRM-core imported or dumb objects. + driver_created: bool, } /// Create a new kernel-owned GEM object. @@ -250,6 +261,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, + driver_created: true, }, )?; @@ -283,6 +295,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + driver_created: true, }, )?; @@ -303,6 +316,11 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, + _allocation: if args.driver_created { + Some(crate::agx_memory_stats::Allocation::gem(size, args.kernel)?) + } else if size != 0 { + Some(crate::agx_memory_stats::Allocation::core(size)?) + } else { None }, size, flags: args.flags, exportable: args.exportable, @@ -318,6 +336,7 @@ impl DriverObject for AsahiObject { flags: 0, exportable: true, kernel: false, + driver_created: false, }) } diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f22..648d1ed9919b82 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -273,7 +273,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; diff --git a/drivers/gpu/drm/asahi/m3/agx_memory_stats.rs b/drivers/gpu/drm/asahi/m3/agx_memory_stats.rs deleted file mode 100644 index 1df3eb309dcede..00000000000000 --- a/drivers/gpu/drm/asahi/m3/agx_memory_stats.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Bounded host accounting for GEM extents and M3 coherent firmware backing. -//! -//! These are allocation extents, not resident/pinned pages or unique imported -//! physical memory. GEM construction/unwind is included. Counters are per -//! module instance and each read is observational, not a transactional snapshot. -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::prelude::*; - -struct Counters { - objects: AtomicU64, - bytes: AtomicU64, - peak_objects: AtomicU64, - peak_bytes: AtomicU64, -} -impl Counters { - const fn new() -> Self { - Self { objects: AtomicU64::new(0), bytes: AtomicU64::new(0), - peak_objects: AtomicU64::new(0), peak_bytes: AtomicU64::new(0) } - } - fn snapshot(&self) -> [u64; 4] { - let objects = self.objects.load(Ordering::Relaxed); - let bytes = self.bytes.load(Ordering::Relaxed); - [objects, bytes, self.peak_objects.load(Ordering::Relaxed).max(objects), - self.peak_bytes.load(Ordering::Relaxed).max(bytes)] - } -} -static USER_GEM: Counters = Counters::new(); -static KERNEL_GEM: Counters = Counters::new(); -static COHERENT: Counters = Counters::new(); - -/// Drop with the actual backing owner, never with an individual GEM handle or -/// its mapping. Retained fault owners therefore remain included after exit. -pub(crate) struct Allocation { - counters: &'static Counters, - bytes: u64, -} -impl Allocation { - fn new(counters: &'static Counters, size: usize) -> Result { - let bytes = size as u64; - if bytes == 0 { return Err(EINVAL); } - let previous = counters.bytes.fetch_update(Ordering::Relaxed, Ordering::Relaxed, - |current| current.checked_add(bytes)).map_err(|_| EOVERFLOW)?; - let objects = counters.objects.fetch_add(1, Ordering::Relaxed) + 1; - counters.peak_bytes.fetch_max(previous + bytes, Ordering::Relaxed); - counters.peak_objects.fetch_max(objects, Ordering::Relaxed); - Ok(Self { counters, bytes }) - } - pub(crate) fn gem(size: usize, kernel: bool) -> Result { - Self::new(if kernel { &KERNEL_GEM } else { &USER_GEM }, size) - } - pub(crate) fn coherent(size: usize) -> Result { Self::new(&COHERENT, size) } -} -impl Drop for Allocation { - fn drop(&mut self) { - self.counters.bytes.fetch_sub(self.bytes, Ordering::Relaxed); - self.counters.objects.fetch_sub(1, Ordering::Relaxed); - } -} - -pub(crate) struct View; -impl kernel::debugfs::Writer for View { - fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "version=1 unit=allocation_extent_bytes")?; - for (name, counters) in [("user_gem", &USER_GEM), ("kernel_gem", &KERNEL_GEM), - ("m3_coherent", &COHERENT)] { - let [objects, bytes, peak_objects, peak_bytes] = counters.snapshot(); - writeln!(f, "{} objects={} bytes={} peak_objects={} peak_bytes={}", - name, objects, bytes, peak_objects, peak_bytes)?; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 3d900a80fef382..4c56318d504f4c 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -77,7 +77,6 @@ mod m3_thermal_policy; mod agx_host_progress; -mod agx_memory_stats; mod agx_timing_stats; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4ea797d398a8b1..d1d9002298d703 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1859,6 +1859,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1916,7 +1947,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2498,6 +2529,12 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Application contexts currently leased, including those of closed queues + /// whose scheduler release has not completed. + pub(crate) fn execution_contexts_in_use(&self) -> usize { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() as usize + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 98006b9459c98b..cc81e78e97f355 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -19,7 +19,11 @@ pub(super) struct DriverMappings { mappings: KVec, reserved: KVec>, compute: Option, - render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], + /// Parameter-buffer views by render pair, one for each graph of this VM + /// that keeps one: a graph displaced from an open queue keeps its view + /// until that queue's scheduler release, while another queue of the same + /// VM installs its own graph on the pair. + render: [KVec; crate::hw::t8140::queues::RENDER_SLOTS], } struct PoolMappings { @@ -37,7 +41,7 @@ impl DriverMappings { self.compute.iter().flat_map(|pool| pool.mappings.iter()), self.render .iter() - .flatten() + .flat_map(|views| views.iter()) .flat_map(|pool| pool.mappings.iter()), ), ) @@ -143,7 +147,7 @@ impl Vm { mappings, reserved, compute: None, - render: core::array::from_fn(|_| None), + render: core::array::from_fn(|_| KVec::new()), }); Ok(()) } @@ -171,12 +175,13 @@ impl Vm { .as_ref()? .render .get(usize::from(slot))? - .as_ref() - .filter(|pool| pool.id == id) + .iter() + .find(|pool| pool.id == id) .map(|pool| pool.blocks) } - /// Retains a new generation after all references to a replaced pool retire. + /// Retains the view of a new graph on pair `slot`. Views of other graphs + /// of this VM on the pair stay until their own graphs are dropped. pub(crate) fn install_render_pool_mappings( &self, slot: u8, @@ -190,16 +195,20 @@ impl Vm { let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; let mut driver = lifetime.driver.lock(); let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; - let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; - if pool.is_some() { + let views = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if views.iter().any(|pool| pool.id == id) { return Err(EBUSY); } - *pool = Some(PoolMappings { - id, - blocks, - mappings, - }); - Ok(()) + // Reserve first: no refusal drops the aliases while the lock is held, + // since unmapping them takes GPUVM locks. + views.reserve(1, GFP_KERNEL)?; + views + .push_within_capacity(PoolMappings { + id, + blocks, + mappings, + }) + .map_err(|_| EIO) } /// Consumes aliases only after the exact cached owner and complete append @@ -220,8 +229,7 @@ impl Vm { let mut driver = lifetime.driver.lock(); let pool = Option::as_mut(&mut *driver) .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .and_then(Option::as_mut) - .filter(|pool| pool.id == id) + .and_then(|views| views.iter_mut().find(|pool| pool.id == id)) .ok_or(EINVAL)?; if pool.blocks != previous { return Err(EBUSY); @@ -258,8 +266,7 @@ impl Vm { let mut driver = lifetime.driver.lock(); let pool = Option::as_mut(&mut *driver) .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .and_then(Option::as_mut) - .filter(|pool| pool.id == id) + .and_then(|views| views.iter_mut().find(|pool| pool.id == id)) .ok_or(EINVAL)?; if pool.blocks != blocks || pool.mappings.len() < count { return Err(EBUSY); @@ -280,10 +287,9 @@ impl Vm { } let previous = self.lifetime.as_ref().and_then(|lifetime| { let mut driver = lifetime.driver.lock(); - Option::as_mut(&mut *driver) - .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .filter(|pool| pool.as_ref().is_some_and(|pool| pool.id == id)) - .and_then(Option::take) + let views = Option::as_mut(&mut *driver)?.render.get_mut(usize::from(slot))?; + let index = views.iter().position(|pool| pool.id == id)?; + views.remove(index).ok() }); drop(previous); } diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c00da634972e50..c1b902ea067662 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,41 @@ MODULE_IMPORT_NS("DMA_BUF"); +/** + * drm_gem_shmem_object_rss - Observe resident shmem backing without populating it + * @obj: shmem GEM object whose handle keeps it alive + * + * Count page-cache pages even after the device has released its page array. + * The object's backing file remains alive throughout the fdinfo walk. An + * imported object retains its DMA mapping instead of a private shmem file. + */ +size_t drm_gem_shmem_object_rss(struct drm_gem_object *obj) +{ + if (obj->import_attach) + return obj->size; + if (!obj->filp) + return 0; + + return min_t(size_t, obj->size, + READ_ONCE(obj->filp->f_mapping->nrpages) << PAGE_SHIFT); +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_object_rss); + +/** + * drm_gem_shmem_object_status - Report resident backing for fdinfo + * @obj: shmem GEM object whose handle keeps it alive + * + * The fdinfo walk holds the client's handle-table spinlock. Observe residency + * without taking the reservation lock or populating pages. Cached pages are + * still resident after unpinning. Reclaim may race this observational snapshot, + * as permitted by drm_gem_object_funcs.status. + */ +enum drm_gem_object_status drm_gem_shmem_object_status(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_rss(obj) ? DRM_GEM_OBJECT_RESIDENT : 0; +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_object_status); + /** * DOC: overview * diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c index 5e69ff1d1ad673..dc4e8f3284ae5e 100644 --- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c +++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -53,6 +54,8 @@ static void drm_gem_shmem_test_obj_create(struct kunit *test) shmem = drm_gem_shmem_create(drm_dev, TEST_SIZE); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, shmem); KUNIT_EXPECT_EQ(test, shmem->base.size, TEST_SIZE); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_status(&shmem->base), 0); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), 0); KUNIT_EXPECT_NOT_NULL(test, shmem->base.filp); KUNIT_EXPECT_NOT_NULL(test, shmem->base.funcs); @@ -140,9 +143,14 @@ static void drm_gem_shmem_test_pin_pages(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); + /* Keep the cached backing resident while checking the unpinned state. */ + mapping_set_unevictable(shmem->base.filp->f_mapping); ret = drm_gem_shmem_pin(shmem); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_ASSERT_NOT_NULL(test, shmem->pages); + KUNIT_EXPECT_TRUE(test, drm_gem_shmem_object_status(&shmem->base) & + DRM_GEM_OBJECT_RESIDENT); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), TEST_SIZE); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->pages_use_count), 1); for (i = 0; i < (shmem->base.size >> PAGE_SHIFT); i++) @@ -150,7 +158,12 @@ static void drm_gem_shmem_test_pin_pages(struct kunit *test) drm_gem_shmem_unpin(shmem); KUNIT_EXPECT_NULL(test, shmem->pages); + /* Releasing a device pin leaves the backing file's cached pages resident. */ + KUNIT_EXPECT_TRUE(test, drm_gem_shmem_object_status(&shmem->base) & + DRM_GEM_OBJECT_RESIDENT); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), TEST_SIZE); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->pages_use_count), 0); + mapping_clear_unevictable(shmem->base.filp->f_mapping); } /* @@ -342,6 +355,8 @@ static void drm_gem_shmem_test_purge(struct kunit *test) KUNIT_EXPECT_NULL(test, shmem->pages); KUNIT_EXPECT_NULL(test, shmem->sgt); KUNIT_EXPECT_EQ(test, shmem->madv, -1); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_rss(&shmem->base), 0); + KUNIT_EXPECT_EQ(test, drm_gem_shmem_object_status(&shmem->base), 0); } static int drm_gem_shmem_test_init(struct kunit *test) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index d25bca5352aa68..7db2bd21d85b21 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -298,6 +298,10 @@ static inline int drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct v * Driver ops */ +enum drm_gem_object_status +drm_gem_shmem_object_status(struct drm_gem_object *obj); +size_t drm_gem_shmem_object_rss(struct drm_gem_object *obj); + struct drm_gem_object * drm_gem_shmem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 4d6674f2450492..a73107c7b143aa 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -191,8 +191,7 @@ impl UnregisteredDevice { gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - - show_fdinfo: None, + show_fdinfo: T::SHOW_FDINFO, fbdev_probe: None, major: T::INFO.major, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 4971f3496b5813..eca6df3005a7fa 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -138,6 +138,16 @@ pub trait Driver { /// their `THIS_MODULE` so open files and exported DMA buffers retain the code. const MODULE: Option<&'static crate::ThisModule> = None; + /// Optional statistics for the standard DRM fdinfo interface. + /// + /// GEM drivers can opt into `drm_show_memory_stats`. Drivers may instead + /// provide engine counters or memory not represented by GEM handles. + /// The default reports only the generic DRM client identity. + const SHOW_FDINFO: Option< + unsafe extern "C" fn(*mut bindings::drm_printer, *mut bindings::drm_file), + > = None; + + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 7eabfbfdb23db0..ee4e1ec08b8018 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -526,6 +526,10 @@ pub(super) const fn create_fops( fops.read = Some(bindings::drm_read); fops.llseek = Some(bindings::noop_llseek); fops.mmap = Some(bindings::drm_gem_mmap); + #[cfg(CONFIG_PROC_FS)] + { + fops.show_fdinfo = Some(bindings::drm_show_fdinfo); + } fops.fop_flags = bindings::FOP_UNSIGNED_OFFSET; fops diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fb7e5c38e5483e..134bd0742bc336 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -157,8 +157,8 @@ impl Object { vmap: Some(bindings::drm_gem_shmem_object_vmap), vunmap: Some(bindings::drm_gem_shmem_object_vunmap), mmap: Some(bindings::drm_gem_shmem_object_mmap), - status: None, - rss: None, + status: Some(bindings::drm_gem_shmem_object_status), + rss: Some(bindings::drm_gem_shmem_object_rss), #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, diff --git a/tools/asahi/g17-load-tests.rs b/tools/asahi/g17-load-tests.rs new file mode 100644 index 00000000000000..b04b992114a25d --- /dev/null +++ b/tools/asahi/g17-load-tests.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Tests the production decoder against power transitions, inter-ring ordering and overlap. +#[path = "../../drivers/gpu/drm/asahi/g17/load.rs"] +mod load; +use load::Load; +fn record(tag: u32, timestamp: u64) -> [u32; 18] { + let mut w = [0; 18]; + w[0] = tag; + w[1] = timestamp as u32; + w[2] = (timestamp >> 32) as u32; + w +} +fn util(timestamp: u64, values: [u32; 4]) -> [u32; 18] { + let mut w = record(0x13, timestamp); + w[3..7].copy_from_slice(&values); + w +} +fn power(timestamp: u64, off: bool) -> [u32; 18] { + let mut w = record(0x19, timestamp); + w[5] = u32::from(!off); + w[6] = u32::from(off); + w +} +#[test] +fn aggregate_is_firmware_union_not_engine_sum() { + let s = Load::new(); + assert!(s.snapshot().unwrap().current().is_none()); + // Captured mixed rendering/compute: engine activity overlaps. + s.record(0, 1000, &util(100, [69, 93, 74, 95])); + assert_eq!(s.snapshot().unwrap().current(), Some([69, 93, 74, 95])); +} +#[test] +fn power_off_clears_retained_load_and_wake_requires_a_fresh_sample() { + let s = Load::new(); + s.record(0, 1, &util(100, [10, 90, 0, 90])); + s.record(1, 2, &power(110, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0; 4])); + s.record(1, 3, &power(120, false)); + assert!(s.snapshot().unwrap().current().is_none()); + s.record(0, 4, &util(130, [0, 0, 94, 94])); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 0, 94, 94])); +} +#[test] +fn late_old_ring_records_do_not_clear_newer_work() { + let s = Load::new(); + s.record(0, 10, &util(200, [0, 0, 90, 90])); + s.record(1, 11, &power(100, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 0, 90, 90])); + s.record(1, 12, &power(190, false)); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 0, 90, 90])); + s.record(1, 13, &power(150, true)); + assert_eq!(s.snapshot().unwrap().power_ticks, 190); +} +#[test] +fn invalid_samples_and_wrong_sources_cannot_replace_a_valid_sample() { + let s = Load::new(); + s.record(0, 1, &util(100, [1, 20, 0, 20])); + s.record(1, 2, &util(200, [2, 100, 0, 100])); + s.record(0, 3, &power(200, true)); + s.record(0, 4, &util(300, [1, 101, 0, 20])); + s.record(0, 5, &util(90, [1, 90, 0, 90])); + let r = s.snapshot().unwrap(); + assert_eq!(r.samples, 1); + assert_eq!(r.invalid, 1); + assert_eq!(r.current(), Some([1, 20, 0, 20])); +} +#[test] +fn unwritten_tails_and_coprocessor_busy_are_not_gpu_utilization() { + let s = Load::new(); + let mut w = util(100, [1, 20, 0, 20]); + w[7..].fill(u32::MAX); + s.record(0, 1, &w); + let mut busy = record(0x1a, 200); + busy[3] = 0; + s.record(0, 2, &busy); + s.record(1, 3, &busy); + assert_eq!(s.snapshot().unwrap().current(), Some([1, 20, 0, 20])); +} +#[test] +fn timestamps_keep_all_64_bits_and_power_residency_is_separate_from_busy() { + let s = Load::new(); + let timestamp = (1u64 << 32) + 100; + s.record(0, 1, &util(timestamp, [0, 0, 25, 25])); + s.record(1, 2, &record(0x11, 48_000_000)); + s.record(1, 3, &record(0x12, 24_000_000)); + let r = s.snapshot().unwrap(); + assert_eq!(r.ticks, timestamp); + assert_eq!(r.powered_ticks, load::CLOCK_HZ); + assert_eq!(r.unpowered_ticks, 2 * load::CLOCK_HZ); + assert_eq!(r.current(), Some([0, 0, 25, 25])); + s.record(1, 4, &record(0x12, u64::MAX)); + assert_eq!(s.snapshot().unwrap().powered_ticks, u64::MAX); +} +#[test] +fn concurrent_snapshots_are_consistent() { + let s = Load::new(); + std::thread::scope(|scope| { + scope.spawn(|| { + for i in 1..=50_000 { + let v = i % 101; + s.record(0, i as u64, &util(i as u64, [v; 4])); + } + }); + for _ in 0..50_000 { + if let Some(r) = s.snapshot() { + if r.samples != 0 { + assert_eq!(r.received_ns, r.ticks); + assert_eq!(r.utilization, [(r.ticks % 101) as u32; 4]); + } + } + } + }); + assert_eq!(s.snapshot().unwrap().samples, 50_000); +} + +#[test] +fn device_loss_stays_unavailable_even_when_queued_records_arrive() { + let s = Load::new(); + s.record(0, 1, &util(100, [10, 90, 0, 90])); + s.invalidate(); + assert!(s.snapshot().unwrap().current().is_none()); + s.record(1, 2, &power(200, true)); + s.record(0, 3, &util(300, [0, 0, 95, 95])); + assert!(s.snapshot().unwrap().current().is_none()); +} + +#[test] +fn same_state_reports_refresh_the_witness_without_resetting_the_wake_epoch() { + let s = Load::new(); + s.record(0, 1, &util(110, [0, 90, 0, 90])); + s.record(1, 2, &power(100, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0, 90, 0, 90])); + s.record(1, 3, &power(120, true)); + assert_eq!(s.snapshot().unwrap().current(), Some([0; 4])); + s.record(1, 4, &power(130, false)); + s.record(1, 5, &power(150, false)); + s.record(0, 6, &util(140, [0, 20, 0, 20])); + let r = s.snapshot().unwrap(); + assert_eq!(r.on_ticks, 130); + assert_eq!(r.power_ticks, 150); + assert_eq!(r.current(), Some([0, 20, 0, 20])); +} +#[test] +fn concurrent_repeated_power_reports_preserve_state_and_epoch_consistency() { + let s = Load::new(); + std::thread::scope(|scope| { + scope.spawn(|| { + for i in 1..=20_000u64 { + let base = i * 20; + s.record(1, base, &power(base, false)); + s.record(0, base + 1, &util(base + 1, [1, 90, 0, 90])); + for j in 2..10 { + s.record(1, base + j, &power(base + j, false)); + } + s.record(1, base + 10, &power(base + 10, true)); + for j in 11..20 { + s.record(1, base + j, &power(base + j, true)); + } + } + }); + for _ in 0..100_000 { + if let Some(r) = s.snapshot() { + if r.power_known { + assert!(r.power_ticks >= r.on_ticks); + } + if r.power_known && !r.powered { + assert_eq!(r.current(), Some([0; 4])); + } + if let Some(v) = r.current() { + assert!(v == [0; 4] || v == [1, 90, 0, 90]); + } + } + } + }); +} + +#[test] +fn unrelated_power_payload_words_do_not_change_decoding() { + let s = Load::new(); + let mut w = power(100, true); + w[5] = u32::MAX; + s.record(1, 1, &w); + assert_eq!(s.snapshot().unwrap().current(), Some([0; 4])); + assert_eq!(s.snapshot().unwrap().invalid, 0); +} diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py new file mode 100644 index 00000000000000..0430a8a0c76597 --- /dev/null +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the ordering between physical-queue owner changes and scheduler releases. + +The production release table, pair switch tracking, release readiness and compute +switch predicate are compiled against host stubs. This is not GPU execution. The +captured firmware crash (a render pair rebound in place and kicked by a same-VM +successor while its previous scheduler's release was published and the pair not +yet parked) must fail the same assertions once either rule is removed. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def ordered(source, *needles): + """Every needle occurs, in this order, inside `source`.""" + position = 0 + for needle in needles: + found = source.find(needle, position) + assert found >= 0, f'{needle!r} missing or out of order' + position = found + len(needle) + + +def check_guards(): + prepare = (ROOT / 'runtime/render/prepare.rs').read_text() + build = item(prepare, 'fn build_render_pair(') + # The in-place rebind is refused while the pair's scheduler release is pending. + ordered(build, 'if !pair.reusable()?', 'firmware.queues.teardown.releasing(pair.context())', + 'return Err(EBUSY);', 'Build::Rebind {') + rebind = item(prepare, 'fn rebind_pair(') + # A live donor that closed while the binding was prepared is caught at commit. + ordered(rebind, 'firmware.queues.teardown.releasing(pair.context())', 'return Err(EBUSY);', + 'firmware.queues.teardown.displace(', 'pair.bind_owner(') + submit = (ROOT / 'runtime/render/submit.rs').read_text() + publish = item(submit, 'pub(super) fn publish_render(') + ordered(publish, '.owner_switch_from(&packet.context)', 'releasing_scheduler(scheduler)', + 'return Err(EBUSY);', 'acquire_owner(buffer_id)', 'pair.publish(') + render = (ROOT / 'queue/render.rs').read_text() + pair_publish = item(render, 'pub(crate) fn publish(') + # The switch is recorded only after the outer publication succeeded. + ordered(pair_publish, 'let outer_next = match result', 'self.ordinal = next_ordinal;', + 'self.switch = Some((self.executed, ordinal));', 'self.executed = scheduler;', + 'self.announce_publication(') + parked = (ROOT / 'queue/render/parked.rs').read_text() + # Parking keeps the last scheduler; only a consumed release clears it. + ordered(item(parked, 'pub(crate) fn park('), 'executed: self.executed,') + ordered(item(parked, 'pub(crate) fn commit('), 'executed: self.executed,', 'switch: None,') + detach = item((ROOT / 'runtime/render/teardown.rs').read_text(), 'fn detach_context(') + ordered(detach, '.park();', 'parked.release_consumed(context.scheduler_va());') + runtime = (ROOT / 'runtime.rs').read_text() + # A compute queue does not change owner while its scheduler is released. + ordered(item(runtime, 'pub(super) fn rebind_compute('), '.filter(|queue| queue.reusable())', + 'teardown.releasing(bound)', 'continue;', 'queue.bind_owner(owner, context.clone())?;') + ordered(item(runtime, 'pub(super) fn take_over_idle_compute('), 'self.teardown.releasing(bound)', + 'queue.yield_idle_owner()?;', 'queue.bind_owner(') + ordered(item(runtime, 'pub(super) fn bind_rearmed('), 'teardown.releasing(bound)', 'return Err(EAGAIN);', + 'queue.bind_owner(') + ordered(item(runtime, 'fn publish_compute('), '.pending_switch_from()', 'queues.teardown.releasing(previous)', + 'return Err(EAGAIN);', 'queue.publish(') + # Takeover only under publication's own device mutex hold. + assert 'take_over_idle_compute' not in item(runtime, 'fn build_compute(') + for name, path in [('take_pair', 'runtime/render.rs'), ('take_render_observation', 'runtime/render/observe.rs')]: + body = item((ROOT / path).read_text(), f'fn {name}(') + ordered(body, 'entry.borrowed_switch = pair.switch_in_flight();', 'entry.pair.take()') + print('PASS guard placement: rebind decision, rebind commit, first publication, switch record, borrows, parked scheduler, compute owner changes') + + +def fixture(): + teardown = (ROOT / 'teardown.rs').read_text() + assert ('#[derive(Clone, Copy, PartialEq, Eq)]\nenum Phase {\n Deferred,\n Draining,\n' + ' Released(u32),\n}') in teardown + pending = '\n'.join(item(teardown, name) for name in [ + 'pub(crate) fn displace(', 'pub(crate) fn close(', 'fn insert(', 'pub(crate) fn releasing(', + 'pub(crate) fn releasing_scheduler(', 'fn find(']).replace('pub(crate) ', '') + pending = pending.replace('self.entries.push(entry, GFP_KERNEL)?;', 'self.entries.push(entry);') + render = (ROOT / 'queue/render.rs').read_text() + switch_methods = '\n'.join(item(render, name) for name in [ + 'pub(crate) fn owner_switch_from(', 'pub(crate) fn switch_in_flight(']).replace('pub(crate) ', '') + record = render[render.index(' let scheduler = packet.context.scheduler_va();'): + render.index(' self.executed = scheduler;') + len(' self.executed = scheduler;')] + idle = item((ROOT / 'runtime/render/teardown.rs').read_text(), + 'pub(in crate::g17::runtime) fn teardown_idle(').replace('pub(in crate::g17::runtime) ', '') + compute_switch = item((ROOT / 'queue/compute.rs').read_text(), + 'pub(crate) fn switching_from(').replace('pub(crate) ', '') + compute_idle = item((ROOT / 'runtime/teardown.rs').read_text(), 'fn compute_idle(') + return r''' +#![allow(dead_code)] +use std::sync::Arc; +type Error = i32; +type Result = std::result::Result; +struct Context { scheduler: u64, published: bool } +impl Context { + fn scheduler_va(&self) -> u64 { self.scheduler } + fn is_published(&self) -> bool { self.published } +} +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { Deferred, Draining, Released(u32) } +struct Entry { context: Arc, phase: Phase, armed: u64, announce_pending: bool } +struct Pending { entries: Vec } +impl Pending { +PENDING +} +struct Ticket { ordinal: u64 } +struct Active { ticket: Ticket } +struct Packet { context: Arc } +struct Pair { executed: u64, switch: Option<(u64, u64)>, replay: Option<()>, active: Vec, idle: bool } +impl Pair { +SWITCH + /// Host stand-in for a successful publication of `ordinal` (stand-in, not production). + fn publish(&mut self, packet: &Packet, ordinal: u64) { +RECORD + self.active.push(Active { ticket: Ticket { ordinal } }); + } + fn idle_for_context(&self, _: &Arc) -> Result { Ok(self.idle) } +} +struct RenderEntry { + context: Option>, + preparing: Option<()>, + growing: bool, + pair: Option>, + borrowed_switch: Option, +} +struct RenderState { entries: Vec> } +impl RenderState { +IDLE +} +struct Previous { context: Arc, publication: Option<(u64, u64)> } +struct Queue { previous: Option, bound: Option> } +impl Queue { +COMPUTE_SWITCH + fn idle_for_context(&self, context: &Arc) -> bool { + !self.bound.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, context)) + } +} +struct ComputeEntry { queue: Option> } +struct Service { compute: Vec> } +impl Service { +COMPUTE_IDLE +} +fn context(scheduler: u64) -> Arc { Arc::new(Context { scheduler, published: true }) } +fn main() { + let (b, c, d) = (context(0x1000), context(0x2000), context(0x3000)); + // Captured crash: GL context B closes, its release is published, then the same-VM + // successor C finds B's pairs still installed (not parked) and rebinds them in place. + let mut table = Pending { entries: Vec::new() }; + table.close(&b, 1).unwrap(); + assert!(table.releasing(&b), "armed release refuses an owner change"); + table.entries[0].phase = Phase::Released(67); + assert!(table.releasing(&b), "published, unconsumed release refuses an owner change"); + assert!(table.releasing_scheduler(0x1000) && !table.releasing_scheduler(0x2000)); + // Consumption and detach remove the entry; the pair is parked and reinstalled fresh. + table.entries.clear(); + assert!(!table.releasing(&b) && !table.releasing_scheduler(0x1000)); + // A forfeit from a live owner is not a release; its later close arms one. + table.displace(&d, false, 2).unwrap(); + assert!(!table.releasing(&d), "a live displaced owner may still lose an idle pair"); + table.close(&d, 3).unwrap(); + assert!(table.releasing(&d), "the donor's close arms its release"); + // An unpublished context has no firmware scheduler state and no release. + let quiet = Arc::new(Context { scheduler: 0x4000, published: false }); + table.close(&quiet, 4).unwrap(); + assert!(!table.releasing(&quiet)); + + // Switch tracking across an owner change of one pair. + let mut pair = Pair { executed: 0, switch: None, replay: None, active: Vec::new(), idle: true }; + assert_eq!(pair.owner_switch_from(&c), None, "a fresh graph has no previous scheduler"); + pair.publish(&Packet { context: d.clone() }, 0); + pair.active.clear(); + assert_eq!(pair.owner_switch_from(&d), None, "the same owner never switches"); + assert_eq!(pair.owner_switch_from(&c), Some(0x3000)); + pair.publish(&Packet { context: c.clone() }, 1); + assert_eq!(pair.switch_in_flight(), Some(0x3000)); + let mut render = RenderState { entries: vec![Some(RenderEntry { + context: Some(c.clone()), preparing: None, growing: false, pair: Some(Box::new(pair)), borrowed_switch: None, + })] }; + assert!(!render.teardown_idle(&d).unwrap(), "release waits for the new owner's first command"); + assert!(render.teardown_idle(&b).unwrap(), "unrelated releases are not held"); + // A borrowed pair is judged by its snapshot; a stale snapshot outside a borrow is ignored. + let borrowed = render.entries[0].as_mut().unwrap(); + let pair = borrowed.pair.take().unwrap(); + borrowed.borrowed_switch = pair.switch_in_flight(); + borrowed.preparing = Some(()); + assert!(!render.teardown_idle(&d).unwrap(), "a borrowed switching pair still holds the release"); + let entry = render.entries[0].as_mut().unwrap(); + entry.preparing = None; + entry.pair = Some(pair); + entry.pair.as_mut().unwrap().active.clear(); + assert!(render.teardown_idle(&d).unwrap(), "retirement of the first command releases the hold"); + let entry = render.entries[0].as_mut().unwrap(); + entry.pair.as_mut().unwrap().active.push(Active { ticket: Ticket { ordinal: 1 } }); + entry.pair = None; + assert!(render.teardown_idle(&d).unwrap(), "a stale snapshot without a borrow holds nothing"); + + // Compute: only a published first command after the change holds the release. + let mut service = Service { compute: vec![Some(ComputeEntry { queue: Some(Box::new(Queue { + previous: Some(Previous { context: b.clone(), publication: None }), bound: Some(c.clone()), + })) })] }; + assert!(service.compute_idle(&b), "an unpublished rebind does not hold the release"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous.as_mut().unwrap().publication = Some((1, 0)); + assert!(!service.compute_idle(&b), "compute release waits for the new owner's first command"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous = None; + assert!(service.compute_idle(&b)); + assert!(!service.compute_idle(&c), "the bound owner still needs its own retirement"); + println!("PASS release table gate, pair switch hold, borrowed snapshot and compute switch hold"); +} +'''.replace('PENDING', pending).replace('SWITCH', switch_methods, 1).replace('RECORD', record) \ + .replace('IDLE', idle, 1).replace('COMPUTE_SWITCH', compute_switch).replace('COMPUTE_IDLE', compute_idle) + + +def main(): + check_guards() + code = fixture() + mutations = { + 'fixed': (None, None, None), + 'no-release-gate': ('.is_some_and(|index| self.entries[index].phase != Phase::Deferred)', + '.is_some_and(|_| false)', 'armed release refuses an owner change'), + 'no-render-hold': ('Some(pair) => pair.switch_in_flight() == scheduler,', 'Some(_) => false,', + "release waits for the new owner's first command"), + 'no-compute-hold': ('queue.idle_for_context(context) && !queue.switching_from(context)', + 'queue.idle_for_context(context)', + "compute release waits for the new owner's first command"), + } + with tempfile.TemporaryDirectory(prefix='g17-owner-release-') as directory: + root = Path(directory) + for name, (old, new, expected) in mutations.items(): + current = code + if old is not None: + assert current.count(old) == 1, name + current = current.replace(old, new) + path = root / (name + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], + check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if old is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-pool-release-tests.py b/tools/asahi/g17-pool-release-tests.py new file mode 100644 index 00000000000000..20039f9cc753ea --- /dev/null +++ b/tools/asahi/g17-pool-release-tests.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production pool-release admission and worker wake predicates. + +Firmware and allocation boundaries are host stubs. This is not GPU execution. +Both original failure modes must fail the same state-transition assertions. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def fixture(): + runtime = (ROOT / 'runtime.rs').read_text() + compute = (ROOT / 'queue/compute.rs').read_text() + events = (ROOT / 'runtime/events.rs').read_text() + predicate = '.any(|queue| queue.retirement_polling() || queue.pool_release_pending())' + assert events.count(predicate) == 1 + methods = '\n'.join(item(runtime, name) for name in [ + 'fn compute_admission_error(', 'pub(super) fn reserve_compute(']).replace('pub(super) ', '') + pending = item(compute, 'pub(crate) fn pool_release_pending(') + return r''' +#![allow(dead_code)] +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENOSPC: Error = -28; +const EAGAIN: Error = -11; +const FIRST_COMPUTE_QID: u8 = 4; +#[derive(Clone, Copy, PartialEq, Eq)] +enum DataMaster { Compute } +#[derive(Clone, Copy)] +enum PoolPhase { Populated, ReleaseSent(u32), Vacant } +mod compute { + use super::*; + pub(super) struct Queue { + pub owner: Option, + pub pool_phase: PoolPhase, + pub retire_pending: bool, + } + impl Queue { + pub(super) fn owner(&self) -> Option { self.owner } + pub(super) fn retirement_polling(&self) -> bool { self.retire_pending } +PENDING + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Id { qid: u8 } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ComputeReservation { slot: usize, id: Id, owner: u64 } +struct ComputeEntry { reservation: ComputeReservation, queue: Option> } +struct QueueIds { free: bool, fail: Option } +impl QueueIds { + fn reserve(&mut self, _: u64, _: DataMaster, preferred: Option) -> Result { + if let Some(error) = self.fail { return Err(error); } + if !self.free { return Err(ENOSPC); } + self.free = false; + Ok(Id { qid: preferred.unwrap() }) + } + fn reserve_compute(&mut self, owner: u64, preferred: Option, _queues: usize) -> Result { + self.reserve(owner, DataMaster::Compute, preferred) + } +} +struct Admission; +impl Admission { + fn queues(&self) -> usize { 0 } +} +struct Identities { next: u64 } +impl Identities { + fn reserve(&mut self, count: u64) -> Result> { + let first = self.next; self.next += count; + Ok(first..self.next) + } +} +struct Registry { + compute: Vec>, + identities: Identities, + qids: QueueIds, + admission: Admission, +} +impl Registry { +METHODS + fn poll(&self) -> bool { + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref())PREDICATE + } +} +fn state(phase: PoolPhase, free: bool) -> Registry { + let mut entries = Vec::new(); + for slot in 0..4 { + entries.push(Some(ComputeEntry { + reservation: ComputeReservation { slot, id: Id { qid: slot as u8 }, owner: slot as u64 + 1 }, + queue: Some(Box::new(compute::Queue { owner: None, pool_phase: phase, retire_pending: false })), + })); + } + entries.push(None); + Registry { compute: entries, identities: Identities { next: 100 }, qids: QueueIds { free, fail: None }, admission: Admission } +} +fn main() { + // Captured failure: four idle released queues, full physical namespace, + // control-release cursor still awaiting observation, no retirement event. + let mut registry = state(PoolPhase::ReleaseSent(3), false); + assert!(registry.poll(), "release needs a worker wake without new GPU events"); + for _ in 0..8 { + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "pending release is temporary admission backpressure"); + assert!(registry.compute[4].is_none(), "failed admission must not install a reservation"); + assert!(!registry.qids.free); + } + // Only the firmware witness may move the retained graph to vacant. + for entry in registry.compute.iter_mut().flatten() { + entry.queue.as_mut().unwrap().pool_phase = PoolPhase::Vacant; + } + assert!(!registry.poll(), "settled idle queues must stop polling"); + assert_eq!(registry.compute_admission_error(ENOSPC), ENOSPC); + for phase in [PoolPhase::Populated, PoolPhase::Vacant] { + let mut registry = state(phase, false); + assert!(!registry.poll()); + assert_eq!(registry.reserve_compute(200), Err(ENOSPC), "real exhaustion remains terminal"); + } + // Pending releases do not throttle spare physical capacity or hide errors. + let mut registry = state(PoolPhase::ReleaseSent(u32::MAX), true); + let reservation = registry.reserve_compute(200).unwrap(); + assert_eq!(reservation.slot, 4); + assert_eq!(reservation.id.qid, FIRST_COMPUTE_QID); + assert!(registry.compute[4].is_some()); + for error in [EINVAL, -5, -12, EAGAIN] { + let mut registry = state(PoolPhase::ReleaseSent(3), false); + registry.qids.fail = Some(error); + assert_eq!(registry.reserve_compute(200), Err(error)); + } + let mut registry = state(PoolPhase::Populated, false); + registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().retire_pending = true; + assert!(registry.poll(), "existing command-retirement polling remains enabled"); + println!("PASS captured four-queue transition, settled wake removal, full/free identity capacity and error preservation"); +} +'''.replace('PENDING', pending).replace('METHODS', methods).replace('PREDICATE', predicate) + + +def main(): + code = fixture() + with tempfile.TemporaryDirectory(prefix='g17-pool-release-') as directory: + root = Path(directory) + for mutation in ['fixed', 'no-wake', 'terminal-admission']: + current = code + if mutation == 'no-wake': + current = current.replace('queue.retirement_polling() || queue.pool_release_pending()', 'queue.retirement_polling()') + elif mutation == 'terminal-admission': + current = current.replace('.map_err(|error| self.compute_admission_error(error))?', '?') + path = root / (mutation + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if mutation == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + expected = 'release needs a worker wake' if mutation == 'no-wake' else 'pending release is temporary' + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', mutation) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py new file mode 100644 index 00000000000000..f3b3518d15fa22 --- /dev/null +++ b/tools/asahi/g17-resource-tests.py @@ -0,0 +1,1122 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual G17 admission, queue-ID, and unused-graph eligibility methods. + +Run with Python and rustc; temporary host binaries are removed after the checks. +""" +from pathlib import Path +import os +import re +import subprocess +import tempfile + +DRIVER = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi' + +def admission_tests(root, source): + s=(source/'g17/queue/admission.rs').read_text();cfg=(source/'hw/t8140.rs').read_text() + def item(start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_PER_OWNER','RENDER_DEPTH']) + mmu=(source/'mmu.rs').read_text() + limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) + header='''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + type KVec = Vec; struct InFlight; + mod cfg {\n'''+constants+'}\n'+limits+'\n' + methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn forfeit_sibling','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' + fn main(){ + let mut state=State::new(); + for owner in 1..=32 { + assert!(state.refusal(owner).is_ok()); + let slot=state.acquire(owner,None).expect("existing owner fits"); + state.release(slot); + } + assert_eq!(state.active,0,"all submitted work retired"); + let retained=state.owners.iter().filter(|o|**o!=0).count(); + assert_eq!(retained,63); + assert_eq!(state.refusal(33),Err(ENOSPC)); + assert!(state.acquire(33,None).is_none()); + println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); + // Yield one unused spare for each additional logical owner. The runtime + // separately proves that these physical graphs have never been published. + for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); + assert_eq!(state.acquire(owner,None),Some(slot as u8));state.release(slot as u8); + } + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner,None).unwrap();state.release(slot);} + assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); + println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); + let mut guarded=State::new();let first=guarded.acquire(1,None).unwrap();guarded.release(first); + let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; + guarded.take_idle(spare,1); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); + guarded.release(spare as u8); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(3),Some(2)),"owner mismatch"); + guarded.quarantined |= 1u64<Vec<(u8,u64)> {(1..cfg::RENDER_SLOTS).filter(|slot|s.owners[*slot]!=0).map(|slot|(slot as u8,s.owners[slot])).collect()}; + let mut live=State::new(); + for owner in 1..=32 {let slot=live.acquire(owner,None).unwrap();live.release(slot);} + assert_eq!(live.refusal(33),Err(ENOSPC)); + let mut gifts=Vec::new(); + for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { + assert_eq!(live.refusal(owner),Err(ENOSPC)); + let (slot,donor)=live.forfeit_sibling(&candidates(&live),0,owner).expect("an owner with two pairs donates one"); + assert_eq!(live.owners[slot as usize],owner);assert_eq!(live.epochs[slot as usize],0); + assert_eq!(live.owners.iter().filter(|o|**o==donor).count(),1,"donor keeps exactly one pair"); + assert_eq!(live.acquire(owner,None),Some(slot));live.release(slot); + gifts.push((slot as usize,donor)); + } + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + assert!(live.refusal(owner).is_ok());let slot=live.acquire(owner,None).unwrap();live.release(slot); + } + println!("forfeit: all{} live logical owners render while idle siblings move; every donor keeps a pair",MAX_EXECUTION_CONTEXTS); + // A donor never receives another pair while open: neither free capacity + // that appears later nor the pair it gave away once its recipient closes. + let (given,donor)=gifts[0]; + let kept=live.owners.iter().position(|o|*o==donor).unwrap(); + let closed=live.owners.iter().position(|o|*o==u64::from(MAX_EXECUTION_CONTEXTS)).unwrap(); + assert_ne!(given,closed,"control: the released pair was not this donor's"); + live.released|=1u64<Vec {(0..cfg::RENDER_SLOTS).filter(|s|g.owners[*s]==o).collect()}; + let (a,b)=(pairs(&g,1)[0],pairs(&g,1)[1]); + g.refs[b]=1;assert_eq!(g.forfeit_sibling(&[(b as u8,1)],0,9),None,"borrowed");g.refs[b]=0; + g.active|=1u64< = std::result::Result; + const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} + #[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} + #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} + '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,name+'(')+'\n' for name in ['pub(crate) fn reserve_initial_render','pub(crate) fn reserve','pub(crate) fn reserve_compute','fn reserve_for','fn pair_identities','pub(crate) fn render_surplus','pub(crate) fn compute_available','pub(crate) fn publish','pub(crate) fn can_cancel_unpublished','pub(crate) fn cancel_unpublished'])+'}\n' + test=''' + fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} + fn main(){ + let floor=|queues:usize| MIN_RENDER_IDENTITIES.max(2*queues).min(QID_COUNT-2); + assert_eq!(render_floor(0),16);assert_eq!(render_floor(26),52);assert_eq!(render_floor(58),116);assert_eq!(render_floor(1000),QID_COUNT-2); + let mut cases=0; + for queues in [0usize,8,20,26,54,58] { + for render in [0usize,2,12,16,32,52,108,116,126] { + let mut q=fresh(); + for i in 0..render {q.reserve(i as u64+1,if i%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();} + assert_eq!(q.render_surplus(queues),render.saturating_sub(floor(queues))); + let mut compute=0; + loop { + let available=q.compute_available(queues); + let result=q.reserve_compute(1000+compute,None,queues); + assert_eq!(available,result.is_ok(),"availability predicts admission"); + if result.is_err() {assert_eq!(result,Err(ENOSPC));break;} + compute+=1; + } + assert_eq!(compute as usize,QID_COUNT-render.max(floor(queues))); + // Render keeps a pair for every open logical queue. + let mut pairs=render; + while q.reserve(5000+pairs as u64,DataMaster::Tiling,None).is_ok() {pairs+=1;} + assert!(pairs>=floor(queues),"render capacity {} below {} open queues",pairs,queues); + cases+=1; + } + } + // compute20: 20 compute clients beside a desktop of six owners with pairs. + let mut q=fresh();q.reserve_initial_render().unwrap(); + for owner in 3..15 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + for owner in 100..126 {q.reserve_compute(owner,Some(4),26).unwrap();} + assert!(q.compute_available(26),"26 compute queues while QIDs are free: no sharing"); + println!("queue IDs: {} open-queue/render/compute capacity boundaries; a render pair for every open queue; contention-free compute keeps one QID per owner",cases); + let mut q=fresh();let first=q.reserve(1,DataMaster::Tiling,None).unwrap(); + assert!(q.can_cancel_unpublished(first));q.publish(first).unwrap(); + assert!(!q.can_cancel_unpublished(first));assert_eq!(q.cancel_unpublished(first),Err(EINVAL));assert_eq!(q.leased,1); + let unused=q.reserve(2,DataMaster::Fragment,None).unwrap();q.cancel_unpublished(unused).unwrap();assert_eq!(q.leased,1); + let next=q.reserve(3,DataMaster::Fragment,None).unwrap();assert_eq!(next.qid,unused.qid); + assert!(!q.can_cancel_unpublished(unused));assert_eq!(q.cancel_unpublished(unused),Err(EINVAL)); + println!("queue IDs: published and stale identities protected"); + // Reproduce the hardware snapshot: 124 installed render identities, two + // installed compute identities, and two initial reservations fill128 IDs. + let mut q=fresh();q.reserve_initial_render().unwrap(); + assert_eq!(q.reserve_initial_render(),Err(EINVAL)); + let initial=q.initial_render; + for owner in 1000..1002 {let id=q.reserve_compute(owner,Some(4),0).unwrap();q.publish(id).unwrap();} + let mut installed=Vec::new(); + for owner in 3..127 {let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();q.publish(id).unwrap();installed.push(id);} + assert_eq!(q.leased,QID_COUNT); + for (n,old) in initial.into_iter().enumerate() { + let old=old.unwrap();let id=q.reserve_compute(2000+n as u64,None,0).unwrap(); + assert_eq!(id.qid,old.qid);assert_eq!(q.leased,QID_COUNT); + assert!(!q.can_cancel_unpublished(old));assert_eq!(q.cancel_unpublished(old),Err(EINVAL)); + q.publish(id).unwrap(); + } + assert_eq!(q.reserve_compute(3000,None,0),Err(ENOSPC)); + for id in installed {assert!(!q.can_cancel_unpublished(id));assert_eq!(q.entries[id.qid as usize].unwrap().id,id);} + // A bootstrap token that became published is never replaced. + let mut q=fresh();q.reserve_initial_render().unwrap();let protected=q.initial_render[0].unwrap();q.publish(protected).unwrap(); + for owner in 3..129 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + assert_eq!(q.leased,QID_COUNT); + assert_eq!(q.reserve_compute(1000,None,0).unwrap().qid,1); + assert_eq!(q.reserve_compute(1001,None,0),Err(ENOSPC)); + assert_eq!(q.entries[0].unwrap().id,protected); + // Compute stops at the render reserve, then replaces only the bootstrap + // identities no render pair can use; the reserve itself stays free. + for queues in [0usize,40,58] { + let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; + for owner in 3..17 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + let mut compute=0; + while q.reserve_compute(1000+compute,None,queues).is_ok() {compute+=1;} + assert_eq!(compute as usize,QID_COUNT-2-render_floor(queues)+2); + assert!(q.initial_render.iter().all(Option::is_none)); + for old in initial.into_iter().flatten() {assert_eq!(q.entries[old.qid as usize].unwrap().id.engine,DataMaster::Compute);} + assert_eq!(QID_COUNT-q.leased,render_floor(queues)-14,"render can still install a pair for every open queue"); + assert!(!q.compute_available(queues)); + } + // Ordinary free capacity leaves the initialization reservations untouched. + let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; + let id=q.reserve_compute(1000,Some(4),0).unwrap();assert_eq!(id.qid,4);assert_eq!(q.initial_render,initial); + println!("initial reservations: replaced only beyond the open-queue render floor; physical count unchanged; published and stale identities plus that floor protected"); + + } + ''' + fixture=root/'queue-id-capacity-controls.rs';fixture.write_text(header+test) + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'queue-id-capacity-controls')],check=True) + subprocess.run([str(root/'queue-id-capacity-controls')],check=True) + s=(source/'queue/render/parked.rs').read_text();guard=item(s,'pub(crate) fn can_yield_unused') + fixture=root/'unused-render-guard-controls.rs';fixture.write_text('''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + struct Context; + struct Pair {owner:Option,ordinal:u64,registered:Option,outer_started:u8,previous:Option<()>,graph_started:bool,backing:Result} + impl Pair { + fn context(&self)->&Context {&Context} + fn can_park_backing(&self,_:&Context)->Result{self.backing} + '''+guard+''' + } + fn fresh()->Pair {Pair {owner:Some(1),ordinal:0,registered:None,outer_started:0,previous:None,graph_started:false,backing:Ok(true)}} + fn main(){ + assert_eq!(fresh().can_yield_unused(),Ok(true)); + let mut p=fresh();p.owner=None;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.ordinal=1;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.registered=Some(2);assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.outer_started=4;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.graph_started=true;assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Ok(false);assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Err(-5);assert_eq!(p.can_yield_unused(),Err(-5)); + println!("unused graph: eight eligibility/error controls protect every publication boundary"); + } + ''') + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) + subprocess.run([str(root/'unused-render-guard-controls')],check=True) + +def compute_takeover_tests(root, source): + s=(source/'g17/queue/compute.rs').read_text() + def item(start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + methods='\n'.join(item('pub(crate) fn '+name+'(') for name in ['release_owner','idle_owned','yield_idle_owner','reusable']) + fixture=root/'compute-takeover-controls.rs' + fixture.write_text('''#![allow(dead_code)] +type Result = std::result::Result; +const EBUSY:i32=-16;const EIO:i32=-5; +fn now_ns()->u64 {42} +#[derive(Clone,Copy,PartialEq,Eq,Debug)] enum PoolPhase {Populated,ReleaseSent(u32),Vacant} +struct Fifo {len:usize} +#[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Binding {context:u32} +struct Previous {binding:Option,context:u32,publication:Option<(u64,u64)>} +struct Graph {owner:u32} +impl Graph {fn set_owner(&mut self,context:&u32)->Result {self.owner=*context;Ok(())}} +struct Queue {owner:Option,released:bool,installed:bool,growing:bool,pool_phase:PoolPhase, + needs_fresh_backing:bool,exit:Option<()>,quarantined:bool,awaiting_witness:bool,active:Fifo,replays:Fifo, + previous:Option,binding:Option,retirement_ready:bool,released_at:u64, + retired_by_teardown:bool,graph:Graph} +impl Queue { +'''+methods+''' +} +fn idle()->Queue {Queue {owner:Some(1),released:false,installed:true,growing:false,pool_phase:PoolPhase::Populated, + needs_fresh_backing:false,exit:None,quarantined:false,awaiting_witness:false,active:Fifo {len:0},replays:Fifo {len:0}, + previous:None,binding:Some(Binding {context:1}),retirement_ready:true,released_at:0,retired_by_teardown:false,graph:Graph {owner:1}}} +fn main(){ + let mut q=idle();assert!(q.idle_owned());assert!(!q.reusable()); + q.yield_idle_owner().unwrap(); + assert!(q.reusable(),"a yielded queue is exactly a closed owner's reusable queue"); + assert_eq!((q.owner,q.released,q.retirement_ready,q.released_at),(None,true,true,42)); + assert_eq!(q.binding,Some(Binding {context:1}),"the old binding stays until bind_owner supersedes it"); + // A queue bound at submission whose packet has not published is also idle. + let mut q=idle();q.retirement_ready=false;assert!(q.idle_owned());q.yield_idle_owner().unwrap();assert!(q.reusable()); + let cases:[(&str,fn(&mut Queue));13]=[ + ("released",|q|q.released=true),("never installed",|q|q.installed=false),("growing",|q|q.growing=true), + ("pool release",|q|q.pool_phase=PoolPhase::ReleaseSent(1)),("vacant",|q|q.pool_phase=PoolPhase::Vacant), + ("fresh backing",|q|q.needs_fresh_backing=true),("exit",|q|q.exit=Some(())),("quarantined",|q|q.quarantined=true), + ("witness",|q|q.awaiting_witness=true),("active",|q|q.active.len=1),("replay",|q|q.replays.len=1), + ("superseded",|q|q.previous=Some(Previous {binding:None,context:0,publication:None})),("detached",|q|q.binding=None), + ]; + for (name,change) in cases { + let mut q=idle();change(&mut q); + assert!(!q.idle_owned(),"{} must not be taken over",name); + let before=(q.owner,q.released,q.retirement_ready,q.released_at); + assert_eq!(q.yield_idle_owner(),Err(EBUSY)); + assert_eq!((q.owner,q.released,q.retirement_ready,q.released_at),before,"{}: refusal changes nothing",name); + } + let mut q=idle();q.owner=None;assert!(!q.idle_owned(),"no owner"); + println!("compute takeover: idle live queue yields into reusable state; 14 busy/transitional states refused unchanged"); + // An owner that rebound a detached queue and closed without publishing. + let mut q=idle();q.retirement_ready=false;assert_eq!(q.release_owner(),Ok(false)); + assert_eq!(q.owner,None);assert!(q.reusable(),"QID returns to service at close"); + // Controls: retained work, a superseded binding and a pending replay keep the owner. + let mut q=idle();q.retirement_ready=false;q.active.len=1;q.release_owner().unwrap();assert_eq!(q.owner,Some(1)); + let mut q=idle();q.retirement_ready=false;q.replays.len=1;q.release_owner().unwrap();assert_eq!(q.owner,Some(1)); + let mut q=idle();q.retirement_ready=false;q.exit=Some(());q.release_owner().unwrap();assert_eq!(q.owner,Some(1)); + let mut q=idle();q.retirement_ready=false;q.binding=Some(Binding {context:2}); + q.previous=Some(Previous {binding:Some(Binding {context:1}),context:1,publication:None}); + q.release_owner().unwrap();assert_eq!((q.owner,q.binding,q.graph.owner),(None,Some(Binding {context:1}),1),"unpublished successor reverts"); + let mut q=idle();q.retirement_ready=false; + q.previous=Some(Previous {binding:None,context:1,publication:Some((1,1))}); + q.release_owner().unwrap();assert_eq!(q.owner,Some(1),"published successor waits for its witness"); + let mut q=idle();q.installed=false;q.retirement_ready=false;assert_eq!(q.release_owner(),Ok(true),"never installed: cancel QID"); + println!("compute close: unpublished rebinding returns its QID; work, replay, exit and superseded controls retained"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'compute-takeover-controls')],check=True) + subprocess.run([str(root/'compute-takeover-controls')],check=True) + +def lower_window_gate_tests(root, source): + s=(source/'g17/freelist.rs').read_text() + start=s.index('static LOWER_FREED');end=s.index('pub(crate) struct Pages {') + gate=s[start:end] + fixture=root/'lower-window-gate-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::sync::atomic::{AtomicU64, Ordering}; +type Result = std::result::Result; +const ENOSPC:i32=-28;const ENOMEM:i32=-12; +static CLOCK: AtomicU64 = AtomicU64::new(1_000_000_000); +fn now_ns()->u64 {CLOCK.load(Ordering::Relaxed)} +mod freelist { +use super::*; +'''+gate+''' +pub(crate) fn freed() {LOWER_FREED.fetch_add(1, Ordering::AcqRel);} +pub(crate) fn place(size: usize, f: impl FnOnce() -> Result) -> Result {place_lower(size, f)} +} +use freelist::*; +fn attempt(size:usize, result:Result, calls:&mut u32)->Result {place(size,||{*calls+=1;result})} +fn main(){ + let mut calls=0; + assert_eq!(attempt(100,Ok(1),&mut calls),Ok(1));assert_eq!(calls,1);assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,2);assert!(lower_pressure()); + for _ in 0..1000 {assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));} + assert_eq!(attempt(200,Ok(1),&mut calls),Err(ENOSPC)); + assert_eq!(calls,2,"a refused size is not placed again while nothing was freed"); + assert_eq!(attempt(50,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"a smaller size still probes"); + assert_eq!(attempt(60,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"the smaller refusal now gates larger sizes"); + freed();assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,4,"freed backing lets a waiter retry"); + // One probe per interval across all callers, then the gate closes again. + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Ok(7),&mut calls),Ok(7));assert_eq!(calls,6);assert!(!lower_pressure(),"a fitting placement clears pressure"); + assert_eq!(attempt(100,Err(ENOMEM),&mut calls),Err(ENOMEM));assert!(!lower_pressure(),"only window refusals gate"); + println!("lower window: refused free-list placements are not repeated until backing is freed or one probe per 100 ms"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'lower-window-gate-controls')],check=True) + subprocess.run([str(root/'lower-window-gate-controls')],check=True) + +def watchdog_publication_guard_tests(source): + # A packet still in publication (capacity or graph waits) must never reach + # pair lookups or a collector allocation from its watchdog: either error + # stops the runtime for every client. + for path, name in [('g17/runtime/render/settle.rs', 'pub(in crate::g17) fn timeout_render('), + ('g17/runtime/compute_control.rs', 'pub(super) fn timeout_compute(')]: + text = (source / path).read_text() + at = text.index(name) + body = text[at:text.index('\n }\n', at)] + guard = body.index('if !packet.is_published() {\n return Ok(());') + collector = body.index('let Ok(mut deferred) = DeferredBatch::worker() else {') + assert guard < collector and 'DeferredBatch::worker()?' not in body, name + print('watchdog: unpublished packets and collector allocation failure never stop the runtime') + +def shared_alias_tests(root, source): + s=(source/'g17/object.rs').read_text() + at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + method=s[at:end] + fixture=root/'shared-alias-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::{cell::RefCell,rc::Rc,ops::Range}; +type Result = std::result::Result;type Error=i32; +const EINVAL:i32=-22;const EOVERFLOW:i32=-75;const ENOSPC:i32=-28;const EIO:i32=-5; +mod hw {pub mod t8140 {pub mod dynamic {pub const LOWER:std::ops::Range=0..256;}}} +mod mmu {pub const UAT_PGSZ:usize=16;pub type Prot=u8;pub type Vm=super::Vm;pub type KernelMapping=super::Mapping;} +#[derive(Clone,Copy)] enum CpuMap {WriteCombined} +#[derive(Clone)] struct Region {vm:usize,start:u64,size:u64,object:u64} +#[derive(Default)] struct State {regions:Vec,allocations:u64,attempts:usize,fault:Option} +struct Vm {id:usize,state:Rc>} +struct Uat {lower:Vm} +impl Uat {fn kernel_lower_vm(&self)->&Vm {&self.lower}} +struct Device {state:Rc>} +struct Mapping {vm:usize,address:u64,size:u64,object:u64,state:Rc>} +impl Mapping {fn iova(&self)->u64 {self.address}} +impl Drop for Mapping {fn drop(&mut self) { + let mut state=self.state.borrow_mut(); + if self.vm==0 {assert!(!state.regions.iter().any(|r|r.vm!=0&&r.object==self.object&&r.start==self.address),"aliases must retire before canonical reservation");} + let index=state.regions.iter().position(|r|r.vm==self.vm&&r.start==self.address&&r.object==self.object).unwrap(); + state.regions.remove(index); +}} +fn overlaps(s:&State,vm:usize,start:u64,size:u64)->bool {s.regions.iter().any(|r|r.vm==vm&&r.startResult { + let mut state=vm.state.borrow_mut(); + if state.fault==Some(vm.id) {return Err(EIO)} + if overlaps(&state,vm.id,address,size) {return Err(ENOSPC)} + state.regions.push(Region {vm:vm.id,start:address,size,object}); + Ok(Mapping {vm:vm.id,address,size,object,state:vm.state.clone()}) +} +struct ObjectRef {id:u64,size:usize} +impl ObjectRef { + fn map_mode(&self,vm:&Vm,range:Range,align:u64,guard:bool,lowest:bool)->Result { + if vm.id==0 {assert!(guard,"canonical guard must be preserved");vm.state.borrow_mut().attempts+=1;} + let length=self.size as u64+if guard {16}else{0}; + let mut used:Vec<_>=vm.state.borrow().regions.iter().filter(|r|r.vm==vm.id).cloned().collect();used.sort_by_key(|r|r.start); + let mut cursor=range.start;let mut holes=Vec::new(); + for r in used { + if r.start>cursor {holes.push(cursor..r.start.min(range.end));} + cursor=cursor.max(r.start+r.size); + } + if cursorclaim(vm,address,length,self.id),None=>Err(ENOSPC)} + } + fn map_into_range(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:bool)->Result { + self.map_mode(vm,range,align,guard,false) + } +} +struct KernelObject {obj:ObjectRef,mapping:Option} +impl KernelObject { + fn backing(dev:&Device,size:usize,_cpu:CpuMap)->Result {let mut s=dev.state.borrow_mut();s.allocations+=1;Ok(Self {obj:ObjectRef {id:s.allocations,size},mapping:None})} + fn map_alias(&self,vm:&Vm,address:u64,_prot:u8)->Result {claim(vm,address,self.obj.size as u64,self.obj.id)} + fn alias_in_lowest(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:usize)->Result {self.obj.map_mode(vm,range,align,guard!=0,true)} +} +struct Allocator<'a> {dev:&'a Device,uat:&'a Uat} +impl Allocator<'_> { +'''+method+''' +} +fn case(obstacles:&[(usize,u64,u64)],fault:Option,size:usize,expected:std::result::Result,attempts:usize) { + let state=Rc::new(RefCell::new(State::default())); + for (vm,start,len) in obstacles {state.borrow_mut().regions.push(Region {vm:*vm,start:*start,size:*len,object:0});} + state.borrow_mut().fault=fault; + let dev=Device {state:state.clone()};let uat=Uat {lower:Vm {id:0,state:state.clone()}}; + let first=Vm {id:1,state:state.clone()};let second=Vm {id:2,state:state.clone()}; + let alloc=Allocator {dev:&dev,uat:&uat}; + let result=alloc.lower_with_aliases(size,16,3,CpuMap::WriteCombined,[&first,&second]); + match (expected,result) { + (Ok(address),Ok((object,aliases)))=>{ + assert_eq!(object.mapping.as_ref().unwrap().iova(),address); + assert!(aliases.iter().all(|m|m.iova()==address&&m.object==object.obj.id)); + assert_eq!(state.borrow().regions.iter().filter(|r|r.object!=0).count(),3); + drop(aliases);drop(object); + }, + (Err(expected),Err(actual))=>assert_eq!(actual,expected), + _=>panic!("unexpected common-address placement result"), + } + assert_eq!(state.borrow().regions.len(),obstacles.len(),"failed attempts leak no reservations or aliases"); + assert_eq!(state.borrow().allocations,if size==0||size%16!=0 {0}else{1},"all retries reuse one backing"); + assert_eq!(state.borrow().attempts,attempts); +} +fn exhaustive_common_holes() { + for canonical in 0u32..64 {for global in 0u32..64 {for client in 0u32..64 { + let masks=[canonical,global,client]; + let available=(0..4).any(|page|canonical&(7<{ + assert!(available,"allocator found a false common hole"); + let address=object.mapping.as_ref().unwrap().iova();assert_eq!(address%16,0);let page=address/16;assert!(page<4); + assert!(canonical&(7<{assert_eq!(error,ENOSPC);assert!(!available,"allocator skipped a common hole");}, + } + assert_eq!(state.borrow().allocations,1);assert_eq!(state.borrow().regions.len(),original); + }}} + println!("shared aliases: all262144 three-VM occupancy combinations match the independent common-hole oracle"); +} +fn main() { + case(&[],None,64,Ok(0),1); + case(&[(1,0,16)],None,64,Ok(16),2); + case(&[(2,0,16)],None,64,Ok(16),2); + case(&[(1,0,16),(2,64,16)],None,64,Ok(80),3); + case(&[(0,0,16)],None,64,Ok(16),1); + case(&[(2,0,256)],None,64,Err(ENOSPC),1); + case(&[(2,0,176)],None,64,Ok(176),2); + case(&[],Some(1),64,Err(EIO),1); + case(&[],Some(2),64,Err(EIO),1); + case(&[],Some(0),64,Err(EIO),1); + case(&[],None,256,Err(ENOSPC),1); + case(&[],None,0,Err(EINVAL),0); + case(&[],None,15,Err(EINVAL),0); + // Best-fit initially selects the small late hole, occupied in the client. + // The larger earlier canonical hole must remain eligible after that collision. + case(&[(0,128,16),(2,144,112)],None,64,Ok(0),2); + case(&[(1,0,256)],None,64,Err(ENOSPC),1); + case(&[(0,128,128),(2,0,128)],None,64,Err(ENOSPC),2); + println!("shared aliases: 16 best-fit, collision, rollback, lifetime, backing-reuse, guard and error controls pass"); + exhaustive_common_holes(); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'shared-alias-controls')],check=True) + subprocess.run([str(root/'shared-alias-controls')],check=True) + +def compute_admission_tests(root, source): + """held40 on hardware: 40 held clients and 12 fresh ones beside a desktop, + with every QID leased (108 render identities, 20 compute queues). Render + surplus beyond the open-queue floor cannot become compute capacity, so a + client without a queue must be able to take an idle one over.""" + g17 = source / "g17" + def item(s,start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + s=(g17/'kick.rs').read_text() + fw=(g17/'fw/kick.rs').read_text() + runtime=(g17/'runtime.rs').read_text() + constants=re.search(r'const QID_MAX: [^;]+;',fw).group(0)+'\n'+'\n'.join(re.search(r'const '+name+r': [^;]+;',s).group(0) for name in ['QID_COUNT','MIN_RENDER_IDENTITIES']) + constants+='\n'+item(s,'fn render_floor(') + header='''#![allow(dead_code, unused_macros)] +type Result = std::result::Result; +const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; +macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } +#[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} +#[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} +#[derive(Clone,Copy)] struct Identity {id:Id,published:bool} +'''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,name+'(')+'\n' for name in ['pub(crate) fn reserve','pub(crate) fn reserve_compute','fn reserve_for','fn pair_identities','pub(crate) fn render_surplus','pub(crate) fn compute_available','pub(crate) fn publish','pub(crate) fn can_cancel_unpublished'])+'}\n'+item(runtime,'fn takeover_allowed(')+'\n' + test=''' +fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} +// Physical compute queues: their bound owner, or none once released. +struct Queues {owners:Vec>,takeovers:usize} +// One publication: the locked rebind or takeover, then the off-lock attempt, +// as compute_packet alternates them. True once the client has a queue. +fn publish(q:&mut QueueIds,queues:&mut Queues,client:u64,open:usize)->bool { + let mut contended=false; + for _ in 0..3 { + if queues.owners.contains(&Some(client)) {return true;} + if let Some(slot)=queues.owners.iter_mut().find(|owner|owner.is_none()) {*slot=Some(client);return true;} + if takeover_allowed(q.compute_available(open),q.render_surplus(open),contended) { + // Every other owner's queue is idle between its iterations. + if let Some(slot)=queues.owners.iter_mut().find(|owner|owner.is_some_and(|o|o!=client)) { + *slot=Some(client);queues.takeovers+=1;return true; + } + } + match q.reserve_compute(10_000+client,None,open) { + Ok(id)=>{q.publish(id).unwrap();queues.owners.push(Some(client));return true;} + Err(error)=>{assert_eq!(error,ENOSPC);contended=true;} + } + } + false +} +fn main(){ + let mut q=fresh(); + for owner in 1..=108u64 { + let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap(); + q.publish(id).unwrap(); + } + let mut queues=Queues {owners:Vec::new(),takeovers:0}; + for n in 0..20u64 {let id=q.reserve_compute(1000+n,None,0).unwrap();q.publish(id).unwrap();queues.owners.push(None);} + assert_eq!(q.leased,QID_COUNT); + let desktop=11; + let mut open=desktop+40; + assert!(!q.compute_available(open)&&q.render_surplus(open)>=2,"snapshot: no fresh QID, render surplus beyond the floor"); + for round in 0..10 { + for client in 1..=40u64 { + assert!(publish(&mut q,&mut queues,client,open),"held client {} could not bind in round {}",client,round); + } + } + for fresh_client in 41..=52u64 { + open+=1; + for _ in 0..3 { + assert!(publish(&mut q,&mut queues,fresh_client,open),"fresh client {} could not bind",fresh_client); + for client in 1..=40u64 {assert!(publish(&mut q,&mut queues,client,open),"held client {} could not bind again",client);} + } + for owner in queues.owners.iter_mut().filter(|owner|**owner==Some(fresh_client)) {*owner=None;} + open-=1; + } + assert_eq!(queues.owners.len(),20,"no QID changed engine"); + assert!(queues.takeovers>0); + let held=queues.takeovers; + // render50: 47 clients fill the 58 contexts, so the render floor leaves no + // surplus. Eight fail render capacity and close; the floor drops and the + // surplus returns while the 39 others still need compute queues. + let mut q=fresh(); + for owner in 1..=108u64 { + let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap(); + q.publish(id).unwrap(); + } + let mut queues=Queues {owners:Vec::new(),takeovers:0}; + for n in 0..20u64 {let id=q.reserve_compute(1000+n,None,0).unwrap();q.publish(id).unwrap();queues.owners.push(None);} + let mut open=desktop+47; + assert!(q.render_surplus(open)<2,"58 open queues: the floor covers every render identity"); + for client in 1..=47u64 {assert!(publish(&mut q,&mut queues,client,open),"render50 client {} could not bind",client);} + for client in 40..=47u64 {for owner in queues.owners.iter_mut().filter(|owner|**owner==Some(client)) {*owner=None;}} + open-=8; + assert!(q.render_surplus(open)>=2&&!q.compute_available(open),"closes leave render surplus and no fresh QID"); + // Every survivor publishes while each other one holds a queue. + for round in 0..5 { + for client in 1..=39u64 { + assert!(publish(&mut q,&mut queues,client,open),"render50 survivor {} could not bind in round {}",client,round); + } + } + println!("compute admission: held40 snapshot, 40 held and 12 fresh clients share 20 queues by takeover ({} takeovers); render50 survivors keep binding after closes restore render surplus ({} takeovers); none starves",held,queues.takeovers); +} +''' + code=header+test + for name,old,new in [('fixed',None,None),('render-surplus-gate', + 'contended || (!compute_available && render_surplus < 2)', + '(!compute_available && render_surplus < 2)')]: + current=code + if old is not None: + assert current.count(old)==1, name + current=current.replace(old,new) + fixture=root/('compute-admission-'+name+'.rs');fixture.write_text(current) + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(fixture.with_suffix(''))],check=True) + result=subprocess.run([str(fixture.with_suffix(''))],capture_output=True,text=True) + if old is None: + assert result.returncode==0,result.stderr + print(result.stdout.strip()) + else: + assert result.returncode!=0 and 'could not bind' in result.stderr,result.stderr + print('PASS negative control:',name) + + +def forfeit_donor_tests(root, source): + """held40 with the compositor as a forfeit donor. Under full slot pressure + a refused held client takes the compositor's least recently used sibling; + the compositor keeps rendering two overlapping commands every frame, while + the recipient holds the pair and after it closes. Runs the actual admission + State; the registry side is modelled from the rules checked below.""" + g17 = source / 'g17' + def item(s, start): + at = s.index(start); begin = s.index('{', at); depth = 1; end = begin + 1 + while depth: + depth += (s[end] == '{') - (s[end] == '}'); end += 1 + return s[at:end] + def body(s, start): + return item(s, start) + # The modelled rules, anchored in the driver source. + pair = (g17 / 'queue/render.rs').read_text() + assert re.search(r'if foreign \{\s*self\.displaced = Some\(retired\);', + body(pair, 'pub(crate) fn bind_owner(')), \ + 'a foreign rebind displaces the old graph: revisit this model' + teardown = (g17 / 'runtime/render/teardown.rs').read_text() + assert re.search(r'if self\.retired\[index\]\.buffer_id\(\)\.is_some\(\) \{\s*index \+= 1;\s*continue;', + body(teardown, 'pub(in crate::g17::runtime) fn collect_reclaimable(')), \ + 'a displaced graph with memory stays until its context is detached: revisit this model' + assert 'pairs: [Option; 2],' in (g17 / 'context/identity.rs').read_text(), \ + 'a context has two render work-state channels: revisit this model' + + s = (g17 / 'queue/admission.rs').read_text(); cfg = (source / 'hw/t8140.rs').read_text() + constants = '\n'.join(re.search(r'pub\(crate\) const ' + name + r': [^;]+;', cfg).group(0) + for name in ['RENDER_SLOTS', 'RENDER_PAIRS_PER_OWNER', 'RENDER_DEPTH']) + mmu = (source / 'mmu.rs').read_text() + limits = '\n'.join(re.search(r'const ' + name + r': [^;]+;', mmu).group(0) + for name in ['EXECUTION_CONTEXT_MASK', 'MAX_EXECUTION_CONTEXTS']) + header = '''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + type KVec = Vec; struct InFlight; + mod cfg {\n''' + constants + '}\n' + limits + '\n' + methods = ['const fn new', 'fn exhausted', 'fn refusal', 'fn advance', 'fn take_idle', 'fn acquire', + 'fn claim_sibling', 'fn forfeit_sibling', 'fn release'] + code = header + item(s, 'struct State') + '\nimpl State {\n' + \ + re.search(r' const MASK: u64 = .*;', s).group(0) + '\n' + \ + '\n'.join(item(s, n) for n in methods) + '\n}\n' + ''' + use std::collections::{HashMap,HashSet}; + // Registry side of a publication: the graph installed on each pair, + // displaced graphs of open owners (retained for their scheduler until + // that owner's own release), and each context's two render work-state + // channels, bound to the pairs it published on. + struct Model {graph:[u64;64],retained:Vec<(u64,usize)>,channels:HashMap>,open:HashSet} + impl Model { + fn publish(&mut self,owner:u64,slot:usize) { + if self.graph[slot]!=owner { + if self.retained.contains(&(owner,slot)) { + panic!("owner {} would install a second graph on pair {} while its displaced graph there is retained for its scheduler",owner,slot); + } + let previous=self.graph[slot]; + if previous!=0 && self.open.contains(&previous) {self.retained.push((previous,slot));} + self.graph[slot]=owner; + } + let channels=self.channels.entry(owner).or_default(); + if !channels.contains(&slot) { + assert!(channels.len()<2,"owner {} has no work-state channel for pair {}: it published on {:?}",owner,slot,channels); + channels.push(slot); + } + } + fn close(&mut self,owner:u64) {self.open.remove(&owner);self.retained.retain(|(vm,_)|*vm!=owner);} + } + // Pool::release_owner on the State: logical close of every slot of `owner`. + fn close(s:&mut State,m:&mut Model,owner:u64) { + for slot in 0..cfg::RENDER_SLOTS {if s.owners[slot]==owner {s.released|=1u64<u8 { + let slot=s.acquire(owner,None).unwrap_or_else(||panic!("owner {} found no pair",owner)); + m.publish(owner,slot as usize); + slot + } + // Forfeit candidates: idle installed pairs of open owners with two installed pairs. + fn candidates(s:&State,m:&Model)->Vec<(u8,u64)> { + (1..cfg::RENDER_SLOTS).filter(|slot|{ + let donor=s.owners[*slot]; + donor!=0 && m.open.contains(&donor) && m.graph[*slot]==donor && s.refs[*slot]==0 + && (1..cfg::RENDER_SLOTS).filter(|other|s.owners[*other]==donor && m.graph[*other]==donor).count()>=2 + }).map(|slot|(slot as u8,s.owners[slot])).collect() + } + fn main(){ + const COMPOSITOR:u64=1; + let mut s=State::new(); + let mut m=Model{graph:[0;64],retained:Vec::new(),channels:HashMap::new(),open:HashSet::new()}; + // The desktop and earlier clients hold every pair: 31 owners render on two, one on one. + for owner in 1..=32 { + m.open.insert(owner); + let a=render(&mut s,&mut m,owner); + if owner<32 {let b=render(&mut s,&mut m,owner);assert_ne!(a,b);s.release(b);} + s.release(a); + } + assert_eq!(s.owners[1..].iter().filter(|o|**o!=0).count(),63); + // Everyone but the compositor renders on both pairs again; the compositor + // keeps drawing on one, so its sibling is the least recently used pair. + for owner in 2..=31 {let a=render(&mut s,&mut m,owner);let b=render(&mut s,&mut m,owner);s.release(b);s.release(a);} + let first=render(&mut s,&mut m,COMPOSITOR);s.release(first); + let mine:Vec=(1..cfg::RENDER_SLOTS).filter(|slot|s.owners[*slot]==COMPOSITOR).collect(); + assert_eq!(mine.len(),2); + // held40: refused clients take idle siblings of open owners. + let mut recipients=Vec::new(); + for owner in 100..140 { + m.open.insert(owner); + assert_eq!(s.refusal(owner),Err(ENOSPC)); + if let Some((slot,donor))=s.forfeit_sibling(&candidates(&s,&m),0,owner) {recipients.push((owner,slot as usize,donor));} + } + let &(holder,given,_)=recipients.iter().find(|r|r.2==COMPOSITOR).expect("the compositor donates its least recently used sibling"); + let kept=*mine.iter().find(|slot|**slot!=given).unwrap(); + for round in 0..10 { + for &(owner,_,_) in &recipients {let slot=render(&mut s,&mut m,owner);s.release(slot);} + // The compositor's recipient draws most recently of all. + let slot=render(&mut s,&mut m,holder);s.release(slot); + // Two overlapping compositor commands per frame. + let x=render(&mut s,&mut m,COMPOSITOR);let y=render(&mut s,&mut m,COMPOSITOR); + assert!(x as usize!=given && y as usize!=given,"round {}: the compositor uses its own pair, never the one its recipient holds",round); + s.release(y);s.release(x); + } + assert_eq!(s.owners[given],holder); + assert!(m.retained.contains(&(COMPOSITOR,given)),"control: the recipient displaced the compositor's graph"); + // The held clients exit; the compositor keeps drawing for 20 frames, + // beside fresh clients that reuse the released pairs. + for &(owner,_,_) in &recipients {close(&mut s,&mut m,owner);} + for owner in 100..140 {m.open.remove(&owner);} + for round in 0..20 { + let x=render(&mut s,&mut m,COMPOSITOR);let y=render(&mut s,&mut m,COMPOSITOR); + assert_eq!((x as usize,y as usize),(kept,kept),"round {}: the compositor stays on the pair it kept",round); + if round<12 { + let fresh=200+round as u64;m.open.insert(fresh); + let slot=render(&mut s,&mut m,fresh);s.release(slot);close(&mut s,&mut m,fresh); + } + s.release(y);s.release(x); + } + assert_eq!(s.owners.iter().filter(|o|**o==COMPOSITOR).count(),1,"a donor receives no other pair while open"); + // Once the compositor itself closes, its record clears. + close(&mut s,&mut m,COMPOSITOR); + assert!(!s.forfeited.contains(&COMPOSITOR)); + println!("forfeit donor: the compositor donates its idle sibling under held40, renders every frame on the pair it kept while the recipient holds the other and after it exits, and never takes back a pair its displaced graph still holds"); + } + ''' + # The donor rule before this fix: a donor takes back the pair it gave away + # once free. Its next command there waits for its own retained graph. + reclaim = [ + (' forfeited: [u64; cfg::RENDER_SLOTS],\n}', + ' forfeited: [u64; cfg::RENDER_SLOTS],\n forfeited_slots: [u8; cfg::RENDER_SLOTS],\n}'), + (' forfeited: [0; cfg::RENDER_SLOTS],\n', + ' forfeited: [0; cfg::RENDER_SLOTS],\n forfeited_slots: [0; cfg::RENDER_SLOTS],\n'), + (''' if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER + || self.forfeited.contains(&owner) + { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; +''', ''' if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + if let Some(record) = self.forfeited.iter().position(|id| *id == owner) { + let slot = usize::from(self.forfeited_slots[record]); + if eligible(slot) + && self.owners[slot] != owner + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + { + self.forfeited[record] = 0; + self.forfeited_slots[record] = 0; + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.epochs[slot] = 0; + } + return; + } +'''), + (''' if !self.forfeited.contains(&donor) { + self.forfeited[record] = donor; + } +''', ''' let record = self + .forfeited + .iter() + .position(|value| *value == donor) + .unwrap_or(record); + self.forfeited[record] = donor; + self.forfeited_slots[record] = slot; +'''), + ] + # Without the donor rule a donor claims any free sibling: a third pair for + # a context whose two work-state channels are bound. + unguarded = [(''' + || self.forfeited.contains(&owner) + { + return; + }''', ''' { + return; + }''')] + for name, edits, expected in [('fixed', [], None), + ('donor-reclaims-its-pair', reclaim, 'second graph on pair'), + ('donor-claims-any-sibling', unguarded, 'no work-state channel')]: + current = code + for old, new in edits: + assert current.count(old) == 1, (name, old[:50]) + current = current.replace(old, new) + fixture = root / ('forfeit-donor-' + name + '.rs'); fixture.write_text(current) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(fixture), '-o', + str(fixture.with_suffix(''))], check=True) + result = subprocess.run([str(fixture.with_suffix(''))], capture_output=True, text=True) + if expected is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + +def shared_vm_donor_tests(root, source): + """A VM with several render queues (one DRM file, as quickshell or a GPU + process with several GL contexts) under held40 pressure. One queue donates + its idle sibling; its displaced graph keeps that VM's view of the pair + until the queue's own scheduler release. The VM's other queues are then + offered the freed pair: an existing queue claiming a sibling, and a new + queue. Runs the actual admission State and the actual per-VM view records + of mmu/lifetime.rs; graph lifetimes follow the rules checked in + forfeit_donor_tests.""" + g17 = source / 'g17' + def item(s, start): + at = s.index(start); begin = s.index('{', at); depth = 1; end = begin + 1 + while depth: + depth += (s[end] == '{') - (s[end] == '}'); end += 1 + return s[at:end] + s = (g17 / 'queue/admission.rs').read_text(); cfg = (source / 'hw/t8140.rs').read_text() + constants = '\n'.join(re.search(r'pub\(crate\) const ' + name + r': [^;]+;', cfg).group(0) + for name in ['RENDER_SLOTS', 'RENDER_PAIRS_PER_OWNER', 'RENDER_DEPTH']) + mmu = (source / 'mmu.rs').read_text() + limits = '\n'.join(re.search(r'const ' + name + r': [^;]+;', mmu).group(0) + for name in ['EXECUTION_CONTEXT_MASK', 'MAX_EXECUTION_CONTEXTS']) + lifetime = (source / 'mmu/lifetime.rs').read_text() + views = item(lifetime, 'pub(super) struct DriverMappings') + '\n' + item(lifetime, 'struct PoolMappings') + '\n' + \ + item(lifetime, 'impl DriverMappings') + '\nimpl Vm {\n' + '\n'.join( + item(lifetime, 'pub(crate) fn ' + name + '(') for name in [ + 'install_driver_mappings', 'render_pool_tvb_blocks', 'install_render_pool_mappings', + 'append_render_pool_mappings_prepared', 'truncate_render_pool_mappings', + 'clear_render_pool_mappings_if_owner']) + '\n}\n' + views = views.replace('pub(super) ', '').replace('pub(crate) ', '') + header = '''#![allow(dead_code, unused_macros, unused_imports)] + use std::ops::Range; + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; const EINVAL:i32=-22; const EBUSY:i32=-16; const EOVERFLOW:i32=-75; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + struct InFlight; + #[derive(Clone,Copy)] struct Flags; const GFP_KERNEL:Flags=Flags; + struct KVec(Vec); + impl KVec { + const fn new()->Self {KVec(Vec::new())} + fn push(&mut self,v:T,_:Flags)->Result {self.0.push(v);Ok(())} + fn push_within_capacity(&mut self,v:T)->std::result::Result<(),T> {if self.0.len()Result {self.0.reserve(n);Ok(())} + fn remove(&mut self,i:usize)->std::result::Result {if iOption {self.0.pop()} + fn drain_all(&mut self)->std::vec::Drain<'_,T> {self.0.drain(..)} + fn len(&self)->usize {self.0.len()} + fn capacity(&self)->usize {self.0.capacity()} + fn is_empty(&self)->bool {self.0.is_empty()} + fn iter(&self)->std::slice::Iter<'_,T> {self.0.iter()} + fn iter_mut(&mut self)->std::slice::IterMut<'_,T> {self.0.iter_mut()} + } + struct KernelMapping(u64); + impl KernelMapping {fn iova_range(&self)->Range {self.0..self.0+1}} + struct Mutex(std::sync::Mutex); + impl Mutex {fn lock(&self)->std::sync::MutexGuard<'_,T> {self.0.lock().unwrap()}} + struct VmLifetime {driver:Mutex>} + struct Vm {lifetime:Option>} + mod hw {pub mod t8140 {pub mod queues {''' + re.search(r'pub\(crate\) const RENDER_SLOTS: [^;]+;', cfg).group(0).replace('pub(crate)', 'pub') + '''}}} + mod cfg {\n''' + constants + '}\n' + limits + '\n' + methods = ['const fn new', 'fn exhausted', 'fn refusal', 'fn advance', 'fn take_idle', 'fn acquire', + 'fn claim_sibling', 'fn forfeit_sibling', 'fn release'] + follow = ',None' if 'fn acquire(&mut self, owner: u64, follow' in s else '' + code = header + views + item(s, 'struct State') + '\nimpl State {\n' + \ + re.search(r' const MASK: u64 = .*;', s).group(0) + '\n' + \ + '\n'.join(item(s, n) for n in methods) + '\n}\n' + ''' + use std::collections::{HashMap,HashSet}; + // The graph on each pair (owner, view id); a parked pair has none. + // Displaced graphs of open queues keep their VM view until that queue's + // scheduler release; a closed queue's graphs are released and parked. + struct Model {graph:[(u64,u64);64],retained:Vec<(u64,usize,u64)>,vm:HashMap,vms:Vec,open:HashSet,next:u64} + impl Model { + fn view(&self,owner:u64)->&Vm {&self.vms[self.vm[&owner]]} + fn publish(&mut self,owner:u64,slot:usize) { + let (previous,id)=self.graph[slot]; + if previous==owner {return;} + if previous!=0 && self.vm[&previous]==self.vm[&owner] { + // Same VM: the pair is rebound without a new graph. + self.graph[slot]=(owner,id);return; + } + self.next+=1;let new=self.next; + if let Err(error)=self.view(owner).install_render_pool_mappings(slot as u8,new,1,KVec(vec![KernelMapping(new)])) { + assert_eq!(error,EBUSY); + panic!("queue {} (VM {}) stalls on pair {}: another graph of its VM keeps its view there until queue {:?} closes", + owner,self.vm[&owner],slot,self.retained.iter().filter(|r|r.1==slot).map(|r|r.0).collect::>()); + } + if previous!=0 { + if self.open.contains(&previous) {self.retained.push((previous,slot,id));} + else {self.view(previous).clear_render_pool_mappings_if_owner(slot as u8,id);} + } + self.graph[slot]=(owner,new); + } + fn close(&mut self,owner:u64) { + self.open.remove(&owner); + for slot in 0..64 { + let (o,id)=self.graph[slot]; + if o==owner {self.view(owner).clear_render_pool_mappings_if_owner(slot as u8,id);self.graph[slot]=(0,0);} + } + let mine:Vec<_>=self.retained.iter().filter(|r|r.0==owner).cloned().collect(); + for (o,slot,id) in mine {self.view(o).clear_render_pool_mappings_if_owner(slot as u8,id);} + self.retained.retain(|r|r.0!=owner); + } + } + fn close(s:&mut State,m:&mut Model,owner:u64) { + for slot in 0..cfg::RENDER_SLOTS {if s.owners[slot]==owner {s.released|=1u64<u8 { + let slot=s.acquire(owner FOLLOW).unwrap_or_else(||panic!("queue {} found no pair",owner)); + m.publish(owner,slot as usize); + slot + } + fn candidates(s:&State,m:&Model)->Vec<(u8,u64)> { + (1..cfg::RENDER_SLOTS).filter(|slot|{ + let donor=s.owners[*slot]; + donor!=0 && m.open.contains(&donor) && m.graph[*slot].0==donor && s.refs[*slot]==0 + && (1..cfg::RENDER_SLOTS).filter(|other|s.owners[*other]==donor && m.graph[*other].0==donor).count()>=2 + }).map(|slot|(slot as u8,s.owners[slot])).collect() + } + fn main(){ + // VM 1 hosts queues 1 (donor), 32 (one pair) and later 50 (new); every other queue has its own VM. + let (donor,sibling,late)=(1u64,32u64,50u64); + let mut s=State::new(); + let mut m=Model{graph:[(0,0);64],retained:Vec::new(),vm:HashMap::new(),vms:Vec::new(),open:HashSet::new(),next:0}; + let vm_of=|m:&mut Model,owner:u64,shared:bool| { + let index=if shared && !m.vms.is_empty() {0} else { + let vm=Vm{lifetime:Some(std::sync::Arc::new(VmLifetime{driver:Mutex(std::sync::Mutex::new(None))}))}; + vm.install_driver_mappings(KVec::new(),KVec::new()).unwrap();m.vms.push(vm);m.vms.len()-1}; + m.vm.insert(owner,index);m.open.insert(owner); + }; + for owner in 1..=32 { + vm_of(&mut m,owner,owner==donor||owner==sibling); + let a=render(&mut s,&mut m,owner); + if owner<32 {let b=render(&mut s,&mut m,owner);s.release(b);} + s.release(a); + } + for owner in 2..=31 {let a=render(&mut s,&mut m,owner);let b=render(&mut s,&mut m,owner);s.release(b);s.release(a);} + let first=render(&mut s,&mut m,donor);s.release(first); + let mut recipients=Vec::new(); + for owner in 100..140 { + vm_of(&mut m,owner,false); + assert_eq!(s.refusal(owner),Err(ENOSPC)); + if let Some((slot,from))=s.forfeit_sibling(&candidates(&s,&m),0,owner) {recipients.push((owner,slot as usize,from));} + } + let &(holder,given,_)=recipients.iter().find(|r|r.2==donor).expect("the VM's queue donates its least recently used sibling"); + for _ in 0..5 { + for &(owner,_,_) in &recipients {let slot=render(&mut s,&mut m,owner);s.release(slot);} + let x=render(&mut s,&mut m,donor);let y=render(&mut s,&mut m,donor);s.release(y);s.release(x); + } + let displaced=m.retained.iter().find(|r|r.0==donor&&r.1==given).expect("control: the donor's displaced graph keeps its view").2; + // Only the donor's recipient exits: the given pair is the one free slot. + close(&mut s,&mut m,holder); + // The VM's queue with one pair claims it as its sibling and draws on both. + for round in 0..10 { + let x=render(&mut s,&mut m,sibling);let y=render(&mut s,&mut m,sibling); + if round==0 {assert_eq!(y as usize,given,"control: the VM's other queue is offered the freed pair");} + s.release(y);s.release(x); + } + // That queue exits; a new queue of the same VM is offered the pair. + close(&mut s,&mut m,sibling); + vm_of(&mut m,late,true); + let x=render(&mut s,&mut m,late);let y=render(&mut s,&mut m,late); + assert!(x as usize==given||y as usize==given,"control: the new queue of the VM is offered the freed pair"); + s.release(y);s.release(x); + let view=m.view(donor); + assert_eq!(view.render_pool_tvb_blocks(given as u8,displaced),Some(1),"the donor's displaced graph keeps its view"); + assert_eq!(view.render_pool_tvb_blocks(given as u8,m.graph[given].1),Some(1),"the new graph has its own view"); + close(&mut s,&mut m,donor); + assert_eq!(m.view(late).render_pool_tvb_blocks(given as u8,displaced),None,"the donor's release drops only its view"); + assert_eq!(m.view(late).render_pool_tvb_blocks(given as u8,m.graph[given].1),Some(1)); + println!("shared VM: a queue of a multi-queue VM donates its sibling under held40; the VM's other queue and a new queue install and draw on the freed pair while the donor's displaced graph keeps its own view"); + } + '''.replace(' FOLLOW', follow) + one_view = [('if views.iter().any(|pool| pool.id == id) {', 'if !views.is_empty() {')] + for name, edits, expected in [('fixed', [], None), ('one-view-per-pair', one_view, 'stalls on pair')]: + current = code + for old, new in edits: + assert current.count(old) == 1, (name, old) + current = current.replace(old, new) + fixture = root / ('shared-vm-' + name + '.rs'); fixture.write_text(current) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(fixture), '-o', + str(fixture.with_suffix(''))], check=True) + result = subprocess.run([str(fixture.with_suffix(''))], capture_output=True, text=True) + if expected is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + +if __name__ == '__main__': + with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: + admission_tests(Path(temporary), DRIVER) + forfeit_donor_tests(Path(temporary), DRIVER) + shared_vm_donor_tests(Path(temporary), DRIVER) + queue_tests(Path(temporary), DRIVER) + compute_takeover_tests(Path(temporary), DRIVER) + compute_admission_tests(Path(temporary), DRIVER) + lower_window_gate_tests(Path(temporary), DRIVER) + watchdog_publication_guard_tests(DRIVER) + shared_alias_tests(Path(temporary), DRIVER) diff --git a/tools/asahi/g17-statistics-tests.rs b/tools/asahi/g17-statistics-tests.rs new file mode 100644 index 00000000000000..3b4f49314dae4b --- /dev/null +++ b/tools/asahi/g17-statistics-tests.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Host tests for the production firmware-report snapshot, including concurrent readers. + +#[path = "../../drivers/gpu/drm/asahi/g17/load.rs"] +mod load; + +#[path = "../../drivers/gpu/drm/asahi/g17/statistics.rs"] +mod statistics; + +use statistics::{Statistics, WORDS}; + +#[test] +fn retains_full_payload_and_separates_sources() { + let stats = Statistics::new(); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 0); + let mut words = core::array::from_fn(|i| i as u32); + words[0] = 0x14; + stats.record(0, 1234, &words); + let report = stats.snapshot(0, 0x14).unwrap(); + assert_eq!(report.words, words); + assert_eq!(report.received_ns, 1234); + assert_eq!(report.count, 1); + assert_eq!(stats.snapshot(1, 0x14).unwrap().count, 0); + assert_eq!(stats.snapshot(0, 0x19).unwrap().count, 0); + words[17] = u32::MAX; + stats.record(0, 4567, &words); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 2); + assert_eq!(stats.snapshot(0, 0x14).unwrap().words, words); +} + +#[test] +fn rejects_out_of_range_sources_without_aliasing() { + let stats = Statistics::new(); + for tag in [0, 15, 31, u32::MAX] { + let mut words = [0; WORDS]; + words[0] = tag; + stats.record(0, 1, &words); + assert!(stats.snapshot(0, tag).is_none()); + } + let mut words = [0; WORDS]; + words[0] = 0x14; + stats.record(2, 1, &words); + assert!(stats.snapshot(2, 0x14).is_none()); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 0); +} + +#[test] +fn readers_never_observe_a_torn_report() { + let stats = Statistics::new(); + std::thread::scope(|scope| { + scope.spawn(|| { + for sequence in 1..=50_000 { + let mut words = [sequence; WORDS]; + words[0] = 0x14; + stats.record(0, sequence as u64, &words); + } + }); + for _ in 0..50_000 { + if let Some(report) = stats.snapshot(0, 0x14) { + if report.count != 0 { + assert_eq!(report.received_ns, report.count); + assert_eq!(report.words[0], 0x14); + assert!(report.words[1..] + .iter() + .all(|&word| word as u64 == report.count)); + } + } + } + }); + assert_eq!(stats.snapshot(0, 0x14).unwrap().count, 50_000); +} + +#[test] +fn full_payload_capture_is_requested_explicitly_and_expires() { + let stats = Statistics::new(); + assert!(!stats.capture_pending()); + assert!(!stats.capture_enabled(1)); + stats.request_capture(100); + assert!(stats.capture_pending()); + assert!(stats.capture_enabled(101)); + stats.request_capture(200); + assert!(stats.capture_enabled(100 + statistics::CAPTURE_NS)); + assert!(!stats.capture_enabled(200 + statistics::CAPTURE_NS)); + assert!(!stats.capture_pending()); +} diff --git a/tools/asahi/g17-telemetry-registration-tests.py b/tools/asahi/g17-telemetry-registration-tests.py new file mode 100644 index 00000000000000..cf5d7b7b9968c9 --- /dev/null +++ b/tools/asahi/g17-telemetry-registration-tests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check actual probe registration branches against a shared sysfs namespace. + +The namespace and optional-registration failures are host boundary stubs, not +kernel execution. A negative control restores the duplicate placeholder probe. +""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / "drivers/gpu/drm/asahi" + + +def fixture(): + gpu = (ROOT / "g17/mod.rs").read_text() + probe = (ROOT / "driver.rs").read_text().split(" fn probe_g17", 1)[1] + telemetry = gpu[gpu.index(" let telemetry ="):gpu.index(" let active_cores =")] + platform = probe[probe.index(" let raw_stats ="):probe.index(" Ok(AsahiPlatformDriverData")] + names = [re.search(r'attr\.attr\.name = "([^"]+)"', (ROOT / name).read_text()).group(1) + for name in ["sysfs.c", "asahi_telemetry.c"]] + assert names == ["agx_stats", "agx_stats"], names + return r''' +#![allow(dead_code, non_upper_case_globals)] +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering::SeqCst}; +type Result = std::result::Result; +static EXPORT: AtomicU8 = AtomicU8::new(1); +static PRESENT: AtomicBool = AtomicBool::new(false); +static FAIL: AtomicBool = AtomicBool::new(false); +static STOPPED: AtomicBool = AtomicBool::new(false); +static DUPLICATE: AtomicBool = AtomicBool::new(false); +macro_rules! dev_warn { ($dev:expr, $($t:tt)*) => {{ let _ = $dev; let _ = format_args!($($t)*); }}; } +mod module_parameters { + pub struct Parameter; + impl Parameter { + pub fn value(&self) -> Box { Box::new(super::EXPORT.load(super::SeqCst)) } + } + pub static stats_export: Parameter = Parameter; +} +struct Device; +impl AsRef for Device { fn as_ref(&self) -> &Self { self } } +impl Device { fn as_raw(&self) -> *mut () { std::ptr::dangling_mut() } } +struct Gpu; +impl Gpu { fn shutdown(&self) { STOPPED.store(true, SeqCst); } } +fn create() -> Result<*mut ()> { + if FAIL.load(SeqCst) { return Err(-12); } + if PRESENT.swap(true, SeqCst) { + DUPLICATE.store(true, SeqCst); + return Err(-17); + } + Ok(std::ptr::dangling_mut()) +} +fn remove() { assert!(PRESENT.swap(false, SeqCst)); } +mod telemetry { + use super::*; + pub struct Telemetry; + impl Telemetry { + pub fn new(_: &Device, _: u32) -> Result { create()?; Ok(Self) } + } + impl Drop for Telemetry { fn drop(&mut self) { remove(); } } +} +mod sysfs_exports { + use super::*; + pub fn register(_: *mut (), snapshot: *const (), enabled: bool) -> Result<*mut ()> { + assert!(snapshot.is_null() && !enabled); + create() + } +} +fn gpu_registration(pdev: &Device) -> Option { + let statistics = 0u32; +TELEMETRY + telemetry +} +fn platform_registration(pdev: &Device, gpu: &Gpu) -> Result<*mut ()> { + let raw_dev = pdev.as_raw(); +PLATFORM + Ok(raw_stats) +} +fn main() { + for export in [1, 0, 2, 0] { + for fail in [false, true] { + EXPORT.store(export, SeqCst); FAIL.store(fail, SeqCst); + STOPPED.store(false, SeqCst); + let telemetry = gpu_registration(&Device); + let platform = platform_registration(&Device, &Gpu); + assert!(!DUPLICATE.load(SeqCst), "both probe owners created agx_stats"); + if export != 0 { + assert_eq!(telemetry.is_some(), !fail); + assert!(platform.unwrap().is_null()); + assert!(!STOPPED.load(SeqCst), "optional telemetry stopped GPU probe"); + } else if fail { + assert_eq!(platform, Err(-12)); + assert!(STOPPED.load(SeqCst)); + } else { + assert!(telemetry.is_none()); + assert!(!platform.unwrap().is_null()); + remove(); + } + drop(telemetry); + assert!(!PRESENT.load(SeqCst), "registration outlived its owner"); + } + } + println!("PASS enabled/disabled probe ownership, allocation failures and repeated teardown"); +} +'''.replace("TELEMETRY", telemetry).replace("PLATFORM", platform) + + +def main(): + code = fixture() + with tempfile.TemporaryDirectory(prefix="g17-telemetry-registration-") as temporary: + root = Path(temporary) + for mutant in [False, True]: + source = root / f"registration-{mutant}.rs" + source.write_text(code.replace( + "let raw_stats = if *module_parameters::stats_export.value() == 0", + "let raw_stats = if true") if mutant else code) + binary = source.with_suffix("") + subprocess.run([os.environ.get("RUSTC", "rustc"), "--edition=2021", + str(source), "-o", str(binary)], check=True) + result = subprocess.run([str(binary)], text=True, capture_output=True) + if mutant: + assert result.returncode != 0 and "both probe owners" in result.stderr + print("PASS negative control rejects duplicate agx_stats probe registration") + else: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/gpu-fdinfo-tests.py b/tools/asahi/gpu-fdinfo-tests.py new file mode 100644 index 00000000000000..1df2da6ea2a7e6 --- /dev/null +++ b/tools/asahi/gpu-fdinfo-tests.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production fdinfo residency helpers with cached and imported backing.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "drivers/gpu/drm/drm_gem_shmem_helper.c" + + +def function(text, signature): + start = text.index(signature) + opening = text.index("{", start) + depth = 1 + end = opening + 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + original = SOURCE.read_text() + functions = "\n".join(function(original, signature) for signature in [ + "size_t drm_gem_shmem_object_rss(", + "enum drm_gem_object_status drm_gem_shmem_object_status(", + ]) + fixture = r''' +#include +#include +#include +#include +#define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x)) +#define min_t(type,a,b) ((type)(a) < (type)(b) ? (type)(a) : (type)(b)) +#define PAGE_SIZE (1UL << PAGE_SHIFT) +struct address_space { unsigned long nrpages; }; +struct file { struct address_space *f_mapping; }; +struct drm_gem_object { size_t size; void *import_attach; struct file *filp; }; +enum drm_gem_object_status { DRM_GEM_OBJECT_RESIDENT = 1 }; +FUNCTIONS +int main(void) { + struct address_space mapping = {0}; + struct file backing = { .f_mapping = &mapping }; + struct drm_gem_object obj = { .size = 5 * PAGE_SIZE }; + assert(drm_gem_shmem_object_rss(&obj) == 0); + assert(drm_gem_shmem_object_status(&obj) == 0); + obj.filp = &backing; + assert(drm_gem_shmem_object_rss(&obj) == 0); + mapping.nrpages = 2; + assert(drm_gem_shmem_object_rss(&obj) == 2 * PAGE_SIZE); + assert(drm_gem_shmem_object_status(&obj) == DRM_GEM_OBJECT_RESIDENT); + /* A cached backing remains resident independently of a device page-array pin. */ + mapping.nrpages = 5; + assert(drm_gem_shmem_object_rss(&obj) == obj.size); + mapping.nrpages = 9; + assert(drm_gem_shmem_object_rss(&obj) == obj.size); + mapping.nrpages = 0; + assert(drm_gem_shmem_object_rss(&obj) == 0); + assert(drm_gem_shmem_object_status(&obj) == 0); + obj.filp = NULL; + obj.import_attach = &mapping; + assert(drm_gem_shmem_object_rss(&obj) == obj.size); + assert(drm_gem_shmem_object_status(&obj) == DRM_GEM_OBJECT_RESIDENT); + assert(obj.size == 5 * PAGE_SIZE && mapping.nrpages == 0); + puts("PASS unpopulated, cached, partial, bounded, evicted and imported backing"); +} +'''.replace("FUNCTIONS", functions) + source = args.output / "fdinfo.c" + source.write_text(fixture) + results = [] + for shift in [12, 14]: + binary = args.output / f"fdinfo-{shift}" + subprocess.run(["cc", "-std=gnu11", "-Wall", "-Werror", "-g", + "-fsanitize=address,undefined", "-fno-pie", "-no-pie", + f"-DPAGE_SHIFT={shift}", str(source), "-o", str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, check=True) + results.append({"page_shift": shift, "stdout": result.stdout, "stderr": result.stderr}) + record = {"production_sha256": hashlib.sha256(functions.encode()).hexdigest(), + "results": results, + "scope": "Actual C helper bodies with typed file/cache boundary stubs; not kernel ABI or hardware execution."} + (args.output / "result.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/gpu-load-tests.py b/tools/asahi/gpu-load-tests.py new file mode 100644 index 00000000000000..ecd1d502193283 --- /dev/null +++ b/tools/asahi/gpu-load-tests.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""A transient kernel snapshot collision must not terminate load monitoring.""" +import errno +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import Mock + +spec = importlib.util.spec_from_file_location('gpu_load', Path(__file__).with_name('gpu-load.py')) +gpu_load = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gpu_load) + + +class LoadTests(unittest.TestCase): + def test_busy_snapshot_is_unavailable(self): + path = Mock() + path.parent.resolve.return_value = '/sys/devices/platform/gpu' + path.read_text.side_effect = BlockingIOError(errno.EAGAIN, 'snapshot busy') + sample = gpu_load.snapshot(path) + self.assertFalse(sample['load_valid']) + self.assertIsNone(sample['busy_percent']) + + def test_other_read_errors_are_reported(self): + path = Mock() + path.read_text.side_effect = PermissionError(errno.EACCES, 'permission denied') + with self.assertRaises(PermissionError): + gpu_load.snapshot(path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/asahi/gpu-load.py b/tools/asahi/gpu-load.py new file mode 100755 index 00000000000000..5fd51b8ff57254 --- /dev/null +++ b/tools/asahi/gpu-load.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Read device-wide firmware load without confusing missing data with zero load.""" +import argparse +import json +from pathlib import Path +import time + + +def snapshot(path): + try: + text = path.read_text() + except BlockingIOError: + # A snapshot can lose its bounded retries to the firmware writer. + # Preserve an unavailable sample instead of terminating a monitor. + return {'device': str(path.parent.resolve()), 'source': 'firmware', + 'load_valid': False, 'busy_percent': None, + 'error': 'statistics snapshot busy'} + fields = dict(line.split(None, 1) for line in text.splitlines() + if len(line.split(None, 1)) == 2) + if fields.get('version') != '2' or fields.get('source') != 'firmware': + raise ValueError(f'{path}: unsupported statistics format') + valid = fields.get('load_valid') == '1' + return { + 'device': str(path.parent.resolve()), + 'source': 'firmware', + 'load_valid': valid, + 'busy_percent': int(fields['busy_percent']) if valid else None, + 'firmware_percentages': [int(fields[f'util{i}']) for i in range(1, 5)] if valid else None, + 'gpu_powered': fields['gpu_powered'] == '1' if fields['gpu_power_known'] == '1' else None, + 'samples': int(fields['samples']), + 'sample_age_seconds': int(fields['sample_age_ns']) / 1e9 if int(fields['samples']) else None, + 'invalid_samples': int(fields['invalid_samples']), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--path', type=Path) + parser.add_argument('--count', type=int, default=1) + parser.add_argument('--interval', type=float, default=1.0) + args = parser.parse_args() + if args.count < 1 or args.interval <= 0: + parser.error('count and interval must be positive') + paths = [args.path] if args.path else sorted(Path('/sys/class/drm').glob('renderD*/device/agx_stats')) + if not paths: + parser.error('no GPU statistics attribute found') + for index in range(args.count): + print(json.dumps([snapshot(path) for path in paths]), flush=True) + if index + 1 != args.count: + time.sleep(args.interval) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/gpu-memory-report-tests.py b/tools/asahi/gpu-memory-report-tests.py new file mode 100644 index 00000000000000..6b8f975dd30adb --- /dev/null +++ b/tools/asahi/gpu-memory-report-tests.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("gpu_memory_report", + Path(__file__).with_name("gpu-memory-report.py")) +reporter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(reporter) + + +class MemoryReportTests(unittest.TestCase): + def test_units_and_unavailable_residency(self): + self.assertEqual(reporter.size_bytes("3 MiB"), 3 * 1024**2) + self.assertEqual(reporter.size_bytes("17"), 17) + self.assertEqual(reporter.size_bytes("64 KiB"), 65536) + with self.assertRaises(ValueError): + reporter.size_bytes("3 bananas") + self.assertIsNone(reporter.parse_fdinfo("pos:\t0\n")) + + def test_shared_descriptors_are_not_recounted_as_allocations(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + node = root / "renderD128" + node.touch() + for pid, fds in [(100, [5, 6]), (200, [8])]: + process = root / str(pid) + (process / "fdinfo").mkdir(parents=True) + (process / "fd").mkdir() + (process / "comm").write_text(f"process-{pid}\n") + for fd in fds: + (process / "fd" / str(fd)).symlink_to(node) + (process / "fdinfo" / str(fd)).write_text( + "drm-driver:\tasahi\ndrm-client-id:\t7\n" + "drm-total-memory:\t10 MiB\ndrm-shared-memory:\t2 MiB\n") + result = reporter.report(root, root / "debug") + self.assertEqual(len(result["clients"]), 1) + client = result["clients"][0] + self.assertEqual(client["fd_count"], 3) + self.assertEqual(len(client["processes"]), 2) + self.assertEqual(result["private_handle_bytes"], 8 * 1024**2) + self.assertIsNone(client["memory_bytes"]["resident"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/asahi/gpu-memory-report.py b/tools/asahi/gpu-memory-report.py new file mode 100644 index 00000000000000..aec96590e527d2 --- /dev/null +++ b/tools/asahi/gpu-memory-report.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Attribute Asahi GEM handles without counting duplicated DRM file descriptors twice.""" +import argparse +import json +from pathlib import Path +import re + + +def size_bytes(value): + match = re.fullmatch(r"(\d+)(?:\s+(B|KiB|MiB|GiB))?", value.strip()) + if not match: + raise ValueError(f"unsupported DRM size: {value!r}") + return int(match[1]) * {None: 1, "B": 1, "KiB": 1024, + "MiB": 1024**2, "GiB": 1024**3}[match[2]] + + +def parse_fdinfo(text): + fields = dict(line.split(":", 1) for line in text.splitlines() if ":" in line) + fields = {key: value.strip() for key, value in fields.items()} + if "drm-driver" not in fields or "drm-client-id" not in fields: + return None + return fields + + +def collect(proc_root, driver="asahi"): + clients = {} + unreadable = 0 + for process in sorted(proc_root.iterdir()): + if not process.name.isdigit(): + continue + try: + name = (process / "comm").read_text().strip() + infos = list((process / "fdinfo").iterdir()) + except (OSError, ValueError): + unreadable += 1 + continue + for info in infos: + try: + fields = parse_fdinfo(info.read_text()) + if not fields or fields["drm-driver"] != driver: + continue + node = (process / "fd" / info.name).resolve(strict=True) + identity = fields.get("drm-pdev", str(node)) + key = (driver, identity, int(fields["drm-client-id"])) + if key not in clients: + memory = { + kind: size_bytes(fields[f"drm-{kind}-memory"]) + if f"drm-{kind}-memory" in fields else None + for kind in ["total", "shared", "resident", "active", "purgeable"] + } + clients[key] = { + "driver": driver, "device": identity, "client_id": key[2], + "name": fields.get("drm-client-name"), "memory_bytes": memory, + "processes": {}, "fd_count": 0, + } + clients[key]["processes"][int(process.name)] = name + clients[key]["fd_count"] += 1 + except (OSError, ValueError): + unreadable += 1 + rows = [] + for client in clients.values(): + client["processes"] = [{"pid": pid, "name": name} + for pid, name in sorted(client["processes"].items())] + rows.append(client) + return rows, unreadable + + +def report(proc_root, debug_root, driver="asahi"): + clients, unreadable = collect(proc_root, driver) + allocation_extents = {} + for generation in ["g17", "m3", "g16"]: + path = debug_root / f"asahi-{generation}" / "memory" + try: + text = path.read_text() + except OSError: + continue + allocation_extents[str(path)] = text + known = [row["memory_bytes"] for row in clients + if row["memory_bytes"]["total"] is not None + and row["memory_bytes"]["shared"] is not None] + return { + "clients": clients, + "clients_with_memory_reporting": len(known), + "private_handle_bytes": sum(row["total"] - row["shared"] for row in known), + "driver_allocation_extents": allocation_extents, + "unreadable_entries": unreadable, + "notes": [ + "Each DRM client is counted once across duplicated and inherited descriptors.", + "Resident/shared values from different clients must not be added as unique physical memory.", + "Private handle bytes exclude kernel allocations and objects retained without client handles.", + "Driver allocation extents are logical sizes, not a resident-page measurement.", + "Missing resident counters mean unavailable, not zero. Root access gives fuller process coverage.", + ], + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proc-root", type=Path, default=Path("/proc")) + parser.add_argument("--debug-root", type=Path, default=Path("/sys/kernel/debug")) + parser.add_argument("--driver", default="asahi") + args = parser.parse_args() + print(json.dumps(report(args.proc_root, args.debug_root, args.driver), indent=2)) diff --git a/tools/asahi/gpu-memory-tests.rs b/tools/asahi/gpu-memory-tests.rs new file mode 100644 index 00000000000000..7aeffeca1166b4 --- /dev/null +++ b/tools/asahi/gpu-memory-tests.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Compile and exercise the production allocation lifetime counters on the host. + +extern crate self as kernel; + +pub mod prelude { + #[derive(Debug, PartialEq, Eq)] + pub enum Error { + Invalid, + Overflow, + } + pub const EINVAL: Error = Error::Invalid; + pub const EOVERFLOW: Error = Error::Overflow; + pub type Result = std::result::Result; +} +pub mod fmt { + pub use std::fmt::{Formatter, Result}; +} +pub mod debugfs { + pub trait Writer { + fn write(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result; + } +} + +#[allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/agx_memory_stats.rs"] +mod memory; diff --git a/tools/asahi/neo-auxiliary-tests.py b/tools/asahi/neo-auxiliary-tests.py index 68236fe19d1927..ab6273432722cc 100644 --- a/tools/asahi/neo-auxiliary-tests.py +++ b/tools/asahi/neo-auxiliary-tests.py @@ -27,10 +27,13 @@ impl KernelMapping{pub fn iova(&self)->u64{self.va}} } mod storage{CONSTANTS} +mod agx_memory_stats{pub enum KernelCategory{RenderGeneration}} +use agx_memory_stats::KernelCategory; mod cfg{pub const AUXILIARY:std::ops::Range=0x100000..0x500000;} enum CpuMap{WriteCombined} struct KernelObject{va:u64,size:usize} impl KernelObject{ + fn classify_memory(&self,_c:KernelCategory){} fn gpu_va(&self)->u64{self.va} fn map_range(&self,_v:&mmu::Vm,_r:Range,_a:Range,_align:u64,_p:u32,_g:u32)->Result{Err(-28)} }