diff --git a/drivers/gpu/drm/asahi/g17/clpc.c b/drivers/gpu/drm/asahi/g17/clpc.c index 58adaba19910f..0b650549841de 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.c +++ b/drivers/gpu/drm/asahi/g17/clpc.c @@ -1,7 +1,45 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include +#include #include "clpc.h" +static unsigned int g17_light_effort = 0x8000; +module_param(g17_light_effort, uint, 0644); +MODULE_PARM_DESC(g17_light_effort, "G17 performance effort for quiet GPU work, of 65536 (0: always full)"); + +/* + * Light effort runs the same work about twice as long, so the way back to + * full effort has to sit at about twice the way in, or work that fits light + * effort sticks at full. Shares in permille, job lengths in microseconds. + */ +static unsigned int g17_light_enter = 350; +module_param(g17_light_enter, uint, 0644); +static unsigned int g17_light_exit = 750; +module_param(g17_light_exit, uint, 0644); +static unsigned int g17_light_job_enter_us = 3000; +module_param(g17_light_job_enter_us, uint, 0644); +static unsigned int g17_light_job_exit_us = 7000; +module_param(g17_light_job_exit_us, uint, 0644); + +unsigned int asahi_g17_light_param(unsigned int which) +{ + switch (which) { + case 0: + return READ_ONCE(g17_light_enter); + case 1: + return READ_ONCE(g17_light_exit); + case 2: + return READ_ONCE(g17_light_job_enter_us); + default: + return READ_ONCE(g17_light_job_exit_us); + } +} + +unsigned int asahi_g17_light_effort(void) +{ + return READ_ONCE(g17_light_effort); +} + int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out) { return apple_clpc_gpu_get(dev, out); @@ -10,6 +48,10 @@ void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) { apple_clpc_gpu_set_active(gpu, active); } +void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) +{ + apple_clpc_gpu_set_busy(gpu, permille); +} void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu) { apple_clpc_gpu_put(gpu); diff --git a/drivers/gpu/drm/asahi/g17/clpc.h b/drivers/gpu/drm/asahi/g17/clpc.h index 372820ade7eff..8b85175c3b789 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.h +++ b/drivers/gpu/drm/asahi/g17/clpc.h @@ -6,4 +6,7 @@ struct apple_clpc_gpu; int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out); void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu); +unsigned int asahi_g17_light_effort(void); +unsigned int asahi_g17_light_param(unsigned int which); +void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille); #endif diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ffb0c20d0267f..4436f07843abf 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -713,7 +713,7 @@ impl Firmware { fn note_submission(&mut self) -> Result { self.port.note_gpu_activity(); self.effort.submit(); - self.apply_effort(power::FULL_EFFORT) + self.apply_effort(power::submission_effort()) } fn apply_effort(&mut self, effort: u32) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 425eb6f517b71..4feda11d9e2dc 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -516,7 +516,9 @@ impl Port { pub(super) fn note_submission(&self) -> bool { self.submissions.fetch_add(1, Ordering::SeqCst); self.note_gpu_activity(); + // Take the slow path when the governor wants a different effort. self.effort_full.load(Ordering::SeqCst) + && super::power::submission_effort() == super::power::FULL_EFFORT } pub(super) fn submissions(&self) -> u64 { @@ -546,6 +548,7 @@ impl Port { self.last_submission_ns .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); + self.demand.report_busy(super::now_ns()); } pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index a2e293df91510..368b62733d35c 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,7 +3,7 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. -use core::sync::atomic::AtomicU32; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; pub(super) const FULL_EFFORT: u32 = 0x10000; pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; @@ -305,6 +305,53 @@ unsafe extern "C" { fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); + fn asahi_g17_light_effort() -> u32; + fn asahi_g17_light_param(which: core::ffi::c_uint) -> core::ffi::c_uint; + fn asahi_clpc_gpu_set_busy(gpu: *mut core::ffi::c_void, permille: core::ffi::c_uint); +} + +/// Busy-share window: long enough to span frames, short enough to follow a burst. +const BUSY_WINDOW_NS: u64 = 200_000_000; +/// A longer gap between reports was idle: start over rather than average it in. +const BUSY_WINDOW_MAX_NS: u64 = 2_000_000_000; +static BUSY_WINDOW_START: AtomicU64 = AtomicU64::new(0); +static BUSY_WINDOW_TOTAL: AtomicU64 = AtomicU64::new(0); + +/// A window under this share (permille) with no long job is quiet. +/// Consecutive quiet windows needed to go light. +const LIGHT_WINDOWS: u32 = 4; +static LIGHT: AtomicBool = AtomicBool::new(false); +static QUIET_WINDOWS: AtomicU32 = AtomicU32::new(0); + +/// The effort for the next submission: light once the GPU has been quiet for a while. +pub(super) fn submission_effort() -> u32 { + // SAFETY: reads a module parameter. + let light = unsafe { asahi_g17_light_effort() }; + if LIGHT.load(Ordering::Relaxed) && light != 0 { + light + } else { + FULL_EFFORT + } +} + +fn light_param(which: u32) -> u64 { + // SAFETY: a plain read of a module parameter. + u64::from(unsafe { asahi_g17_light_param(which) }) +} + +fn light_step(permille: u64, longest_ns: u64) { + if LIGHT.load(Ordering::Relaxed) { + if permille >= light_param(1) || longest_ns >= light_param(3) * 1000 { + LIGHT.store(false, Ordering::Relaxed); + QUIET_WINDOWS.store(0, Ordering::Relaxed); + } + } else if permille < light_param(0) && longest_ns < light_param(2) * 1000 { + if QUIET_WINDOWS.fetch_add(1, Ordering::Relaxed) + 1 >= LIGHT_WINDOWS { + LIGHT.store(true, Ordering::Relaxed); + } + } else { + QUIET_WINDOWS.store(0, Ordering::Relaxed); + } } /// DT-selected memory-demand provider. Its independent allocation outlives a @@ -332,6 +379,29 @@ impl Demand { // SAFETY: the retained reference is live; NULL denotes no DT provider. unsafe { asahi_clpc_gpu_set_active(self.0, active) }; } + + /// Reports the share of the last window the GPU spent executing, once a window has run. + /// Racing submitters can only lose one window's report. + pub(super) fn report_busy(&self, now_ns: u64) { + let start = BUSY_WINDOW_START.load(Ordering::Relaxed); + let elapsed = now_ns.saturating_sub(start); + if elapsed < BUSY_WINDOW_NS { + return; + } + let total = crate::usage::TOTAL_NS.load(Ordering::Relaxed); + let busy = total.saturating_sub(BUSY_WINDOW_TOTAL.swap(total, Ordering::Relaxed)); + BUSY_WINDOW_START.store(now_ns, Ordering::Relaxed); + if start == 0 || elapsed > BUSY_WINDOW_MAX_NS { + return; + } + let permille = busy.saturating_mul(1000) / elapsed; + light_step(permille, crate::usage::MAX_NS.swap(0, Ordering::Relaxed)); + // Full effort does the same work in about half the time; report the share in light + // effort's terms so that one memory ladder fits both. + let scaled = if LIGHT.load(Ordering::Relaxed) { permille } else { permille * 2 }; + // SAFETY: the retained reference is live; NULL denotes no DT provider. + unsafe { asahi_clpc_gpu_set_busy(self.0, scaled.min(1000) as core::ffi::c_uint) }; + } } impl Drop for Demand { diff --git a/drivers/gpu/drm/asahi/usage.rs b/drivers/gpu/drm/asahi/usage.rs index 496c722dba651..13fbe6b2a6b4a 100644 --- a/drivers/gpu/drm/asahi/usage.rs +++ b/drivers/gpu/drm/asahi/usage.rs @@ -8,6 +8,11 @@ use core::sync::atomic::{AtomicU64, Ordering}; +/// Engine time charged across all clients, in ns, for the GPU busy share. +pub(crate) static TOTAL_NS: AtomicU64 = AtomicU64::new(0); +/// Longest single stage charged since the governor last read it, in ns. +pub(crate) static MAX_NS: AtomicU64 = AtomicU64::new(0); + #[derive(Clone, Copy)] pub(crate) enum Engine { Vertex, @@ -35,6 +40,8 @@ impl Client { return; } let ns = ticks_to_ns(end - start, frequency_hz); + TOTAL_NS.fetch_add(ns, Ordering::Relaxed); + MAX_NS.fetch_max(ns, Ordering::Relaxed); let counter = &self.ns[engine as usize]; let mut old = counter.load(Ordering::Relaxed); loop { diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index c7b29f0e6a49b..6f5ed47d3f9b3 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -63,6 +63,9 @@ enum apple_clpc_rail { */ #define APPLE_CLPC_GPU_BACKOFF_MS 100 +/* A lower GPU busy level must last this long; a higher one applies at once. */ +#define APPLE_CLPC_GPU_DOWN_MS 400 + /** * struct apple_clpc_hw - per-SoC CLPC parameters * @rail: PMC rail of the DCS and of the fabric @@ -80,7 +83,12 @@ enum apple_clpc_rail { * dense, see apple_clpc_gpu_dense(); 0 for none * @gpu_input: the GPU driver reports its work in flight through * apple_clpc_gpu_get(); while there is some, both rails go to the top - * of @map + * of @map, or to @gpu_busy if the driver reports a busy share + * @gpu_busy: state of each rail for reported GPU work below gpu_busy_mid, + * gpu_busy_high and gpu_busy_top and at or above it; all zero to always + * use the top of @map + * @level_khz: frequency @map's quarters are taken from, so that clusters of + * different speed share them; 0 for each cluster's own maximum */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; @@ -93,6 +101,8 @@ struct apple_clpc_hw { unsigned int pmp_agent; unsigned int gpu_dense; bool gpu_input; + u8 gpu_busy[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; + unsigned int level_khz; }; struct apple_clpc_cluster { @@ -116,9 +126,32 @@ struct apple_clpc_gpu { bool active; u64 updated_ns; u64 sequence; + /* Busy share of the last window in permille, if the GPU reports one */ + unsigned int busy; + bool busy_valid; + unsigned int level; + unsigned long level_since; struct delayed_work *wake; }; +/* + * The GPU busy share (apple_clpc_gpu_set_busy()) at which reported GPU work + * gets the top memory and fabric floor, and below which it gets the lowest + * GPU state; in between it gets the second. The share is time spent + * executing at whatever clock the GPU runs, not bandwidth: a 120 fps game + * reads about a third, a scrolling browser a few percent. A GPU that + * reports no share keeps the top floor for all its work. + */ +static unsigned int gpu_busy_top = 800; +module_param(gpu_busy_top, uint, 0644); +MODULE_PARM_DESC(gpu_busy_top, "GPU busy share (permille) for the top memory/fabric floor"); +static unsigned int gpu_busy_high = 500; +module_param(gpu_busy_high, uint, 0644); +MODULE_PARM_DESC(gpu_busy_high, "GPU busy share (permille) for the floor below the top one"); +static unsigned int gpu_busy_mid = 160; +module_param(gpu_busy_mid, uint, 0644); +MODULE_PARM_DESC(gpu_busy_mid, "GPU busy share (permille) below which GPU work gets the lowest floor state"); + struct apple_clpc { struct device *dev; struct device *pmc; @@ -197,6 +230,26 @@ void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) } EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_active); +/** + * apple_clpc_gpu_set_busy() - Report the GPU busy share of the last window + * @gpu: the GPU input, from apple_clpc_gpu_get() + * @permille: share of the window the GPU spent executing work, 0..1000 + * + * Scales the floor that reported GPU work gets, see gpu_busy_top. + */ +void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) +{ + unsigned long flags; + + if (!gpu) + return; + spin_lock_irqsave(&gpu->lock, flags); + gpu->busy = min(permille, 1000U); + gpu->busy_valid = true; + spin_unlock_irqrestore(&gpu->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_busy); + void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) { if (gpu) { @@ -263,6 +316,30 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) } EXPORT_SYMBOL_GPL(apple_clpc_gpu_get); +/* The CPU-quartile column for reported GPU work, from its busy share */ +static unsigned int apple_clpc_gpu_level(struct apple_clpc_gpu *gpu) +{ + unsigned long flags; + unsigned int level; + + spin_lock_irqsave(&gpu->lock, flags); + if (!gpu->busy_valid || gpu->busy >= READ_ONCE(gpu_busy_top)) + level = APPLE_CLPC_LEVELS - 1; + else if (gpu->busy >= READ_ONCE(gpu_busy_high)) + level = 2; + else + level = gpu->busy >= READ_ONCE(gpu_busy_mid) ? 1 : 0; + if (level >= gpu->level || + time_after_eq(jiffies, gpu->level_since + + msecs_to_jiffies(APPLE_CLPC_GPU_DOWN_MS))) { + gpu->level = level; + gpu->level_since = jiffies; + } + level = gpu->level; + spin_unlock_irqrestore(&gpu->lock, flags); + return level; +} + static bool apple_clpc_gpu_active(struct apple_clpc_gpu *gpu, u64 *updated, u64 *sequence) { unsigned long flags; @@ -350,7 +427,8 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, policy = cpufreq_cpu_get(cl->policy_cpu); if (policy) { if (policy->cpuinfo.max_freq) - cl->level = policy->cur * 100 / policy->cpuinfo.max_freq; + cl->level = policy->cur * 100 / + (clpc->hw->level_khz ?: policy->cpuinfo.max_freq); cpufreq_cpu_put(policy); } @@ -532,13 +610,17 @@ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, unsigned int *demand) { u64 updated, sequence; - unsigned int rail; + unsigned int level, rail; if (!gpu || hw->gpu_demand || !apple_clpc_gpu_active(gpu, &updated, &sequence)) return false; + level = apple_clpc_gpu_level(gpu); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - demand[rail] = max(demand[rail], hw->map[rail][APPLE_CLPC_LEVELS - 1]); + demand[rail] = max_t(unsigned int, demand[rail], + hw->gpu_busy[APPLE_CLPC_DCS][APPLE_CLPC_LEVELS - 1] ? + hw->gpu_busy[rail][level] : + hw->map[rail][APPLE_CLPC_LEVELS - 1]); return true; } @@ -654,6 +736,10 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) u64 updated, sequence; bool active = apple_clpc_gpu_active(clpc->gpu, &updated, &sequence); + seq_printf(s, "host_gpu busy: %u permille (%s) -> level %u\n", + READ_ONCE(clpc->gpu->busy), + READ_ONCE(clpc->gpu->busy_valid) ? "reported" : "none", + apple_clpc_gpu_level(clpc->gpu)); seq_printf(s, "host_gpu: active %u request_ns %llu sequence %llu\n", active, updated, sequence); } @@ -947,12 +1033,17 @@ static const struct apple_clpc_hw apple_clpc_hw_t8140 = { static const struct apple_clpc_hw apple_clpc_hw_t6040 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, .map = { - [APPLE_CLPC_DCS] = { 1, 2, 5, 5 }, - [APPLE_CLPC_SOC] = { 0, 1, 2, 2 }, + [APPLE_CLPC_DCS] = { 1, 1, 3, 4 }, + [APPLE_CLPC_SOC] = { 0, 0, 0, 0 }, }, .sustain = 3, .gpu_demand = false, .gpu_input = true, + .gpu_busy = { + [APPLE_CLPC_DCS] = { 1, 2, 4, 5 }, + [APPLE_CLPC_SOC] = { 0, 0, 0, 2 }, + }, + .level_khz = 4512000, }; /* diff --git a/include/linux/soc/apple/clpc.h b/include/linux/soc/apple/clpc.h index 37b2cfce5594a..9475aaef14500 100644 --- a/include/linux/soc/apple/clpc.h +++ b/include/linux/soc/apple/clpc.h @@ -11,6 +11,7 @@ struct apple_clpc_gpu; #if IS_REACHABLE(CONFIG_APPLE_CLPC) int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille); void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu); #else static inline int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) @@ -19,6 +20,7 @@ static inline int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu * return device_property_present(dev, "apple,clpc") ? -ENODEV : 0; } static inline void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) {} +static inline void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) {} static inline void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) {} #endif #endif