diff --git a/drivers/gpu/drm/asahi/g17/clpc.c b/drivers/gpu/drm/asahi/g17/clpc.c index 58adaba19910f..f4d8d36c8ea8c 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.c +++ b/drivers/gpu/drm/asahi/g17/clpc.c @@ -10,6 +10,10 @@ void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) { apple_clpc_gpu_set_active(gpu, active); } +void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) +{ + apple_clpc_gpu_set_busy(gpu, permille); +} void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu) { apple_clpc_gpu_put(gpu); diff --git a/drivers/gpu/drm/asahi/g17/clpc.h b/drivers/gpu/drm/asahi/g17/clpc.h index 372820ade7eff..fa36e30ef8896 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.h +++ b/drivers/gpu/drm/asahi/g17/clpc.h @@ -6,4 +6,5 @@ struct apple_clpc_gpu; int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out); void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu); +void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille); #endif diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 250264e01ce13..1713a48f8bf49 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -252,6 +252,8 @@ pub(crate) struct Config { pub(crate) gpu_generation: hw::GpuGen, /// The firmware builds this SoC runs. pub(crate) firmware: fw::variant::Firmware, + /// Lower firmware effort for quiet GPU work; `None` runs every submission at full effort. + pub(crate) light_effort: Option, /// Layouts of the command buffers that follow the GPU's core layout. pub(crate) core_layout: CoreLayout, } @@ -929,7 +931,7 @@ impl Firmware { self.primary.state.shared.note_gpu_activity(); self.effort.submit(); self.effort.note_waiters(now_ns(), fence::waiters()); - self.apply_effort(power::FULL_EFFORT) + self.apply_effort(power::submission_effort()) } /// One governor step with the hotspot temperature reading; returns whether to run again and @@ -1579,6 +1581,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + power::configure_light(cfg.light_effort.as_ref()); let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { Some(calibration::Calibration::read(dev).inspect_err(|_| { dev_err!( diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index f9bd31f13bc7d..2c8805b4fac6d 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -529,7 +529,9 @@ impl Port { pub(super) fn note_submission(&self) -> bool { self.submissions.fetch_add(1, Ordering::SeqCst); self.note_gpu_activity(); + // Take the slow path when the governor wants a different effort. self.effort_full.load(Ordering::SeqCst) + && super::power::submission_effort() == super::power::FULL_EFFORT } pub(super) fn submissions(&self) -> u64 { @@ -564,6 +566,7 @@ impl Port { self.last_submission_ns .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); + self.demand.report_busy(super::now_ns()); } pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 8be3bae04afb0..48debe2906d1f 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,7 +3,7 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. -use core::sync::atomic::AtomicU32; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; pub(super) const FULL_EFFORT: u32 = 0x10000; pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; @@ -351,6 +351,86 @@ unsafe extern "C" { fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); + fn asahi_clpc_gpu_set_busy(gpu: *mut core::ffi::c_void, permille: core::ffi::c_uint); +} + +/// Busy-share window: long enough to span frames, short enough to follow a burst. +const BUSY_WINDOW_NS: u64 = 200_000_000; +/// A longer gap between reports was idle: start over rather than average it in. +const BUSY_WINDOW_MAX_NS: u64 = 2_000_000_000; +static BUSY_WINDOW_START: AtomicU64 = AtomicU64::new(0); +static BUSY_WINDOW_TOTAL: AtomicU64 = AtomicU64::new(0); + +/// A window under this share (permille) with no long job is quiet. +/// Consecutive quiet windows needed to go light. +const LIGHT_WINDOWS: u32 = 4; +static LIGHT: AtomicBool = AtomicBool::new(false); +static QUIET_WINDOWS: AtomicU32 = AtomicU32::new(0); + +/// A SoC's light effort for quiet GPU work. Light effort runs the same work about twice as +/// long, so the way back to full effort sits at about twice the way in, or work that fits +/// light effort sticks at full. Shares in permille of a busy window. +pub(crate) struct LightEffort { + /// Firmware performance effort, of 65536. + pub(crate) effort: u32, + /// A window below this share with no job longer than `job_enter_us` is quiet. + pub(crate) enter_permille: u32, + /// Back to full effort from this share, or from a job of `job_exit_us`. + pub(crate) exit_permille: u32, + pub(crate) job_enter_us: u32, + pub(crate) job_exit_us: u32, +} + +static LIGHT_EFFORT: AtomicU32 = AtomicU32::new(0); +static LIGHT_ENTER: AtomicU32 = AtomicU32::new(0); +static LIGHT_EXIT: AtomicU32 = AtomicU32::new(0); +static LIGHT_JOB_ENTER_US: AtomicU32 = AtomicU32::new(0); +static LIGHT_JOB_EXIT_US: AtomicU32 = AtomicU32::new(0); + +/// Takes the SoC's light effort from its configuration; `None` keeps full effort. +pub(super) fn configure_light(light: Option<&LightEffort>) { + let (effort, enter, exit, job_enter, job_exit) = light.map_or((0, 0, 0, 0, 0), |l| { + (l.effort, l.enter_permille, l.exit_permille, l.job_enter_us, l.job_exit_us) + }); + LIGHT_ENTER.store(enter, Ordering::Relaxed); + LIGHT_EXIT.store(exit, Ordering::Relaxed); + LIGHT_JOB_ENTER_US.store(job_enter, Ordering::Relaxed); + LIGHT_JOB_EXIT_US.store(job_exit, Ordering::Relaxed); + LIGHT.store(false, Ordering::Relaxed); + QUIET_WINDOWS.store(0, Ordering::Relaxed); + LIGHT_EFFORT.store(effort, Ordering::Relaxed); +} + +/// The effort for the next submission: light once the GPU has been quiet for a while. +pub(super) fn submission_effort() -> u32 { + let light = LIGHT_EFFORT.load(Ordering::Relaxed); + if LIGHT.load(Ordering::Relaxed) && light != 0 { + light + } else { + FULL_EFFORT + } +} + +fn light_param(param: &AtomicU32) -> u64 { + u64::from(param.load(Ordering::Relaxed)) +} + +fn light_step(permille: u64, longest_ns: u64) { + if LIGHT_EFFORT.load(Ordering::Relaxed) == 0 { + return; + } + if LIGHT.load(Ordering::Relaxed) { + if permille >= light_param(&LIGHT_EXIT) || longest_ns >= light_param(&LIGHT_JOB_EXIT_US) * 1000 { + LIGHT.store(false, Ordering::Relaxed); + QUIET_WINDOWS.store(0, Ordering::Relaxed); + } + } else if permille < light_param(&LIGHT_ENTER) && longest_ns < light_param(&LIGHT_JOB_ENTER_US) * 1000 { + if QUIET_WINDOWS.fetch_add(1, Ordering::Relaxed) + 1 >= LIGHT_WINDOWS { + LIGHT.store(true, Ordering::Relaxed); + } + } else { + QUIET_WINDOWS.store(0, Ordering::Relaxed); + } } /// DT-selected memory-demand provider. Its independent allocation outlives a @@ -378,6 +458,33 @@ impl Demand { // SAFETY: the retained reference is live; NULL denotes no DT provider. unsafe { asahi_clpc_gpu_set_active(self.0, active) }; } + + /// Reports the share of the last window the GPU spent executing, once a window has run. + /// Of submitters racing at the end of a window, only the one that moves its start reports. + pub(super) fn report_busy(&self, now_ns: u64) { + let start = BUSY_WINDOW_START.load(Ordering::Relaxed); + let elapsed = now_ns.saturating_sub(start); + if elapsed < BUSY_WINDOW_NS + || BUSY_WINDOW_START + .compare_exchange(start, now_ns, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + let total = crate::usage::TOTAL_NS.load(Ordering::Relaxed); + let busy = total.saturating_sub(BUSY_WINDOW_TOTAL.swap(total, Ordering::Relaxed)); + if start == 0 || elapsed > BUSY_WINDOW_MAX_NS { + return; + } + let permille = busy.saturating_mul(1000) / elapsed; + light_step(permille, crate::usage::MAX_NS.swap(0, Ordering::Relaxed)); + // Where light effort is configured, full effort does the same work in about half the + // time; report the share in light effort's terms so that one memory ladder fits both. + let light = LIGHT_EFFORT.load(Ordering::Relaxed) != 0; + let scaled = if !light || LIGHT.load(Ordering::Relaxed) { permille } else { permille * 2 }; + // SAFETY: the retained reference is live; NULL denotes no DT provider. + unsafe { asahi_clpc_gpu_set_busy(self.0, scaled.min(1000) as core::ffi::c_uint) }; + } } impl Drop for Demand { diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index f9321745cecea..c81c45998b928 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -110,6 +110,7 @@ pub(crate) const CONFIG: Config = Config { config: None, }]), // Measured on the 20-core M4 Pro; the alternative below remains experimental. + light_effort: None, core_layout: CoreLayout { cores: Some((2, 20)), // Admit two clusters of ten slots with eight enabled cores in each cluster. diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 93f1d79a8d457..e2da015335876 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -53,6 +53,14 @@ pub(super) const BASE: Config = Config { recovery_slot_keys: None, gpu_generation: GpuGen::G16, firmware: fw::variant::Firmware::Identified(&[]), + // Measured on the M4 Max only: scrolling 24.9 -> 22.1 W with the T6041 CLPC table. + light_effort: Some(crate::g17::power::LightEffort { + effort: 0x8000, + enter_permille: 350, + exit_permille: 750, + job_enter_us: 3000, + job_exit_us: 7000, + }), core_layout: CoreLayout { cores: Some((4, 40)), alternate_core_masks: None, diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index 949bd81d00edd..6f72e8bd51be4 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -72,6 +72,7 @@ pub(crate) const CONFIG: Config = Config { }]), // Two clusters of ten cores. The buffers are sized for every core slot of the board, so the // board's cluster and slot counts admit the GPU, whichever cores are fused off. + light_effort: None, core_layout: CoreLayout { cores: None, alternate_core_masks: None, @@ -143,6 +144,7 @@ pub(crate) const CONFIG_MAX: Config = Config { }]), // Four clusters of ten cores. The buffers are sized for every core slot of the board, so the // board's cluster and slot counts admit the GPU, whichever cores are fused off. + light_effort: None, core_layout: CoreLayout { cores: None, alternate_core_masks: None, @@ -229,6 +231,7 @@ pub(crate) const CONFIG_ULTRA: Config = Config { // (`fw::m5::render_registers`), so the deflake layout is the M5 Pro's. As on the M5 Pro, the // buffers are sized for all 80 core slots and the board admits the GPU by clusters and slots: // parts with fused-off cores (64 enabled) take the same layout. + light_effort: None, core_layout: CoreLayout { cores: None, alternate_core_masks: None, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 1ade3df12c2cb..46e665179ccff 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -59,6 +59,7 @@ pub(crate) const CONFIG: Config = Config { sleep_restart: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), // The layouts the A18 Pro has always run with. + light_effort: None, core_layout: crate::g17::CoreLayout { cores: None, alternate_core_masks: None, diff --git a/drivers/gpu/drm/asahi/hw/t8142.rs b/drivers/gpu/drm/asahi/hw/t8142.rs index 26fe2e49abcdb..47932cbb12cf4 100644 --- a/drivers/gpu/drm/asahi/hw/t8142.rs +++ b/drivers/gpu/drm/asahi/hw/t8142.rs @@ -48,6 +48,7 @@ pub(crate) const PROBE: Config = Config { recovery_slot_keys: None, gpu_generation: GpuGen::G17, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428_M5), + light_effort: None, core_layout: CoreLayout { cores: None, alternate_core_masks: None, diff --git a/drivers/gpu/drm/asahi/usage.rs b/drivers/gpu/drm/asahi/usage.rs index 496c722dba651..13fbe6b2a6b4a 100644 --- a/drivers/gpu/drm/asahi/usage.rs +++ b/drivers/gpu/drm/asahi/usage.rs @@ -8,6 +8,11 @@ use core::sync::atomic::{AtomicU64, Ordering}; +/// Engine time charged across all clients, in ns, for the GPU busy share. +pub(crate) static TOTAL_NS: AtomicU64 = AtomicU64::new(0); +/// Longest single stage charged since the governor last read it, in ns. +pub(crate) static MAX_NS: AtomicU64 = AtomicU64::new(0); + #[derive(Clone, Copy)] pub(crate) enum Engine { Vertex, @@ -35,6 +40,8 @@ impl Client { return; } let ns = ticks_to_ns(end - start, frequency_hz); + TOTAL_NS.fetch_add(ns, Ordering::Relaxed); + MAX_NS.fetch_max(ns, Ordering::Relaxed); let counter = &self.ns[engine as usize]; let mut old = counter.load(Ordering::Relaxed); loop { diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index ff859036c040d..8a9a77d6449da 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -76,6 +76,9 @@ enum apple_clpc_rail { #define APPLE_CLPC_REASON_HOLD BIT(4) #define APPLE_CLPC_REASON_ANE BIT(5) +/* A lower GPU busy level must last this long; a higher one applies at once. */ +#define APPLE_CLPC_GPU_DOWN_MS 400 + /** * struct apple_clpc_hw - per-SoC CLPC parameters * @rail: PMC rail of the DCS and of the fabric @@ -100,7 +103,7 @@ enum apple_clpc_rail { * measured on; SoCs sharing this CLPC without it get no cluster floor * @gpu_input: the GPU driver reports its work in flight through * apple_clpc_gpu_get(); while there is some, both rails go to the top - * of @map + * of @map, or to @gpu_busy if the driver reports a busy share * @idle_on_time: sample every period even with no vote held, instead of * from a deferrable timer * @low_power_khz: CPU frequency ceiling of the low-power platform profile, @@ -109,6 +112,18 @@ enum apple_clpc_rail { * sharing this CLPC without it get no low-power profile. NULL for any * @low_power_mw: package power (the SMC's PHPC key) the low-power profile * holds the CPU clusters to; 0 for none + * @gpu_busy: state of each rail for reported GPU work below @gpu_busy_mid, + * @gpu_busy_high and @gpu_busy_top and at or above it; all zero to always + * use the top of @map + * @gpu_busy_mid: GPU busy share in permille (apple_clpc_gpu_set_busy()) + * from which reported GPU work gets the second column of @gpu_busy + * @gpu_busy_high: busy share from which it gets the third column + * @gpu_busy_top: busy share from which it gets the top column, as does a + * GPU that reports no share. The share is time spent executing at + * whatever clock the GPU runs, not bandwidth: a 120 fps game reads + * about a third, a scrolling browser a few percent. + * @level_khz: frequency @map's quarters are taken from, so that clusters of + * different speed share them; 0 for each cluster's own maximum */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; @@ -121,6 +136,11 @@ struct apple_clpc_hw { unsigned int pmp_agent; unsigned int gpu_dense; bool gpu_input; + u8 gpu_busy[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; + unsigned int gpu_busy_mid; + unsigned int gpu_busy_high; + unsigned int gpu_busy_top; + unsigned int level_khz; u8 ane[APPLE_CLPC_RAILS]; unsigned int ane_cluster; unsigned int ane_cluster_khz; @@ -155,6 +175,11 @@ struct apple_clpc_gpu { bool active; u64 updated_ns; u64 sequence; + /* Busy share of the last window in permille, if the GPU reports one */ + unsigned int busy; + bool busy_valid; + unsigned int level; + unsigned long level_since; struct delayed_work *wake; }; @@ -262,12 +287,35 @@ void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) if (active && !gpu->active && gpu->wake) mod_delayed_work(system_freezable_power_efficient_wq, gpu->wake, 0); gpu->active = active; + /* A new burst after idle starts from the top, not from an old share. */ + if (!active) + gpu->busy_valid = false; gpu->updated_ns = ktime_get_mono_fast_ns(); gpu->sequence++; spin_unlock_irqrestore(&gpu->lock, flags); } EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_active); +/** + * apple_clpc_gpu_set_busy() - Report the GPU busy share of the last window + * @gpu: the GPU input, from apple_clpc_gpu_get() + * @permille: share of the window the GPU spent executing work, 0..1000 + * + * Scales the floor that reported GPU work gets, see &struct apple_clpc_hw.gpu_busy. + */ +void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) +{ + unsigned long flags; + + if (!gpu) + return; + spin_lock_irqsave(&gpu->lock, flags); + gpu->busy = min(permille, 1000U); + gpu->busy_valid = true; + spin_unlock_irqrestore(&gpu->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_busy); + void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) { if (gpu) { @@ -335,6 +383,36 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) } EXPORT_SYMBOL_GPL(apple_clpc_gpu_get); +/* + * The column of @hw->gpu_busy for reported GPU work, from its busy share. + * Called once per CLPC sample: a higher column applies at once, a lower one + * at the first sample at least APPLE_CLPC_GPU_DOWN_MS after the last one + * that asked for the current column or a higher one. + */ +static unsigned int apple_clpc_gpu_level(const struct apple_clpc_hw *hw, + struct apple_clpc_gpu *gpu) +{ + unsigned long flags; + unsigned int level; + + spin_lock_irqsave(&gpu->lock, flags); + if (!gpu->busy_valid || gpu->busy >= hw->gpu_busy_top) + level = APPLE_CLPC_LEVELS - 1; + else if (gpu->busy >= hw->gpu_busy_high) + level = 2; + else + level = gpu->busy >= hw->gpu_busy_mid ? 1 : 0; + if (level >= gpu->level || + time_after_eq(jiffies, gpu->level_since + + msecs_to_jiffies(APPLE_CLPC_GPU_DOWN_MS))) { + gpu->level = level; + gpu->level_since = jiffies; + } + level = gpu->level; + spin_unlock_irqrestore(&gpu->lock, flags); + return level; +} + static bool apple_clpc_gpu_active(struct apple_clpc_gpu *gpu, u64 *updated, u64 *sequence) { unsigned long flags; @@ -541,7 +619,8 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, policy = cpufreq_cpu_get(cl->policy_cpu); if (policy) { if (policy->cpuinfo.max_freq) - cl->level = policy->cur * 100 / policy->cpuinfo.max_freq; + cl->level = policy->cur * 100 / + (clpc->hw->level_khz ?: policy->cpuinfo.max_freq); cpufreq_cpu_put(policy); } @@ -718,22 +797,26 @@ static void apple_clpc_gpu_floor_count(struct apple_clpc *clpc, bool released) /* * The GPU driver's input, where it drives demand instead of the GFX power - * state: every rail at the top of @map, or at @gpu where that is higher, - * while the driver reports work. + * state: while the driver reports work, every rail at its top state, or at + * the gpu_busy state for the busy share the driver reports. */ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, struct apple_clpc_gpu *gpu, unsigned int *demand) { u64 updated, sequence; - unsigned int rail; + unsigned int level, rail; if (!gpu || hw->gpu_demand || !apple_clpc_gpu_active(gpu, &updated, &sequence)) return false; + level = apple_clpc_gpu_level(hw, gpu); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - demand[rail] = max(demand[rail], - max(hw->map[rail][APPLE_CLPC_LEVELS - 1], hw->gpu[rail])); + demand[rail] = max_t(unsigned int, demand[rail], + max_t(unsigned int, hw->gpu[rail], + hw->gpu_busy[APPLE_CLPC_DCS][APPLE_CLPC_LEVELS - 1] ? + hw->gpu_busy[rail][level] : + hw->map[rail][APPLE_CLPC_LEVELS - 1])); return true; } @@ -1019,9 +1102,21 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) READ_ONCE(clpc->last_floor), READ_ONCE(clpc->last_reason), READ_ONCE(clpc->last_vote_ns)); if (clpc->gpu) { + struct apple_clpc_gpu *gpu = clpc->gpu; + unsigned int busy, level; + unsigned long flags; u64 updated, sequence; - bool active = apple_clpc_gpu_active(clpc->gpu, &updated, &sequence); - + bool active = apple_clpc_gpu_active(gpu, &updated, &sequence); + bool busy_valid; + + /* Show the level the last sample chose; reading must not move it. */ + spin_lock_irqsave(&gpu->lock, flags); + busy = gpu->busy; + busy_valid = gpu->busy_valid; + level = gpu->level; + spin_unlock_irqrestore(&gpu->lock, flags); + seq_printf(s, "host_gpu busy: %u permille (%s) level %u\n", + busy, busy_valid ? "reported" : "none", level); seq_printf(s, "host_gpu: active %u request_ns %llu sequence %llu\n", active, updated, sequence); } @@ -1288,6 +1383,8 @@ static const struct platform_profile_ops apple_clpc_profile_ops = { .profile_set = apple_clpc_profile_set, }; +static const struct apple_clpc_hw *apple_clpc_machine_hw(const struct apple_clpc_hw *hw); + static int apple_clpc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1302,7 +1399,7 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->dev = dev; clpc->pmc = dev->parent; - clpc->hw = of_device_get_match_data(dev); + clpc->hw = apple_clpc_machine_hw(of_device_get_match_data(dev)); /* T6040 firmware keeps GFX powered for its whole lifetime. A separate * host publication/idle input, not that supplier state, drives demand; * the T8132 and M5 (T8142) GPU drivers report through the same input. @@ -1598,6 +1695,58 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { .low_power_machine = "apple,t6041", }; +/* + * T6041 (M4 Max): the M4 Pro's DCS and fabric states, with the floor scaled + * to what the work needs. The M4 Pro shares the CLPC compatible and keeps + * apple_clpc_hw_t6040; apple_clpc_probe() picks this table on an + * "apple,t6041" machine. + * + * A busy CPU cluster asks for DCS 1, 1, 3 or 4 and fabric 0. Both clusters + * take their quarters from 4512 MHz, so the E cluster at its top speed + * (57 %) gets the third column, not the fourth. Measured on a J616c with a + * 3 GB/s copy: DCS 0 / fabric 0 5.46 W at 309 ns DRAM latency, DCS 3 6.22 W + * at 166 ns, DCS 4 6.92 W at 114 ns, DCS 5 7.04 W at 113 ns; fabric 1 and 2 + * add 0.3 and 1.4 W with less gain than a DCS step. + * + * GPU work takes its states from gpu_busy by the busy share the GPU driver + * reports, DCS 5 / fabric 2 at the top or with no share reported, until + * publication/in-flight demand ends and HOLD_MS expires. Scrolling a + * browser went from 24.9 to 22.1 W on AC with this and the G17 light + * effort, against the T6040 table. + * + * The ANE keeps the T6040 rails. Its cluster floor was measured on the + * M4 Pro only and is left out here. + */ +static const struct apple_clpc_hw apple_clpc_hw_t6041 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 1, 1, 3, 4 }, + [APPLE_CLPC_SOC] = { 0, 0, 0, 0 }, + }, + .sustain = 3, + .gpu_demand = false, + .gpu_input = true, + .gpu_busy = { + [APPLE_CLPC_DCS] = { 1, 2, 4, 5 }, + [APPLE_CLPC_SOC] = { 0, 0, 0, 2 }, + }, + .gpu_busy_mid = 160, + .gpu_busy_high = 500, + .gpu_busy_top = 800, + .level_khz = 4512000, + .ane = { [APPLE_CLPC_DCS] = 5, [APPLE_CLPC_SOC] = 2 }, + .low_power_khz = 2808000, + .low_power_machine = "apple,t6041", +}; + +/* The M4 Max shares the M4 Pro's compatible but has its own table. */ +static const struct apple_clpc_hw *apple_clpc_machine_hw(const struct apple_clpc_hw *hw) +{ + if (hw == &apple_clpc_hw_t6040 && of_machine_is_compatible("apple,t6041")) + return &apple_clpc_hw_t6041; + return hw; +} + /* * T8142: DCS states 0-7 on rail 0, fabric states 0-4 on rail 1. The fabric * state bounds CPU memory bandwidth: four P cores copy at ~40 GB/s with the diff --git a/include/linux/soc/apple/clpc.h b/include/linux/soc/apple/clpc.h index 0a4c4f439ac56..5f83b2aedc609 100644 --- a/include/linux/soc/apple/clpc.h +++ b/include/linux/soc/apple/clpc.h @@ -12,6 +12,7 @@ struct apple_clpc_ane; #if IS_REACHABLE(CONFIG_APPLE_CLPC) int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille); void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu); /* Optional, single-publisher input: success with NULL means no provider. */ int apple_clpc_ane_get(struct device *dev, struct apple_clpc_ane **out); @@ -24,6 +25,7 @@ static inline int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu * return device_property_present(dev, "apple,clpc") ? -ENODEV : 0; } static inline void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) {} +static inline void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) {} static inline void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) {} /* ANE remains usable when the optional floor provider is not reachable. */ diff --git a/tools/asahi/clpc-gpu-demand-tests.py b/tools/asahi/clpc-gpu-demand-tests.py index 2e852e6f91643..08f7be9e58085 100644 --- a/tools/asahi/clpc-gpu-demand-tests.py +++ b/tools/asahi/clpc-gpu-demand-tests.py @@ -26,10 +26,12 @@ def main(): rail_shift = re.search(r'^#define APPLE_PMC_RAIL_SHIFT\(rail\).*$', header, re.M).group(0) dense_samples = re.search(r'^#define APPLE_CLPC_GPU_DENSE_SAMPLES\s.*$', source, re.M).group(0) reasons = '\n'.join(re.findall(r'^#define APPLE_CLPC_REASON_\w+\s.*$', source, re.M)) + gpu_down = re.search(r'^#define APPLE_CLPC_GPU_DOWN_MS\s.*$', source, re.M).group(0) types = '\n'.join(item(source, s) + ';' for s in [ 'struct apple_clpc_hw {', 'struct apple_clpc_vote {']) tables = '\n'.join(item(source, 'static const struct apple_clpc_hw ' + n) + ';' - for n in ['apple_clpc_hw_t8132', 'apple_clpc_hw_t8140', 'apple_clpc_hw_t6040']) + for n in ['apple_clpc_hw_t8132', 'apple_clpc_hw_t8140', 'apple_clpc_hw_t6040', + 'apple_clpc_hw_t6041']) methods = '\n'.join(item(source, s) for s in [ 'static unsigned int apple_clpc_vote(', 'static u32 apple_clpc_floor(', 'static bool apple_clpc_gpu_streaming(', @@ -37,6 +39,7 @@ def main(): 'static bool apple_clpc_gpu_dense(', 'static bool apple_clpc_gpu_limited(', 'static void apple_clpc_gpu_floor_count(', + 'static unsigned int apple_clpc_gpu_level(', 'static bool apple_clpc_gpu_input_demand(', 'static bool apple_clpc_ane_input_demand(', 'static void apple_clpc_ane_cluster(', @@ -56,6 +59,7 @@ def main(): #define APPLE_CLPC_GPU_BACKOFF_MS 100 #define HZ 1000 #define max(a,b) ((a) > (b) ? (a) : (b)) +#define max_t(t,a,b) ((t)(a) > (t)(b) ? (t)(a) : (t)(b)) #define GENMASK(h,l) (((1u << ((h)-(l)+1))-1) << (l)) #define BIT(n) (1u << (n)) #define APPLE_PMC_RAIL_SHIFT(r) ((r)*8) @@ -67,7 +71,10 @@ def main(): #define dev_warn_once(d, ...) ((void)(d), warned++) enum apple_clpc_rail { APPLE_CLPC_DCS, APPLE_CLPC_SOC, APPLE_CLPC_RAILS }; ''' + types + r''' -struct apple_clpc_gpu { bool active; u64 sequence; }; +struct apple_clpc_gpu { + int lock; bool active; u64 sequence; + unsigned int busy; bool busy_valid; unsigned int level; unsigned long level_since; +}; struct apple_clpc_ane { int lock; bool active, pending; }; #define spin_lock_irqsave(l, f) ((void)(l), (f) = 0) #define spin_unlock_irqrestore(l, f) ((void)(l), (void)(f)) @@ -134,7 +141,7 @@ def main(): static void apple_clpc_queue(struct apple_clpc *c, bool idle) { (void)c; queued_idle=idle; } #define READ_ONCE(x) (x) static void apple_clpc_profile_limit(struct apple_clpc *c) { (void)c; } -''' + dense_samples + '\n' + reasons + '\n' + tables + '\n' + methods + r''' +''' + dense_samples + '\n' + reasons + '\n' + gpu_down + '\n' + tables + '\n' + methods + r''' // T8140 ANE work holds its DRAM and fabric states while it runs, a request // shorter than one sample is still seen once, and the hold then expires. static void check_t8140_ane(void) { @@ -171,6 +178,46 @@ def main(): ane.active=true; jiffies=2040; apple_clpc_sample(&c); assert(qos_adds==1 && qos_min==3756000); } +// T6041 GPU work takes its floor from the busy share the driver reports: +// DCS 1, 2, 4 or 5 and fabric 0, 0, 0 or 2, split at 160, 500 and 800 +// permille. A higher level applies at once, a lower one at the first sample +// at least 400 ms after the last one that asked for the current level or more. +static void check_t6041_busy(void) { + struct apple_clpc_gpu gpu={0}; + struct apple_clpc c={.hw=&apple_clpc_hw_t6041,.agent=1,.mask=3,.gpu=&gpu}; + assert(c.hw->gpu_busy_mid==160 && c.hw->gpu_busy_high==500 && c.hw->gpu_busy_top==800); + read_error=vote_error=0; pmgr_status=0xf0; pmp_floor=0; published=~0u; + // No share reported: the top of the table, as before the busy share. + gpu.active=true; jiffies=3000; apple_clpc_sample(&c); + assert(published==0x25 && c.last_reason==2 && gpu.level==3); + gpu.active=false; jiffies=3100; apple_clpc_sample(&c); assert(published==0); + // Below gpu_busy_mid: the lowest GPU state (after a long idle, at once). + gpu.active=true; gpu.busy_valid=true; gpu.busy=100; jiffies=3500; + apple_clpc_sample(&c); assert(published==0x01 && c.last_reason==2 && gpu.level==0); + // Between the thresholds, going up: each level applies at the next sample. + gpu.busy=160; jiffies=3504; apple_clpc_sample(&c); assert(published==0x02 && gpu.level==1); + gpu.busy=499; jiffies=3508; apple_clpc_sample(&c); assert(published==0x02 && gpu.level==1); + gpu.busy=500; jiffies=3512; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + gpu.busy=799; jiffies=3516; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + gpu.busy=800; jiffies=3520; apple_clpc_sample(&c); assert(published==0x25 && gpu.level==3); + // Going down waits 400 ms from the last sample at the top. + gpu.busy=100; + for (jiffies=3524; jiffies<3920; jiffies+=4) { + apple_clpc_sample(&c); assert(published==0x25 && gpu.level==3); + } + jiffies=3919; apple_clpc_sample(&c); assert(published==0x25 && gpu.level==3); + // At 400 ms the DCS drops at once; the fabric state keeps HOLD_MS. + jiffies=3920; apple_clpc_sample(&c); assert(published==0x21 && gpu.level==0); + jiffies=3952; apple_clpc_sample(&c); assert(published==0x01 && c.last_reason==2); + // A sample that asks for the current level again restarts the 400 ms. + gpu.busy=600; jiffies=4000; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + gpu.busy=100; jiffies=4200; apple_clpc_sample(&c); assert(gpu.level==2); + gpu.busy=550; jiffies=4300; apple_clpc_sample(&c); assert(gpu.level==2); + gpu.busy=100; jiffies=4699; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + jiffies=4700; apple_clpc_sample(&c); assert(published==0x01 && gpu.level==0); + gpu.active=false; jiffies=4800; apple_clpc_sample(&c); assert(published==0 && queued_idle); +} + // T8132's G16 firmware keeps graphics powered while the GPU idles: the // M5/F3 floor follows the work the GPU driver reports, as on T6040. static void check_t8132(void) { @@ -268,13 +315,14 @@ def main(): assert(published==0x12 && c.last_reason==1); c.clusters[0].request[0]=c.clusters[0].request[1]=0; jiffies=264; apple_clpc_sample(&c); assert(published==0); + check_t6041_busy(); check_t8132(); check_read_error(); check_t8140(); check_t8140_dense(); check_t8140_ane(); check_t6040_ane(); - puts("PASS: T6040 and T8132 host demand; T8140 ANE, streaming, dense work, read errors and PMP backoff; idle release"); + puts("PASS: T6040 and T8132 host demand; T6041 GPU busy share and down-step hold; T8140 ANE, streaming, dense work, read errors and PMP backoff; idle release"); } ''' code = code.replace('#define APPLE_PMC_RAIL_SHIFT(r) ((r)*8)', rail_shift) @@ -299,6 +347,11 @@ def main(): ('omit-pmp-backoff', code.replace('bool released = apple_clpc_gpu_limited(clpc, now);', 'bool released = (apple_clpc_gpu_limited(clpc, now), false);')), ('release-too-early', code.replace('msecs_to_jiffies(APPLE_CLPC_HOLD_MS)', '0')), + ('gpu-down-at-once', code.replace('msecs_to_jiffies(APPLE_CLPC_GPU_DOWN_MS)', '0')), + ('gpu-busy-ignored', code.replace('if (!gpu->busy_valid || gpu->busy >= hw->gpu_busy_top)', + 'if (true || gpu->busy >= hw->gpu_busy_top)')), + ('gpu-no-share-uses-table', code.replace('if (!gpu->busy_valid || gpu->busy >= hw->gpu_busy_top)', + 'if (gpu->busy >= hw->gpu_busy_top)')), ('retain-at-idle', code.replace('vote->state = 0;', '')), ]: # A mutation that no longer matches the driver tests nothing.