diff --git a/arch/arm64/boot/dts/apple/t8132-j713-gpu.dtsi b/arch/arm64/boot/dts/apple/t8132-j713-gpu.dtsi index 3fa3db191e805..50cd6807d7735 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713-gpu.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-j713-gpu.dtsi @@ -72,6 +72,7 @@ /* G16 is admitted on non-coherent DMA only. */ dma-noncoherent; apple,pmp = <&pmp>; + apple,clpc = <&clpc>; memory-region = <&gpu_ttbs>, <&gpu_pagetables>, <&gpu_handoff>, <&gpu_shared_l2>, <&gpu_fw_text>, <&gpu_fw_data>; memory-region-names = "ttbs", "pagetables", "handoff", diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 647f8f37507d4..6b28266b6b4d1 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -32,6 +32,7 @@ config DRM_ASAHI_G16 bool "Apple M4 (G16G) support (experimental)" depends on DRM_ASAHI && ARM64 depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI + depends on APPLE_CLPC=y || APPLE_CLPC=DRM_ASAHI || APPLE_CLPC=n depends on THERMAL_OF help Rust backend for the qualified J713/T8132 GPU firmware. diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index f53293578316f..dee03ab92fab0 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -855,6 +855,9 @@ impl Firmware { if !self.engine.chan_wait { self.recovery.publication_wait_deadline = None; } + // Polling retires before it publishes, so this sees work start and + // drain; a fail-stop reports from fail_flights(). + self.power.report_work(!self.engine.flights.is_empty()); result } @@ -1358,5 +1361,7 @@ impl Firmware { s.starved_ns += s.pub_ns.wrapping_sub(s.starve_from_ns); } s.starve_from_ns = 0; + let now = s.pub_ns; + self.power.utilization.busy(now); } } diff --git a/drivers/gpu/drm/asahi/g16/glue.c b/drivers/gpu/drm/asahi/g16/glue.c index db0576a1e0fe0..07403c7324246 100644 --- a/drivers/gpu/drm/asahi/g16/glue.c +++ b/drivers/gpu/drm/asahi/g16/glue.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,9 @@ void asahi_g16_cooling_unregister(struct asahi_g16_cooling *cooling); int asahi_g16_cooling_sample(struct asahi_g16_cooling *cooling, u64 now, long *temperature, u32 *governor, u32 *power, u32 *backstop); +int asahi_g16_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); +void asahi_g16_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void asahi_g16_clpc_gpu_put(struct apple_clpc_gpu *gpu); static int asahi_g16_cooling_max(struct thermal_cooling_device *cdev, unsigned long *state) @@ -261,3 +265,32 @@ int asahi_g16_power_domain_state(struct device *dev, u32 *offset, u32 *state) return PTR_ERR(pmgr); return regmap_read(pmgr, *offset, state); } + +/* + * The CLPC holding the memory floor for GPU work is a performance aid: the GPU + * works without one, only slower. A kernel without the CLPC runs without it; + * one that has not probed yet defers the GPU, with the reason recorded. + */ +int asahi_g16_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +{ + int ret = apple_clpc_gpu_get(dev, out); + + if (ret == -ENODEV) { + dev_warn(dev, "no CLPC: GPU work gets no memory floor\n"); + *out = NULL; + return 0; + } + if (ret) + return dev_err_probe(dev, ret, "no CLPC for GPU work\n"); + return 0; +} + +void asahi_g16_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) +{ + apple_clpc_gpu_set_active(gpu, active); +} + +void asahi_g16_clpc_gpu_put(struct apple_clpc_gpu *gpu) +{ + apple_clpc_gpu_put(gpu); +} diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index b94fb3a4cd36f..eab58fcac8afd 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -79,6 +79,16 @@ impl Shared { let evented = self.evented.swap(false, Ordering::AcqRel); match firmware.poll(evented, &mut done) { Ok(true) => self.rearm(firmware.engine.chan_wait), + // Idle: one more pass after a window gives the state back. A failed + // engine's poll no longer services power, so it has nothing to give back. + Ok(false) + if firmware.recovery.failed().is_none() + && firmware.power.utilization.decaying() => + { + self.mod_poll(kernel::time::msecs_to_jiffies( + (super::power::policy::WINDOW_NS / 1_000_000) as u32, + )) + } Ok(false) => {} Err(error) => { firmware.fail_stop(error, &mut done); @@ -535,6 +545,7 @@ impl Firmware { if self.engine.flights.is_empty() { // Drained: a gap until the next publication. self.engine.stats.idle_from_ns = now_ns(); + self.power.utilization.idle(self.engine.stats.idle_from_ns); } // The next prefix starts executing about now. self.renew_deadlines(); @@ -861,9 +872,13 @@ impl Firmware { super::recovery::logic::FIRMWARE_STAMP_PERIOD, self.engine.ordinal, self.engine.draw)?; writeln!( f, - "power: governor {} budget {} effective S{} temperature_mc {} updates {} sensor_errors {}", - self.power.governor, self.power.budget, self.power.effective, - self.power.temperature_mc, self.power.updates, self.power.sensor_errors + "power: governor {} budget {} demand S{} utilization {} effective S{} temperature_mc {} updates {} sensor_errors {}", + self.power.governor, + self.power.budget, + self.power.utilization.demand, + self.power.utilization.average, + self.power.effective, self.power.temperature_mc, self.power.updates, + self.power.sensor_errors )?; writeln!( f, diff --git a/drivers/gpu/drm/asahi/g16/power.rs b/drivers/gpu/drm/asahi/g16/power.rs index c6738f4bc4ac2..9c23f8688e41e 100644 --- a/drivers/gpu/drm/asahi/g16/power.rs +++ b/drivers/gpu/drm/asahi/g16/power.rs @@ -1,11 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! PMP-owned cooling requests and retained-session performance ceiling. +//! PMP-owned cooling requests, retained-session performance ceiling and the +//! utilization governor. //! //! The firmware owns voltage and clock transitions through PMP. The host //! consumes PMP-owner telemetry and publishes only three Globals selectors, //! in hundredths of a performance state. Firmware power tables and policy //! words from other GPU generations are never modified here. +//! +//! The firmware runs at the published state, busy or not, so the host requests +//! one from the GPU's busy share of each window, bounded by the ceiling. +//! +//! The firmware keeps GFX powered while the GPU idles, so the host reports +//! when it has work in flight to the DT-selected CLPC, which holds the memory +//! and fabric floor for that work. pub(crate) mod policy; @@ -23,6 +31,9 @@ static_assert!(globals::PSTATE_SELECTORS[1] == 0x8c); static_assert!(globals::PSTATE_SELECTORS[2] == 0x98); extern "C" { + fn asahi_g16_clpc_gpu_get(dev: *mut c_void, out: *mut *mut c_void) -> core::ffi::c_int; + fn asahi_g16_clpc_gpu_set_active(gpu: *mut c_void, active: bool); + fn asahi_g16_clpc_gpu_put(gpu: *mut c_void); fn asahi_g16_cooling_register(dev: *mut c_void) -> *mut c_void; fn asahi_g16_cooling_unregister(cooling: *mut c_void); fn asahi_g16_cooling_sample( @@ -96,9 +107,49 @@ struct Sample { backstop: u32, } +/// DT-selected memory-demand provider (`apple,clpc`), or none. Its +/// independent allocation outlives a detached provider; a device link orders +/// supplier removal and suspend. +struct Demand(*mut c_void); + +// SAFETY: the provider's leaf spinlock serializes all access; no GPU callback. +unsafe impl Send for Demand {} +// SAFETY: shared access only changes that locked request, never its lifetime. +unsafe impl Sync for Demand {} + +impl Demand { + fn new(dev: &device::Device) -> Result { + let mut out = core::ptr::null_mut(); + // SAFETY: live platform device and writable result; C returns an owned + // reference, or NULL without an `apple,clpc` provider. + let ret = unsafe { asahi_g16_clpc_gpu_get(dev.as_raw().cast(), &mut out) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(Self(out)) + } + } + + fn set_active(&self, active: bool) { + // SAFETY: the retained reference is live; NULL denotes no provider. + unsafe { asahi_g16_clpc_gpu_set_active(self.0, active) }; + } +} + +impl Drop for Demand { + fn drop(&mut self) { + // SAFETY: releases exactly the reference returned by get, clearing + // the demand first. + unsafe { asahi_g16_clpc_gpu_put(self.0) }; + } +} + /// All fields are protected by the device mutex. Before the first valid /// provider sample the shared cooling policy keeps the ceiling at S1. pub(crate) struct Power { + clpc: Demand, + /// The work-in-flight state last reported to `clpc`. + reported: Option, cooling: Cooling, sample: Sample, last_sample_ns: Option, @@ -113,6 +164,7 @@ pub(crate) struct Power { pub(crate) sensor_errors: u64, pub(crate) decisions: u64, last_decision: Option<(u32, u32, u32, u32, Option)>, + pub(crate) utilization: policy::Governor, } impl Power { @@ -120,6 +172,9 @@ impl Power { /// startup S1 before INIT; this object starts with that same effective state. pub(crate) fn new(dev: &device::Device) -> Result { Ok(Self { + // First: a CLPC that has not probed yet defers before the PMP vote. + clpc: Demand::new(dev)?, + reported: None, cooling: Cooling::new(dev)?, sample: Sample { temperature_mc: 0, @@ -139,9 +194,19 @@ impl Power { sensor_errors: 0, decisions: 0, last_decision: None, + utilization: policy::Governor::new(), }) } + /// Reports a change of work in flight to the CLPC: the memory and fabric + /// floor follows it, not the GFX power state. + pub(crate) fn report_work(&mut self, in_flight: bool) { + if self.reported != Some(in_flight) { + self.clpc.set_active(in_flight); + self.reported = Some(in_flight); + } + } + fn ceiling(&mut self, now: u64) -> (u32, bool) { if policy::read_due(self.last_sample_ns, now) { self.last_sample_ns = Some(now); @@ -184,6 +249,10 @@ impl Firmware { } let now = now_ns(); let (limit, decision) = self.power.ceiling(now); + let target = self + .power + .utilization + .govern(now, limit, !self.engine.flights.is_empty()); if decision { dev_info!( self.dev(), @@ -197,15 +266,15 @@ impl Firmware { limit ); } - if limit == self.power.effective { + if target == self.power.effective { return Ok(()); } let globals = self.init.globals.as_ref().ok_or(ENODEV)?; for offset in globals::PSTATE_SELECTORS { - globals.w32(offset, limit * 100)?; + globals.w32(offset, target * 100)?; } wmb(); - self.power.effective = limit; + self.power.effective = target; self.power.updates += 1; Ok(()) } diff --git a/drivers/gpu/drm/asahi/g16/power/policy.rs b/drivers/gpu/drm/asahi/g16/power/policy.rs index 598ce43fa65dc..0d127b8a5f817 100644 --- a/drivers/gpu/drm/asahi/g16/power/policy.rs +++ b/drivers/gpu/drm/asahi/g16/power/policy.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Composition and sampling cadence for the retained firmware ceiling. +//! Composition and sampling cadence for the retained firmware ceiling, and the +//! utilization governor below it. //! //! Temperature decoding and regulation belong to the PMP provider and the //! shared C cooling policy. This module defines no temperature thresholds. @@ -8,6 +9,11 @@ pub(crate) const PSTATE_MAX: u32 = 9; pub(crate) const COOLING_MAX: u32 = 8; pub(crate) const SAMPLE_NS: u64 = 5_000_000; +/// Utilization window of the governor. +pub(crate) const WINDOW_NS: u64 = 20_000_000; +/// Lowest governed state. On the J713 S1-S3 draw under 0.1 W for a desktop +/// load and S4 several times that; S3 runs at 2.4 times the clock of S1. +pub(crate) const FLOOR: u32 = 3; /// Completion polling must not turn into a PMP MMIO read on every pass. pub(crate) fn read_due(last: Option, now: u64) -> bool { @@ -20,6 +26,111 @@ pub(crate) fn limit(configured: u32, ceiling: u32, user: u32) -> u32 { ceiling.min(PSTATE_MAX).min(configured).min(user).max(1) } +/// Percentage of `window` ns the GPU was busy for. +pub(crate) fn utilization(busy: u64, window: u64) -> u32 { + if window == 0 { + 100 + } else { + let busy = busy.min(window); + // No 128-bit division in the kernel; a window near u64::MAX loses nothing. + let util = match busy.checked_mul(100) { + Some(scaled) => scaled / window, + None => busy / (window / 100), + }; + util.min(100) as u32 + } +} + +/// Utilization averaged over about four windows; a longer gap restarts it. +pub(crate) fn smooth(average: u32, util: u32, window: u64) -> u32 { + if window >= 4 * WINDOW_NS { + util + } else { + (3 * average.min(100) + util.min(100)) / 4 + } +} + +/// The state to request from the `current` published one: up on the last +/// window's utilization `util`, to the top state under heavy load, and down +/// on the smoothed `average`, to the floor when idle. +pub(crate) fn demand(current: u32, util: u32, average: u32) -> u32 { + let next = if util >= 70 { + PSTATE_MAX + } else if util >= 40 { + current.saturating_add(1) + } else if average < 10 { + FLOOR + } else if average < 25 { + current.saturating_sub(1) + } else { + current + }; + next.clamp(FLOOR, PSTATE_MAX) +} + +/// Busy-time accounting over the governor's windows. The engine reports each +/// publication and drain; each power service closes the window once it is due. +pub(crate) struct Governor { + /// The requested state and the smoothed utilization. + pub(crate) demand: u32, + pub(crate) average: u32, + window_ns: u64, + busy_since: Option, + busy_ns: u64, +} + +impl Governor { + pub(crate) const fn new() -> Self { + Self { + demand: FLOOR, + average: 0, + window_ns: 0, + busy_since: None, + busy_ns: 0, + } + } + + /// The engine published work; it stays busy until [`Governor::idle`]. + pub(crate) fn busy(&mut self, now: u64) { + self.busy_since.get_or_insert(now); + } + + /// The engine drained. + pub(crate) fn idle(&mut self, now: u64) { + if let Some(since) = self.busy_since.take() { + self.busy_ns += now.saturating_sub(since); + } + } + + /// Whether an idle engine still has a state above the floor to give back. + /// Its poll comes back once a window while this holds. + pub(crate) fn decaying(&self) -> bool { + self.busy_since.is_none() && self.demand > FLOOR + } + + /// Closes the window once it is due and returns the state to publish + /// under the ceiling `limit`; `busy` is whether flights are outstanding. + pub(crate) fn govern(&mut self, now: u64, limit: u32, busy: bool) -> u32 { + // Recovery can drop flights without a drain. + if !busy { + self.idle(now); + } + let window = now.wrapping_sub(self.window_ns); + if window >= WINDOW_NS { + if let Some(since) = self.busy_since.as_mut() { + self.busy_ns += now.saturating_sub(*since); + *since = now; + } + let util = utilization(self.busy_ns, window); + self.average = smooth(self.average, util, window); + self.demand = demand(self.demand.min(limit), util, self.average); + self.window_ns = now; + self.busy_ns = 0; + } + self.demand.min(limit) + } +} + #[cfg(test)] mod tests { use super::*; @@ -48,4 +159,171 @@ mod tests { } } } + + #[test] + fn utilization_is_a_bounded_share_of_the_window() { + let w = WINDOW_NS; + assert_eq!(utilization(0, w), 0); + assert_eq!(utilization(w / 4, w), 25); + assert_eq!(utilization(w, w), 100); + assert_eq!(utilization(2 * w, w), 100); + assert_eq!(utilization(0, 0), 100); + assert_eq!(utilization(u64::MAX, u64::MAX), 100); + } + + #[test] + fn the_average_decays_and_restarts_after_a_gap() { + let w = WINDOW_NS; + assert_eq!(smooth(100, 0, w), 75); + assert_eq!(smooth(0, 100, w), 25); + assert_eq!(smooth(100, 0, 4 * w), 0); + assert_eq!(smooth(u32::MAX, u32::MAX, w), 100); + let mut average = 100; + let mut windows = 0; + while demand(PSTATE_MAX, 0, average) != FLOOR { + average = smooth(average, 0, w); + windows += 1; + } + assert!(windows <= 10); + } + + #[test] + fn demand_rises_fast_and_falls_on_the_average() { + assert_eq!(demand(FLOOR, 100, 0), PSTATE_MAX); + assert_eq!(demand(FLOOR, 70, 0), PSTATE_MAX); + assert_eq!(demand(4, 50, 0), 5); + assert_eq!(demand(PSTATE_MAX, 50, 0), PSTATE_MAX); + // A short dip under a busy average holds the state. + assert_eq!(demand(PSTATE_MAX, 5, 80), PSTATE_MAX); + assert_eq!(demand(6, 30, 30), 6); + assert_eq!(demand(6, 20, 20), 5); + assert_eq!(demand(PSTATE_MAX, 0, 5), FLOOR); + // A light 60 fps load near 25% busy settles on the floor. + assert_eq!(demand(FLOOR, 25, 24), FLOOR); + assert_eq!(demand(1, 30, 30), FLOOR); + for current in [0, 1, FLOOR, 7, PSTATE_MAX, u32::MAX] { + for util in [0, 20, 50, 70, 100, u32::MAX] { + for average in [0, 9, 20, 30, 100, u32::MAX] { + let next = demand(current, util, average); + assert!((FLOOR..=PSTATE_MAX).contains(&next)); + } + } + } + } + + const T: u64 = 1_000_000_000; + + /// A governor whose first window closed idle at `T`. + fn started() -> Governor { + let mut governor = Governor::new(); + assert_eq!(governor.govern(T, PSTATE_MAX, false), FLOOR); + assert_eq!(governor.average, 0); + governor + } + + #[test] + fn busy_time_runs_from_publication_to_drain() { + let w = WINDOW_NS; + let mut g = started(); + // A second publication before the drain does not restart the span. + g.busy(T + 1_000_000); + g.busy(T + 3_000_000); + g.idle(T + 6_000_000); + // A drain without a publication adds nothing. + g.idle(T + 9_000_000); + // A service before the window is due changes nothing. + assert_eq!(g.govern(T + w - 1, PSTATE_MAX, false), FLOOR); + assert_eq!(g.average, 0); + // 5 ms of 20 ms: 25%, a quarter of it in the average. + assert_eq!(g.govern(T + w, PSTATE_MAX, false), FLOOR); + assert_eq!(g.average, 6); + } + + #[test] + fn a_busy_engine_counts_to_each_window_end() { + let w = WINDOW_NS; + let mut g = started(); + g.busy(T); + assert!(!g.decaying()); + // Busy through the window: all of it counts, and the load gets the top state. + assert_eq!(g.govern(T + w, PSTATE_MAX, true), PSTATE_MAX); + assert_eq!(g.average, 25); + // Busy above the floor: the engine's own polls govern it, not the idle re-arm. + assert!(!g.decaying()); + // The span carries on from the window's end, not from its publication. + g.idle(T + w + w / 2); + assert_eq!(g.govern(T + 2 * w, PSTATE_MAX, false), PSTATE_MAX); + assert_eq!(g.average, (3 * 25 + 50) / 4); + } + + #[test] + fn flights_dropped_without_a_drain_end_the_busy_span() { + let w = WINDOW_NS; + let mut g = started(); + g.busy(T); + // Recovery dropped the flights and reported no drain: the next service, + // finding none outstanding, ends the span there. + assert_eq!(g.govern(T + w / 2, PSTATE_MAX, false), FLOOR); + assert!(!g.decaying()); + assert_eq!(g.govern(T + w, PSTATE_MAX, false), FLOOR + 1); + assert_eq!(g.average, 50 / 4); + // Idle above the floor: the poll keeps coming back. + assert!(g.decaying()); + // A late drain report counts nothing more. + g.idle(T + w + 1); + assert_eq!(g.govern(T + 2 * w, PSTATE_MAX, false), FLOOR); + assert!(!g.decaying()); + } + + #[test] + fn an_idle_engine_is_polled_back_to_the_floor_and_then_left_alone() { + let w = WINDOW_NS; + let mut g = started(); + g.busy(T); + for n in 1..=8 { + assert_eq!(g.govern(T + n * w, PSTATE_MAX, true), PSTATE_MAX); + } + let mut now = T + 8 * w; + g.idle(now); + // As in run_poll: an idle pass re-arms one window later while decaying. + let mut passes = 0; + while g.decaying() { + now += w; + g.govern(now, PSTATE_MAX, false); + passes += 1; + assert!(passes <= 10); + } + assert_eq!(g.demand, FLOOR); + assert!(passes > 1); + // At the floor nothing re-arms, and a later service holds it there. + assert_eq!(g.govern(now + 10 * w, PSTATE_MAX, false), FLOOR); + assert!(!g.decaying()); + // A long sleep between passes restarts the average and drops at once. + let mut g = started(); + g.busy(T); + assert_eq!(g.govern(T + w, PSTATE_MAX, true), PSTATE_MAX); + g.idle(T + w); + assert_eq!(g.govern(T + 6 * w, PSTATE_MAX, false), FLOOR); + assert!(!g.decaying()); + } + + #[test] + fn ceilings_bound_the_published_state_even_below_the_floor() { + let w = WINDOW_NS; + let mut g = started(); + g.busy(T); + // A thermal or user ceiling under load holds the request to it. + assert_eq!(g.govern(T + w, 5, true), 5); + // A ceiling below the floor is published as it is. + assert_eq!(g.govern(T + 2 * w, 1, true), 1); + assert_eq!(g.govern(T + 2 * w + 1, 2, true), 2); + // Released, the load's request applies at once. + assert_eq!(g.govern(T + 2 * w + 2, PSTATE_MAX, true), PSTATE_MAX); + // Idle under a ceiling below the floor: one window gives it all back. + g.idle(T + 3 * w); + assert!(g.decaying()); + assert_eq!(g.govern(T + 4 * w, 1, false), 1); + assert_eq!(g.demand, FLOOR); + assert!(!g.decaying()); + } } diff --git a/drivers/gpu/drm/asahi/g16/recovery.rs b/drivers/gpu/drm/asahi/g16/recovery.rs index e15590048dacb..b0fc2ed96976c 100644 --- a/drivers/gpu/drm/asahi/g16/recovery.rs +++ b/drivers/gpu/drm/asahi/g16/recovery.rs @@ -253,6 +253,8 @@ impl Firmware { /// flights are retained whole, never released to the free slot lists. pub(crate) fn fail_flights(&mut self, error: Error, done: &mut Completions) { let flights = core::mem::take(&mut self.engine.flights); + // A failed device never publishes again: release the memory floor now. + self.power.report_work(false); for flight in flights.iter() { for job in Iterator::chain(core::iter::once(&flight.job), flight.group.iter()) { push_completion(done, job, Err(error)); diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 25b28c0522fe7..ae0d6d48bac7f 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -227,6 +227,7 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) if (!np) return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; if (!of_device_is_compatible(np, "apple,t6041-clpc") && + !of_device_is_compatible(np, "apple,t8132-clpc") && !of_device_is_compatible(np, "apple,t8140-clpc")) { of_node_put(np); return -EINVAL; @@ -517,6 +518,25 @@ static void apple_clpc_gpu_floor_count(struct apple_clpc *clpc, bool released) clpc->gpu_floor_released = released; } +/* + * The GPU driver's input, where it drives demand instead of the GFX power + * state: every rail at its top state while the driver reports work. + */ +static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, + struct apple_clpc_gpu *gpu, + unsigned int *demand) +{ + u64 updated, sequence; + unsigned int rail; + + if (!gpu || hw->gpu_demand || + !apple_clpc_gpu_active(gpu, &updated, &sequence)) + return false; + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + demand[rail] = max(demand[rail], hw->map[rail][APPLE_CLPC_LEVELS - 1]); + return true; +} + static void apple_clpc_sample(struct apple_clpc *clpc) { unsigned int demand[APPLE_CLPC_RAILS] = {}; @@ -542,16 +562,8 @@ static void apple_clpc_sample(struct apple_clpc *clpc) for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) if (demand[rail]) reason |= 1; - if (clpc->gpu && !clpc->hw->gpu_demand) { - u64 updated, sequence; - - if (apple_clpc_gpu_active(clpc->gpu, &updated, &sequence)) { - reason |= 2; - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - demand[rail] = max(demand[rail], - clpc->hw->map[rail][APPLE_CLPC_LEVELS - 1]); - } - } + if (apple_clpc_gpu_input_demand(clpc->hw, clpc->gpu, demand)) + reason |= 2; if (clpc->gpu_pmgr) { /* Both run every sample: each tracks state across samples. */ bool late = time_after(now, clpc->gpu_sampled + @@ -733,9 +745,11 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->hw = of_device_get_match_data(dev); /* T6040 firmware keeps GFX powered for its whole lifetime. A separate * host publication/idle input, not that supplier state, drives demand. - * On T8140 the same input counts submissions for dense light work. + * On T8140 the same input counts submissions for dense light work. The + * T8132 GPU driver reports its work through it as well. */ if (of_device_is_compatible(dev->of_node, "apple,t6041-clpc") || + of_device_is_compatible(dev->of_node, "apple,t8132-clpc") || clpc->hw->gpu_dense) { clpc->gpu = kzalloc_obj(*clpc->gpu); if (!clpc->gpu) @@ -853,6 +867,13 @@ static DEFINE_SIMPLE_DEV_PM_OPS(apple_clpc_pm_ops, apple_clpc_suspend, /* Only the stock M5/F3 busy floor is qualified on T8132. Do not invent * intermediate frequency-to-memory tables; idle can still release the vote. + * + * The G16 firmware keeps graphics powered while the GPU idles, so its power + * state would hold the floor up all the time: on J713 that keeps the DCS and + * the fabric at their top states at an idle desktop, about 0.45 W more than + * with the floor released. The GPU driver reports its work instead, as on + * T6040, and gets the M5/F3 floor while it has work in flight. Without that + * floor, rendering is about 45% slower. */ static const struct apple_clpc_hw apple_clpc_hw_t8132 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, @@ -860,8 +881,7 @@ static const struct apple_clpc_hw apple_clpc_hw_t8132 = { [APPLE_CLPC_DCS] = { 5, 5, 5, 5 }, [APPLE_CLPC_SOC] = { 3, 3, 3, 3 }, }, - .gpu_demand = true, - .gpu = { [APPLE_CLPC_DCS] = 5, [APPLE_CLPC_SOC] = 3 }, + .gpu_demand = false, }; /* @@ -1040,7 +1060,50 @@ static void apple_clpc_gpu_dense_test(struct kunit *test) sequence, 0, false)); } +/* T8132: the GPU driver's reported work, not GFX power, gets the M5/F3 floor. */ +static void apple_clpc_t8132_gpu_input_test(struct kunit *test) +{ + struct apple_clpc_hw power_state = apple_clpc_hw_t8132; + unsigned int demand[APPLE_CLPC_RAILS] = {}; + struct apple_clpc_gpu *gpu = kzalloc_obj(*gpu); + + KUNIT_ASSERT_NOT_NULL(test, gpu); + kref_init(&gpu->ref); + spin_lock_init(&gpu->lock); + KUNIT_EXPECT_FALSE(test, apple_clpc_hw_t8132.gpu_demand); + + /* No input, or no work reported: no GPU floor. */ + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_input_demand(&apple_clpc_hw_t8132, + NULL, demand)); + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_input_demand(&apple_clpc_hw_t8132, + gpu, demand)); + KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_DCS], 0u); + KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_SOC], 0u); + + /* Work in flight: DCS 5 and fabric 3. */ + apple_clpc_gpu_set_active(gpu, true); + KUNIT_EXPECT_TRUE(test, apple_clpc_gpu_input_demand(&apple_clpc_hw_t8132, + gpu, demand)); + KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_DCS], 5u); + KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_SOC], 3u); + + /* A table that follows GFX power ignores the input. */ + power_state.gpu_demand = true; + memset(demand, 0, sizeof(demand)); + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_input_demand(&power_state, gpu, + demand)); + KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_DCS], 0u); + + /* Drained: nothing more is asked; the vote's hold releases it. */ + apple_clpc_gpu_set_active(gpu, false); + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_input_demand(&apple_clpc_hw_t8132, + gpu, demand)); + KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_DCS], 0u); + apple_clpc_gpu_put(gpu); +} + static struct kunit_case apple_clpc_cases[] = { + KUNIT_CASE(apple_clpc_t8132_gpu_input_test), KUNIT_CASE(apple_clpc_gpu_dense_test), KUNIT_CASE(apple_clpc_gpu_cold_wake_test), KUNIT_CASE(apple_clpc_gpu_request_lifetime_test), diff --git a/drivers/thermal/apple-pmp-thermal.c b/drivers/thermal/apple-pmp-thermal.c index 81b7bb6093ade..203ea9f5608bd 100644 --- a/drivers/thermal/apple-pmp-thermal.c +++ b/drivers/thermal/apple-pmp-thermal.c @@ -174,10 +174,11 @@ static void apple_pmp_power_work(struct work_struct *work) u64 epoch = apple_pmp_gpu_epoch(pmp_thermal.sensors); u64 now; int ret = apple_pmp_power_read_rails(mw); - /* The running CPUs/reader cannot plausibly consume <0.1 W in total. - * A firmware reset/all-zero image must not refill the energy bucket. + /* The running reader cannot read zero on both CPU rails. A firmware + * reset/all-zero image must not refill the energy bucket; an idle + * system may still draw well under 0.1 W on all three rails. */ - if (!ret && mw[0] + mw[1] + mw[2] < 100) + if (!ret && !mw[0] && !mw[1]) ret = -ENODATA; /* Clock under the lock: a raw sample published while the SMC reads ran * must not make the unsigned age below wrap into a false -EAGAIN. diff --git a/tools/asahi/clpc-gpu-demand-tests.py b/tools/asahi/clpc-gpu-demand-tests.py index b10dd91882371..68407dd744dd6 100644 --- a/tools/asahi/clpc-gpu-demand-tests.py +++ b/tools/asahi/clpc-gpu-demand-tests.py @@ -36,6 +36,7 @@ def main(): 'static bool apple_clpc_gpu_dense(', 'static bool apple_clpc_gpu_limited(', 'static void apple_clpc_gpu_floor_count(', + 'static bool apple_clpc_gpu_input_demand(', 'static void apple_clpc_sample(']) code = r''' #include @@ -103,21 +104,34 @@ def main(): } static void apple_clpc_queue(struct apple_clpc *c, bool idle) { (void)c; queued_idle=idle; } ''' + dense_samples + '\n' + tables + '\n' + methods + r''' +// T8132's G16 firmware keeps graphics powered while the GPU idles: the +// M5/F3 floor follows the work the GPU driver reports, as on T6040. static void check_t8132(void) { - struct apple_clpc c={.hw=&apple_clpc_hw_t8132,.agent=1,.mask=3,.gpu_pmgr=(void *)1}; - read_error=vote_error=0; pmgr_status=0; pmp_floor=0; jiffies=0; - apple_clpc_sample(&c); assert(published==0 && queued_idle); - jiffies=100; pmgr_status=0xf0; - apple_clpc_sample(&c); assert(published==0x35 && c.last_reason==4); - jiffies=131; pmgr_status=0; apple_clpc_sample(&c); + struct apple_clpc_gpu gpu={0}; + struct apple_clpc c={.hw=&apple_clpc_hw_t8132,.agent=1,.mask=3,.gpu=&gpu}; + assert(!c.hw->gpu_demand); + read_error=vote_error=0; pmgr_status=0xf0; pmp_floor=0; jiffies=0; published=~0u; + apple_clpc_sample(&c); assert(published==0 && queued_idle && c.last_reason==0); + gpu.active=true; jiffies=100; + apple_clpc_sample(&c); assert(published==0x35 && c.last_reason==2); + gpu.active=false; jiffies=131; apple_clpc_sample(&c); assert(published==0x35 && c.last_reason==16); jiffies=132; apple_clpc_sample(&c); assert(published==0 && queued_idle); - // A failed power-state read still protects an active graphics consumer. - jiffies=200; pmgr_status=0xe0; read_error=-EIO; warned=0; - apple_clpc_sample(&c); assert(published==0x35); +} + +// A failed power-state read still protects an active graphics consumer. +static void check_read_error(void) { + struct apple_clpc c={.hw=&apple_clpc_hw_t8140,.agent=1,.mask=3,.gpu_pmgr=(void *)1}; + read_error=vote_error=0; pmp_floor=4; pmgr_status=0xf0; warned=0; + for (jiffies=100; jiffies<=164; jiffies+=4) apple_clpc_sample(&c); + assert(published==0x34 && c.last_reason==4); + jiffies=168; pmgr_status=0; read_error=-EIO; + apple_clpc_sample(&c); assert(published==0x34 && c.last_reason==4); // The failure is counted for debugfs and warned about once. assert(c.gpu_read_errors==1 && warned==1); - jiffies=232; read_error=0; apple_clpc_sample(&c); assert(published==0); + jiffies=172; read_error=0; apple_clpc_sample(&c); + assert(published==0x34 && c.last_reason==16); + jiffies=204; apple_clpc_sample(&c); assert(published==0); assert(c.gpu_read_errors==1); } @@ -188,9 +202,10 @@ def main(): c.clusters[0].request[0]=c.clusters[0].request[1]=0; jiffies=264; apple_clpc_sample(&c); assert(published==0); check_t8132(); + check_read_error(); check_t8140(); check_t8140_dense(); - puts("PASS: T6040 host demand; T8132 powered floor; T8140 streaming, dense work and PMP backoff; idle release"); + puts("PASS: T6040 and T8132 host demand; T8140 streaming, dense work, read errors and PMP backoff; idle release"); } ''' code = code.replace('#define APPLE_PMC_RAIL_SHIFT(r) ((r)*8)', rail_shift) @@ -198,8 +213,8 @@ def main(): root = Path(tmp) for name, text in [ ('fixed', code), - ('omit-host-demand', code.replace('if (clpc->gpu && !clpc->hw->gpu_demand) {', 'if (false) {')), - ('omit-streaming-demand', code.replace('bool streaming = apple_clpc_gpu_streaming(clpc, now);', + ('omit-host-demand', code.replace('if (apple_clpc_gpu_input_demand(clpc->hw, clpc->gpu, demand))', + 'if (false && apple_clpc_gpu_input_demand(clpc->hw, clpc->gpu, demand))')), ('omit-streaming-demand', code.replace('bool streaming = apple_clpc_gpu_streaming(clpc, now);', 'bool streaming = (apple_clpc_gpu_streaming(clpc, now), false);')), ('omit-dense-demand', code.replace('if (streaming || dense) {', 'if (streaming || (dense && false)) {')), ('dense-ignores-late', code.replace('if (!threshold || late || submissions < threshold)', diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-thermal-policy.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-thermal-policy.py index af6e1ad7c3976..2df317430ea52 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-thermal-policy.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-thermal-policy.py @@ -189,6 +189,22 @@ def operation(op, now=0, mc=90000, error=0, gov=0, power=0, configured=9, cap=9, report={'passed':True,'scope':'Shared production C cooling policy against frozen regression traces, plus production Rust scheduling/selector units; no hardware qualification.', 'rows':len(cases), 'production_sha256':sha(ROOT/'include/linux/soc/apple/pmp-cooling.h'), 'rust_composition_sha256':sha(ROOT/'drivers/gpu/drm/asahi/g16/power/policy.rs'), 'oracle_sha256':sha(FIXTURES/'thermal-policy-oracle.json')} (OUT/'thermal-policy-host.json').write_text(json.dumps(report,indent=2)+'\n') print(unit) + +# The governor's accounting is policy::Governor, run by the units above. These +# are the calls that connect it: publication, drain, the power service with +# whether flights are outstanding, and the idle poll's re-arm, which a failed +# engine does not take. +G16 = ROOT/'drivers/gpu/drm/asahi/g16' +for path, call in [ + ('engine.rs', 'self.power.utilization.busy(now);'), + ('poll.rs', 'self.power.utilization.idle(self.engine.stats.idle_from_ns);'), + ('power.rs', '.govern(now, limit, !self.engine.flights.is_empty());'), + ('poll.rs', 'Ok(false)\n if firmware.recovery.failed().is_none()\n' + ' && firmware.power.utilization.decaying() =>\n' + ' {\n self.mod_poll('), +]: + assert (G16/path).read_text().count(call) == 1, (path, call) +print('Governor connected at publication, drain, power service and idle re-arm') print(json.dumps(report,indent=2)) # Read/reattach interleavings exercise the provider's actual epoch state helpers.