soc: apple: clpc, drm/asahi: g17: busy-share memory floor and light effort for T604x

drivers/gpu/drm/asahi/g17/clpc.c

@@ -1,7 +1,45 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 #include <linux/soc/apple/clpc.h>
+#include <linux/moduleparam.h>
 #include "clpc.h"
 
+static unsigned int g17_light_effort = 0x8000;
+module_param(g17_light_effort, uint, 0644);
+MODULE_PARM_DESC(g17_light_effort, "G17 performance effort for quiet GPU work, of 65536 (0: always full)");
+
+/*
+ * Light effort runs the same work about twice as long, so the way back to
+ * full effort has to sit at about twice the way in, or work that fits light
+ * effort sticks at full. Shares in permille, job lengths in microseconds.
+ */
+static unsigned int g17_light_enter = 350;
+module_param(g17_light_enter, uint, 0644);
+static unsigned int g17_light_exit = 750;
+module_param(g17_light_exit, uint, 0644);
+static unsigned int g17_light_job_enter_us = 3000;
+module_param(g17_light_job_enter_us, uint, 0644);
+static unsigned int g17_light_job_exit_us = 7000;
+module_param(g17_light_job_exit_us, uint, 0644);
+
+unsigned int asahi_g17_light_param(unsigned int which)
+{
+	switch (which) {
+	case 0:
+		return READ_ONCE(g17_light_enter);
+	case 1:
+		return READ_ONCE(g17_light_exit);
+	case 2:
+		return READ_ONCE(g17_light_job_enter_us);
+	default:
+		return READ_ONCE(g17_light_job_exit_us);
+	}
+}
+
+unsigned int asahi_g17_light_effort(void)
+{
+	return READ_ONCE(g17_light_effort);
+}
+
 int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out)
 {
 	return apple_clpc_gpu_get(dev, out);
@@ -10,6 +48,10 @@ void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active)
 {
 	apple_clpc_gpu_set_active(gpu, active);
 }
+void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille)
+{
+	apple_clpc_gpu_set_busy(gpu, permille);
+}
 void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu)
 {
 	apple_clpc_gpu_put(gpu);

drivers/gpu/drm/asahi/g17/clpc.h

@@ -6,4 +6,7 @@ struct apple_clpc_gpu;
 int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out);
 void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active);
 void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu);
+unsigned int asahi_g17_light_effort(void);
+unsigned int asahi_g17_light_param(unsigned int which);
+void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille);
 #endif

drivers/gpu/drm/asahi/g17/mod.rs

@@ -713,7 +713,7 @@ impl Firmware {
     fn note_submission(&mut self) -> Result {
         self.port.note_gpu_activity();
         self.effort.submit();
-        self.apply_effort(power::FULL_EFFORT)
+        self.apply_effort(power::submission_effort())
     }
 
     fn apply_effort(&mut self, effort: u32) -> Result {

drivers/gpu/drm/asahi/g17/port.rs

@@ -516,7 +516,9 @@ impl Port {
     pub(super) fn note_submission(&self) -> bool {
         self.submissions.fetch_add(1, Ordering::SeqCst);
         self.note_gpu_activity();
+        // Take the slow path when the governor wants a different effort.
         self.effort_full.load(Ordering::SeqCst)
+            && super::power::submission_effort() == super::power::FULL_EFFORT
     }
 
     pub(super) fn submissions(&self) -> u64 {
@@ -546,6 +548,7 @@ impl Port {
         self.last_submission_ns
             .store(super::now_ns(), Ordering::Relaxed);
         self.demand.set_active(true);
+        self.demand.report_busy(super::now_ns());
     }
 
     pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {

drivers/gpu/drm/asahi/g17/power.rs

@@ -3,7 +3,7 @@
 //! Work-based firmware performance effort. Idle power gating is managed separately by the
 //! device-control ring; this input never keeps an otherwise idle GPU powered.
 
-use core::sync::atomic::AtomicU32;
+use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
 
 pub(super) const FULL_EFFORT: u32 = 0x10000;
 pub(super) const IDLE_HOLD_NS: u64 = 16_000_000;
@@ -305,6 +305,53 @@ unsafe extern "C" {
     fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int;
     fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool);
     fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void);
+    fn asahi_g17_light_effort() -> u32;
+    fn asahi_g17_light_param(which: core::ffi::c_uint) -> core::ffi::c_uint;
+    fn asahi_clpc_gpu_set_busy(gpu: *mut core::ffi::c_void, permille: core::ffi::c_uint);
+}
+
+/// Busy-share window: long enough to span frames, short enough to follow a burst.
+const BUSY_WINDOW_NS: u64 = 200_000_000;
+/// A longer gap between reports was idle: start over rather than average it in.
+const BUSY_WINDOW_MAX_NS: u64 = 2_000_000_000;
+static BUSY_WINDOW_START: AtomicU64 = AtomicU64::new(0);
+static BUSY_WINDOW_TOTAL: AtomicU64 = AtomicU64::new(0);
+
+/// A window under this share (permille) with no long job is quiet.
+/// Consecutive quiet windows needed to go light.
+const LIGHT_WINDOWS: u32 = 4;
+static LIGHT: AtomicBool = AtomicBool::new(false);
+static QUIET_WINDOWS: AtomicU32 = AtomicU32::new(0);
+
+/// The effort for the next submission: light once the GPU has been quiet for a while.
+pub(super) fn submission_effort() -> u32 {
+    // SAFETY: reads a module parameter.
+    let light = unsafe { asahi_g17_light_effort() };
+    if LIGHT.load(Ordering::Relaxed) && light != 0 {
+        light
+    } else {
+        FULL_EFFORT
+    }
+}
+
+fn light_param(which: u32) -> u64 {
+    // SAFETY: a plain read of a module parameter.
+    u64::from(unsafe { asahi_g17_light_param(which) })
+}
+
+fn light_step(permille: u64, longest_ns: u64) {
+    if LIGHT.load(Ordering::Relaxed) {
+        if permille >= light_param(1) || longest_ns >= light_param(3) * 1000 {
+            LIGHT.store(false, Ordering::Relaxed);
+            QUIET_WINDOWS.store(0, Ordering::Relaxed);
+        }
+    } else if permille < light_param(0) && longest_ns < light_param(2) * 1000 {
+        if QUIET_WINDOWS.fetch_add(1, Ordering::Relaxed) + 1 >= LIGHT_WINDOWS {
+            LIGHT.store(true, Ordering::Relaxed);
+        }
+    } else {
+        QUIET_WINDOWS.store(0, Ordering::Relaxed);
+    }
 }
 
 /// DT-selected memory-demand provider. Its independent allocation outlives a
@@ -332,6 +379,29 @@ impl Demand {
         // SAFETY: the retained reference is live; NULL denotes no DT provider.
         unsafe { asahi_clpc_gpu_set_active(self.0, active) };
     }
+
+    /// Reports the share of the last window the GPU spent executing, once a window has run.
+    /// Racing submitters can only lose one window's report.
+    pub(super) fn report_busy(&self, now_ns: u64) {
+        let start = BUSY_WINDOW_START.load(Ordering::Relaxed);
+        let elapsed = now_ns.saturating_sub(start);
+        if elapsed < BUSY_WINDOW_NS {
+            return;
+        }
+        let total = crate::usage::TOTAL_NS.load(Ordering::Relaxed);
+        let busy = total.saturating_sub(BUSY_WINDOW_TOTAL.swap(total, Ordering::Relaxed));
+        BUSY_WINDOW_START.store(now_ns, Ordering::Relaxed);
+        if start == 0 || elapsed > BUSY_WINDOW_MAX_NS {
+            return;
+        }
+        let permille = busy.saturating_mul(1000) / elapsed;
+        light_step(permille, crate::usage::MAX_NS.swap(0, Ordering::Relaxed));
+        // Full effort does the same work in about half the time; report the share in light
+        // effort's terms so that one memory ladder fits both.
+        let scaled = if LIGHT.load(Ordering::Relaxed) { permille } else { permille * 2 };
+        // SAFETY: the retained reference is live; NULL denotes no DT provider.
+        unsafe { asahi_clpc_gpu_set_busy(self.0, scaled.min(1000) as core::ffi::c_uint) };
+    }
 }
 
 impl Drop for Demand {

drivers/gpu/drm/asahi/usage.rs

@@ -8,6 +8,11 @@
 
 use core::sync::atomic::{AtomicU64, Ordering};
 
+/// Engine time charged across all clients, in ns, for the GPU busy share.
+pub(crate) static TOTAL_NS: AtomicU64 = AtomicU64::new(0);
+/// Longest single stage charged since the governor last read it, in ns.
+pub(crate) static MAX_NS: AtomicU64 = AtomicU64::new(0);
+
 #[derive(Clone, Copy)]
 pub(crate) enum Engine {
     Vertex,
@@ -35,6 +40,8 @@ impl Client {
             return;
         }
         let ns = ticks_to_ns(end - start, frequency_hz);
+        TOTAL_NS.fetch_add(ns, Ordering::Relaxed);
+        MAX_NS.fetch_max(ns, Ordering::Relaxed);
         let counter = &self.ns[engine as usize];
         let mut old = counter.load(Ordering::Relaxed);
         loop {

drivers/soc/apple/clpc.c

@@ -63,6 +63,9 @@ enum apple_clpc_rail {
  */
 #define APPLE_CLPC_GPU_BACKOFF_MS	100
 
+/* A lower GPU busy level must last this long; a higher one applies at once. */
+#define APPLE_CLPC_GPU_DOWN_MS		400
+
 /**
  * struct apple_clpc_hw - per-SoC CLPC parameters
  * @rail: PMC rail of the DCS and of the fabric
@@ -80,7 +83,12 @@ enum apple_clpc_rail {
  *	dense, see apple_clpc_gpu_dense(); 0 for none
  * @gpu_input: the GPU driver reports its work in flight through
  *	apple_clpc_gpu_get(); while there is some, both rails go to the top
- *	of @map
+ *	of @map, or to @gpu_busy if the driver reports a busy share
+ * @gpu_busy: state of each rail for reported GPU work below gpu_busy_mid,
+ *	gpu_busy_high and gpu_busy_top and at or above it; all zero to always
+ *	use the top of @map
+ * @level_khz: frequency @map's quarters are taken from, so that clusters of
+ *	different speed share them; 0 for each cluster's own maximum
  */
 struct apple_clpc_hw {
 	unsigned int rail[APPLE_CLPC_RAILS];
@@ -93,6 +101,8 @@ struct apple_clpc_hw {
 	unsigned int pmp_agent;
 	unsigned int gpu_dense;
 	bool gpu_input;
+	u8 gpu_busy[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS];
+	unsigned int level_khz;
 };
 
 struct apple_clpc_cluster {
@@ -116,9 +126,32 @@ struct apple_clpc_gpu {
 	bool active;
 	u64 updated_ns;
 	u64 sequence;
+	/* Busy share of the last window in permille, if the GPU reports one */
+	unsigned int busy;
+	bool busy_valid;
+	unsigned int level;
+	unsigned long level_since;
 	struct delayed_work *wake;
 };
 
+/*
+ * The GPU busy share (apple_clpc_gpu_set_busy()) at which reported GPU work
+ * gets the top memory and fabric floor, and below which it gets the lowest
+ * GPU state; in between it gets the second. The share is time spent
+ * executing at whatever clock the GPU runs, not bandwidth: a 120 fps game
+ * reads about a third, a scrolling browser a few percent. A GPU that
+ * reports no share keeps the top floor for all its work.
+ */
+static unsigned int gpu_busy_top = 800;
+module_param(gpu_busy_top, uint, 0644);
+MODULE_PARM_DESC(gpu_busy_top, "GPU busy share (permille) for the top memory/fabric floor");
+static unsigned int gpu_busy_high = 500;
+module_param(gpu_busy_high, uint, 0644);
+MODULE_PARM_DESC(gpu_busy_high, "GPU busy share (permille) for the floor below the top one");
+static unsigned int gpu_busy_mid = 160;
+module_param(gpu_busy_mid, uint, 0644);
+MODULE_PARM_DESC(gpu_busy_mid, "GPU busy share (permille) below which GPU work gets the lowest floor state");
+
 struct apple_clpc {
 	struct device *dev;
 	struct device *pmc;
@@ -197,6 +230,26 @@ void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active)
 }
 EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_active);
 
+/**
+ * apple_clpc_gpu_set_busy() - Report the GPU busy share of the last window
+ * @gpu: the GPU input, from apple_clpc_gpu_get()
+ * @permille: share of the window the GPU spent executing work, 0..1000
+ *
+ * Scales the floor that reported GPU work gets, see gpu_busy_top.
+ */
+void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille)
+{
+	unsigned long flags;
+
+	if (!gpu)
+		return;
+	spin_lock_irqsave(&gpu->lock, flags);
+	gpu->busy = min(permille, 1000U);
+	gpu->busy_valid = true;
+	spin_unlock_irqrestore(&gpu->lock, flags);
+}
+EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_busy);
+
 void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu)
 {
 	if (gpu) {
@@ -263,6 +316,30 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out)
 }
 EXPORT_SYMBOL_GPL(apple_clpc_gpu_get);
 
+/* The CPU-quartile column for reported GPU work, from its busy share */
+static unsigned int apple_clpc_gpu_level(struct apple_clpc_gpu *gpu)
+{
+	unsigned long flags;
+	unsigned int level;
+
+	spin_lock_irqsave(&gpu->lock, flags);
+	if (!gpu->busy_valid || gpu->busy >= READ_ONCE(gpu_busy_top))
+		level = APPLE_CLPC_LEVELS - 1;
+	else if (gpu->busy >= READ_ONCE(gpu_busy_high))
+		level = 2;
+	else
+		level = gpu->busy >= READ_ONCE(gpu_busy_mid) ? 1 : 0;
+	if (level >= gpu->level ||
+	    time_after_eq(jiffies, gpu->level_since +
+			  msecs_to_jiffies(APPLE_CLPC_GPU_DOWN_MS))) {
+		gpu->level = level;
+		gpu->level_since = jiffies;
+	}
+	level = gpu->level;
+	spin_unlock_irqrestore(&gpu->lock, flags);
+	return level;
+}
+
 static bool apple_clpc_gpu_active(struct apple_clpc_gpu *gpu, u64 *updated, u64 *sequence)
 {
 	unsigned long flags;
@@ -350,7 +427,8 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc,
 	policy = cpufreq_cpu_get(cl->policy_cpu);
 	if (policy) {
 		if (policy->cpuinfo.max_freq)
-			cl->level = policy->cur * 100 / policy->cpuinfo.max_freq;
+			cl->level = policy->cur * 100 /
+				(clpc->hw->level_khz ?: policy->cpuinfo.max_freq);
 		cpufreq_cpu_put(policy);
 	}
 
@@ -532,13 +610,17 @@ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw,
 					unsigned int *demand)
 {
 	u64 updated, sequence;
-	unsigned int rail;
+	unsigned int level, rail;
 
 	if (!gpu || hw->gpu_demand ||
 	    !apple_clpc_gpu_active(gpu, &updated, &sequence))
 		return false;
+	level = apple_clpc_gpu_level(gpu);
 	for (rail = 0; rail < APPLE_CLPC_RAILS; rail++)
-		demand[rail] = max(demand[rail], hw->map[rail][APPLE_CLPC_LEVELS - 1]);
+		demand[rail] = max_t(unsigned int, demand[rail],
+				     hw->gpu_busy[APPLE_CLPC_DCS][APPLE_CLPC_LEVELS - 1] ?
+				     hw->gpu_busy[rail][level] :
+				     hw->map[rail][APPLE_CLPC_LEVELS - 1]);
 	return true;
 }
 
@@ -654,6 +736,10 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused)
 		u64 updated, sequence;
 		bool active = apple_clpc_gpu_active(clpc->gpu, &updated, &sequence);
 
+		seq_printf(s, "host_gpu busy: %u permille (%s) -> level %u\n",
+			   READ_ONCE(clpc->gpu->busy),
+			   READ_ONCE(clpc->gpu->busy_valid) ? "reported" : "none",
+			   apple_clpc_gpu_level(clpc->gpu));
 		seq_printf(s, "host_gpu: active %u request_ns %llu sequence %llu\n",
 			   active, updated, sequence);
 	}
@@ -947,12 +1033,17 @@ static const struct apple_clpc_hw apple_clpc_hw_t8140 = {
 static const struct apple_clpc_hw apple_clpc_hw_t6040 = {
 	.rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 },
 	.map = {
-		[APPLE_CLPC_DCS] = { 1, 2, 5, 5 },
-		[APPLE_CLPC_SOC] = { 0, 1, 2, 2 },
+		[APPLE_CLPC_DCS] = { 1, 1, 3, 4 },
+		[APPLE_CLPC_SOC] = { 0, 0, 0, 0 },
 	},
 	.sustain = 3,
 	.gpu_demand = false,
 	.gpu_input = true,
+	.gpu_busy = {
+		[APPLE_CLPC_DCS] = { 1, 2, 4, 5 },
+		[APPLE_CLPC_SOC] = { 0, 0, 0, 2 },
+	},
+	.level_khz = 4512000,
 };
 
 /*