drm/asahi: g17: drop the completion thread's frequency boost while the GPU is the bottleneck

drivers/gpu/drm/asahi/g17/feed.rs

@@ -44,6 +44,8 @@ struct Detector {
     busy: AtomicU64,
     streak: AtomicU32,
     until: AtomicU64,
+    /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`).
+    bound_until: AtomicU64,
     active: AtomicBool,
     pending: AtomicBool,
     stopping: AtomicBool,
@@ -57,6 +59,7 @@ impl Detector {
             busy: AtomicU64::new(0),
             streak: AtomicU32::new(0),
             until: AtomicU64::new(0),
+            bound_until: AtomicU64::new(0),
             active: AtomicBool::new(false),
             pending: AtomicBool::new(false),
             stopping: AtomicBool::new(false),
@@ -108,17 +111,26 @@ impl Detector {
                 Err(_) => 0,
             }
         };
-        if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum {
+        if streak < STREAK_ON {
+            return;
+        }
+        if idle_sum.saturating_mul(4) >= busy_sum {
             self.until.store(now + LINGER_NS, Ordering::Release);
             if !self.active.swap(true, Ordering::AcqRel) {
                 self.pending.store(true, Ordering::Release);
             }
+        } else {
+            self.bound_until.store(now + LINGER_NS, Ordering::Release);
         }
     }
 
     fn wanted(&self, now: u64) -> bool {
         !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire)
     }
+
+    fn bound(&self, now: u64) -> bool {
+        !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire)
+    }
 }
 
 #[cfg(CONFIG_CPU_FREQ)]
@@ -416,6 +428,17 @@ impl Feed {
         }
     }
 
+    /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps
+    /// under a quarter of their busy time, and the feed does not see host-refill-bound work.
+    /// Lingers like streaming, so a pause of more than a few milliseconds ends it.
+    pub(crate) fn gpu_bound(&self) -> bool {
+        if !cfg!(CONFIG_CPU_FREQ) {
+            return false;
+        }
+        let now = Self::now();
+        self.detector.bound(now) && !self.detector.wanted(now)
+    }
+
     /// Called after a firmware event pass, before queuing the single feed work item.
     pub(crate) fn take_pending(&self) -> bool {
         self.detector.pending.swap(false, Ordering::AcqRel)
@@ -498,3 +521,51 @@ impl PinnedDrop for Feed {
         self.shutdown();
     }
 }
+
+#[kernel::macros::kunit_tests(asahi_g17_feed_detector)]
+mod detector_tests {
+    use super::*;
+
+    /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart.
+    fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 {
+        let mut start = 1000;
+        let mut now = 1_000_000;
+        for _ in 0..passes {
+            detector.note(start, start + busy, now);
+            start += busy + gap;
+            now += 1_000_000;
+        }
+        now
+    }
+
+    #[test]
+    fn back_to_back_long_passes_are_gpu_bound() {
+        let detector = Detector::new();
+        // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1.
+        let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000);
+        assert!(detector.bound(now));
+        assert!(!detector.wanted(now));
+        assert!(!detector.bound(now + LINGER_NS));
+    }
+
+    #[test]
+    fn refill_gaps_stream_instead() {
+        let detector = Detector::new();
+        // 20 us passes with 60 us refill gaps: bound by host refills.
+        let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667);
+        assert!(detector.wanted(now));
+        assert!(!detector.bound(now));
+    }
+
+    #[test]
+    fn paced_or_short_runs_are_neither() {
+        let detector = Detector::new();
+        // Each pass starts after a pause of at least the refill limit.
+        let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS);
+        assert!(!detector.bound(now) && !detector.wanted(now));
+        // Fewer back-to-back passes than the streak needs.
+        let detector = Detector::new();
+        let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0);
+        assert!(!detector.bound(now) && !detector.wanted(now));
+    }
+}

drivers/gpu/drm/asahi/g17/mod.rs

@@ -440,6 +440,16 @@ impl Mailbox {
     fn send(&self, endpoint: u8, message: u64) -> Result {
         self.with(|rtkit| rtkit.send_message(endpoint, message))
     }
+
+    /// A handle on the receive thread's minimum utilization, usable from any context.
+    ///
+    /// # Safety
+    ///
+    /// The caller must not use the handle after this mailbox is dropped.
+    unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin {
+        // SAFETY: The caller's obligation is the handle's.
+        self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() })
+    }
 }
 
 /// A running firmware coprocessor.
@@ -1386,7 +1396,10 @@ impl Gpu {
                 secondary,
                 primary,
                 bridge,
-                port: port::Port::new(power::Demand::new(pdev.as_ref().as_raw())?)?,
+                port: port::Port::new(
+                    power::Demand::new(pdev.as_ref().as_raw())?,
+                    shared.feed.clone(),
+                )?,
                 seen_submissions: 0,
                 rings: channel::Rings::new()?,
                 recovery: recovery::State::new(),

drivers/gpu/drm/asahi/g17/port.rs

@@ -22,11 +22,20 @@ use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering};
 use kernel::{
     new_mutex, new_spinlock,
     prelude::*,
+    soc::apple::rtkit,
     sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock},
 };
 
 const PRIORITIES: usize = Policy::PRIORITIES as usize;
 
+/// Minimum utilization of the primary coprocessor's receive thread, which handles every
+/// completion, while the GPU itself is the bottleneck (see `Feed::gpu_bound`). Otherwise the
+/// thread keeps the scheduler's RT default, which takes its CPU cluster to the top frequency for
+/// each completion: the completion may be on the critical path, of a waiter, of an idle GPU or of
+/// work bound by host refills. While the GPU is the bottleneck the burst gains nothing, and at the
+/// SoC's thermal limit the power it takes comes out of the GPU's clock.
+const RX_UTIL_MIN_GPU_BOUND: u32 = 256;
+
 /// One priority class's compute work ring: its slot array and the cursors of its two firmware
 /// consumers and of the host producer.
 #[derive(Clone, Copy)]
@@ -89,6 +98,12 @@ pub(super) struct Port {
     rings: SpinLock<[u32; PRIORITIES]>,
     #[pin]
     qos: SpinLock<qos::Accounting>,
+    /// Sets the completion receive thread's minimum utilization, from `feed`'s view of the GPU.
+    rx: SetOnce<rtkit::RxUtilMin>,
+    feed: Arc<super::feed::Feed>,
+    /// The request last made, `None` before the first.
+    #[pin]
+    rx_applied: SpinLock<Option<Option<u32>>>,
     /// Serializes every render publication, with or without the device mutex: the tiling
     /// and fragment work rings have one host producer at a time, and same-VM attachment
     /// overlaps are ordered by publication order (see hazard.rs).
@@ -144,13 +159,19 @@ impl Drop for OuterClaim<'_> {
 }
 
 impl Port {
-    pub(super) fn new(demand: super::power::Demand) -> Result<Arc<Self>> {
+    pub(super) fn new(
+        demand: super::power::Demand,
+        feed: Arc<super::feed::Feed>,
+    ) -> Result<Arc<Self>> {
         Arc::pin_init(
             pin_init!(Self {
                 views: SetOnce::new(),
                 regs: SetOnce::new(),
                 rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"),
                 qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"),
+                rx: SetOnce::new(),
+                feed,
+                rx_applied <- new_spinlock!(None, "g17::Port::rx_applied"),
                 render <- new_mutex!((), "g17::Port::render"),
                 generation: AtomicU64::new(0),
                 publications: AtomicU64::new(0),
@@ -167,7 +188,10 @@ impl Port {
 
     /// Called once while the session is built, before any publisher can see the port.
     pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result {
-        if self.views.populate(views) && self.regs.populate(regs) {
+        // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port, and
+        // with it every use of the handle.
+        let rx = unsafe { (*views.mailbox).rx_util_min_handle() };
+        if self.views.populate(views) && self.regs.populate(regs) && self.rx.populate(rx) {
             Ok(())
         } else {
             Err(EBUSY)
@@ -393,7 +417,9 @@ impl Port {
         policy: Policy,
     ) -> Result<qos::Publication> {
         let view = self.qos_view()?;
-        self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)
+        let publication = self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)?;
+        self.rx_update();
+        Ok(publication)
     }
 
     /// Publishes the two owners of a render command, fragment then tiling.
@@ -404,7 +430,10 @@ impl Port {
         policy: Policy,
     ) -> Result<[qos::Publication; 2]> {
         let view = self.qos_view()?;
-        self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)
+        let publications =
+            self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)?;
+        self.rx_update();
+        Ok(publications)
     }
 
     pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result {
@@ -414,7 +443,24 @@ impl Port {
 
     pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result {
         let view = self.qos_view()?;
-        self.qos.lock().complete(&view, owners)
+        self.qos.lock().complete(&view, owners)?;
+        self.rx_update();
+        Ok(())
+    }
+
+    /// Requests the receive thread's minimum utilization for the feed's current view of the GPU.
+    /// Called at every publication and completion: the feed's state lingers for a few
+    /// milliseconds, so after a pause the publication that ends it restores the default before
+    /// its completion arrives.
+    fn rx_update(&self) {
+        let Some(rx) = self.rx.as_ref() else {
+            return;
+        };
+        let want = self.feed.gpu_bound().then_some(RX_UTIL_MIN_GPU_BOUND);
+        let mut applied = self.rx_applied.lock();
+        if *applied != Some(want) && rx.set(want).is_ok() {
+            *applied = Some(want);
+        }
     }
 
     /// Rings the primary coprocessor's doorbell.

drivers/soc/apple/mailbox-g17-test.h

@@ -54,9 +54,49 @@ static void apple_g17_tx_stall_bounded_test(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ENODEV);
 }
 
+/* The receive thread's minimum utilization: request validation, and applying
+ * it to an RT thread (the test thread stands in) without losing its policy.
+ */
+static void apple_g17_rx_util_min_test(struct kunit *test)
+{
+	struct apple_mbox mbox = {
+		.rx_util_min = -1, .rx_util_min_applied = -1,
+	};
+
+	KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 0), -EOPNOTSUPP);
+	mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX;
+	if (!IS_ENABLED(CONFIG_UCLAMP_TASK)) {
+		KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 0), -EOPNOTSUPP);
+		kunit_skip(test, "needs CONFIG_UCLAMP_TASK");
+	}
+	KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, -2), -EINVAL);
+	KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, SCHED_CAPACITY_SCALE + 1), -EINVAL);
+	KUNIT_EXPECT_EQ(test, mbox.rx_util_min, -1);
+#ifdef CONFIG_UCLAMP_TASK
+	sched_set_fifo(current);
+	KUNIT_ASSERT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 256), 0);
+	apple_mbox_apply_rx_util_min(&mbox);
+	KUNIT_EXPECT_EQ(test, mbox.rx_util_min_applied, 256);
+	KUNIT_EXPECT_EQ(test, current->policy, SCHED_FIFO);
+	KUNIT_EXPECT_EQ(test, current->rt_priority, MAX_RT_PRIO / 2);
+	KUNIT_EXPECT_EQ(test, (unsigned int)current->uclamp_req[UCLAMP_MIN].value, 256U);
+	KUNIT_EXPECT_TRUE(test, current->uclamp_req[UCLAMP_MIN].user_defined);
+
+	/* -1 hands the clamp back to the scheduler's RT default. */
+	KUNIT_ASSERT_EQ(test, apple_mbox_set_rx_util_min(&mbox, -1), 0);
+	apple_mbox_apply_rx_util_min(&mbox);
+	KUNIT_EXPECT_EQ(test, mbox.rx_util_min_applied, -1);
+	KUNIT_EXPECT_EQ(test, current->policy, SCHED_FIFO);
+	KUNIT_EXPECT_EQ(test, current->rt_priority, MAX_RT_PRIO / 2);
+	KUNIT_EXPECT_FALSE(test, current->uclamp_req[UCLAMP_MIN].user_defined);
+	sched_set_normal(current, 0);
+#endif
+}
+
 static struct kunit_case apple_g17_stall_cases[] = {
 	KUNIT_CASE(apple_g17_rx_flood_bounded_test),
 	KUNIT_CASE(apple_g17_tx_stall_bounded_test),
+	KUNIT_CASE(apple_g17_rx_util_min_test),
 	{ }
 };
 static struct kunit_suite apple_g17_stall_suite = {

drivers/soc/apple/mailbox.c

@@ -29,9 +29,11 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/sched.h>
 #include <linux/soc/apple/mailbox.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
+#include <uapi/linux/sched/types.h>
 
 #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16)
 #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17)
@@ -451,12 +453,49 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox)
 	return ret;
 }
 
+/*
+ * Threaded receive handler only: apply the consumer's minimum utilization to
+ * this thread. As an RT thread it otherwise runs with the RT default, which
+ * makes schedutil raise its CPU cluster to the top frequency on every message.
+ */
+static void apple_mbox_apply_rx_util_min(struct apple_mbox *mbox)
+{
+	int util_min = READ_ONCE(mbox->rx_util_min);
+	struct sched_attr attr = {
+		.sched_policy = current->policy,
+		.sched_priority = current->rt_priority,
+		.sched_flags = SCHED_FLAG_KEEP_ALL | SCHED_FLAG_UTIL_CLAMP_MIN,
+	};
+
+	if (util_min == mbox->rx_util_min_applied)
+		return;
+	/* -1 resets the clamp to the default, which follows the RT sysctl. */
+	attr.sched_util_min = util_min;
+	if (!sched_setattr_nocheck(current, &attr))
+		mbox->rx_util_min_applied = util_min;
+}
+
+int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min)
+{
+	if (!IS_ENABLED(CONFIG_UCLAMP_TASK) || !apple_mbox_is_g17(mbox))
+		return -EOPNOTSUPP;
+	if (util_min < -1 || util_min > SCHED_CAPACITY_SCALE)
+		return -EINVAL;
+	WRITE_ONCE(mbox->rx_util_min, util_min);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min);
+
 static irqreturn_t apple_mbox_recv_irq(int irq, void *data)
 {
 	struct apple_mbox *mbox = data;
 	unsigned long flags;
 	int consumed = 0;
 
+	/* Only the G17 receive handler is threaded (see apple_mbox_probe()). */
+	if (apple_mbox_is_g17(mbox))
+		apple_mbox_apply_rx_util_min(mbox);
+
 	spin_lock_irqsave(&mbox->rx_lock, flags);
 	if (likely(!READ_ONCE(mbox->removing)))
 		consumed = apple_mbox_poll_locked(mbox);
@@ -784,6 +823,8 @@ static int apple_mbox_probe(struct platform_device *pdev)
 	if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO)
 		return mbox->irq_send_empty;
 
+	mbox->rx_util_min = -1;
+	mbox->rx_util_min_applied = -1;
 	spin_lock_init(&mbox->rx_lock);
 	spin_lock_init(&mbox->tx_lock);
 	mutex_init(&mbox->lifecycle_lock);

drivers/soc/apple/rtkit.c

@@ -1147,6 +1147,12 @@ void apple_rtkit_resume_rx(struct apple_rtkit *rtk)
 }
 EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx);
 
+int apple_rtkit_set_rx_util_min(struct apple_rtkit *rtk, int util_min)
+{
+	return apple_mbox_set_rx_util_min(rtk->mbox, util_min);
+}
+EXPORT_SYMBOL_GPL(apple_rtkit_set_rx_util_min);
+
 void apple_rtkit_flush_rx(struct apple_rtkit *rtk)
 {
 	flush_workqueue(rtk->wq);