diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index e7afc4dd87e8f1..e616317adcb8d8 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -66,6 +66,13 @@ properties: operating-points-v2: true + apple,clpc: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Host performance controller (CLPC) that the GPU reports its + submissions to, for the memory controller and fabric floors it + raises for GPU work. + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 037ffb71a9d4cf..a3eda5e0f51183 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/g17/clpc.c b/drivers/gpu/drm/asahi/g17/clpc.c new file mode 100644 index 00000000000000..39b7f3c58174d1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/clpc.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include "clpc.h" + +int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out) +{ + return apple_clpc_gpu_get(dev, out); +} + +void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) +{ + apple_clpc_gpu_set_active(gpu, active); +} + +void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu) +{ + apple_clpc_gpu_put(gpu); +} diff --git a/drivers/gpu/drm/asahi/g17/clpc.h b/drivers/gpu/drm/asahi/g17/clpc.h new file mode 100644 index 00000000000000..4279d178564f74 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/clpc.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _ASAHI_G17_CLPC_H +#define _ASAHI_G17_CLPC_H +#include +struct apple_clpc_gpu; +/* + * The CLPC's GPU activity input (include/linux/soc/apple/clpc.h). @dev is the + * GPU's struct device, opaque to Rust, whose bindings cannot name it here. + */ +int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out); +void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu); +#endif diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 31c242a1323f8d..193d5bf1720600 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -647,6 +647,8 @@ impl Drop for Firmware { secondary.stop(); } self.primary.stop(); + // Only once both processors stop can they no longer consume work. + self.port.set_effort_full(false); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while // those references still keep the register block powered. @@ -693,6 +695,7 @@ impl Firmware { /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.port.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } @@ -1332,7 +1335,7 @@ impl Gpu { secondary, primary, bridge, - port: port::Port::new()?, + port: port::Port::new(power::Demand::new(pdev.as_ref().as_raw())?)?, seen_submissions: 0, rings: channel::Rings::new(), recovery: recovery::State::new(), diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index db043ef9d6f91e..e158b34ccc37ca 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -105,6 +105,8 @@ pub(super) struct Port { compute_announced: [AtomicU64; PRIORITIES], /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, + /// The CLPC's GPU activity input: told of every submission and effort change. + demand: super::power::Demand, submissions: AtomicU64, } @@ -142,7 +144,7 @@ impl Drop for OuterClaim<'_> { } impl Port { - pub(super) fn new() -> Result> { + pub(super) fn new(demand: super::power::Demand) -> Result> { Arc::pin_init( pin_init!(Self { views: SetOnce::new(), @@ -155,6 +157,7 @@ impl Port { compute_written: [const { AtomicU64::new(0) }; PRIORITIES], compute_announced: [const { AtomicU64::new(0) }; PRIORITIES], effort_full: AtomicBool::new(false), + demand, submissions: AtomicU64::new(0), }), GFP_KERNEL, @@ -453,6 +456,7 @@ impl Port { /// clear sees this submission or this submission sees the flag withdrawn. pub(super) fn note_submission(&self) -> bool { self.submissions.fetch_add(1, Ordering::SeqCst); + self.note_gpu_activity(); self.effort_full.load(Ordering::SeqCst) } @@ -462,5 +466,11 @@ impl Port { pub(super) fn set_effort_full(&self, full: bool) { self.effort_full.store(full, Ordering::SeqCst); + self.demand.set_active(full); + } + + /// Reports a submission, published with or without the device mutex, to the CLPC. + pub(super) fn note_gpu_activity(&self) { + self.demand.set_active(true); } } diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index b0e21e70bbe18e..cd913dc6faddff 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -233,6 +233,51 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { result } +unsafe extern "C" { + fn asahi_clpc_gpu_get( + dev: *mut core::ffi::c_void, + out: *mut *mut core::ffi::c_void, + ) -> core::ffi::c_int; + fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// The GPU activity input of the CLPC that the device tree links the GPU to, if any (see +/// `apple_clpc_gpu_get()`). The input has a lifetime of its own, so it may outlive the CLPC; a +/// device link orders the CLPC before the GPU. +pub(super) struct Demand(*mut core::ffi::c_void); + +// SAFETY: the input's own spinlock serializes every access to it; it calls back into nothing. +unsafe impl Send for Demand {} +// SAFETY: shared access only reports activity under that lock, never changes the lifetime. +unsafe impl Sync for Demand {} + +impl Demand { + pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { + let mut out = core::ptr::null_mut(); + // SAFETY: `dev` is the live platform device and `out` is writable; on success the C + // side returns an owned reference, or NULL when the device tree names no CLPC. + let ret = unsafe { asahi_clpc_gpu_get(dev.cast(), &mut out) }; + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(Self(out)) + } + } + + pub(super) fn set_active(&self, active: bool) { + // SAFETY: the reference taken in `new` is live; NULL is accepted and ignored. + unsafe { asahi_clpc_gpu_set_active(self.0, active) }; + } +} + +impl Drop for Demand { + fn drop(&mut self) { + // SAFETY: releases exactly the reference taken in `new`, clearing the input first. + unsafe { asahi_clpc_gpu_put(self.0) }; + } +} + #[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] mod tests { use super::*; diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index bfbc687a8c27a9..149a7cc8bf912c 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -24,13 +24,20 @@ #include #include #include +#include +#include #include +#include #include #include +#include #include #include #include +#include +#include #include +#include #include #define APPLE_CLPC_MAX_CLUSTERS 8 @@ -86,6 +93,20 @@ struct apple_clpc_vote { unsigned long last_demand; }; +/* + * The GPU driver's activity input, see apple_clpc_gpu_get(). It has a + * lifetime of its own, so that a late put by the GPU driver never touches a + * removed CLPC. + */ +struct apple_clpc_gpu { + struct kref ref; + spinlock_t lock; + bool active; + u64 updated_ns; + /* Counts every report, active or not */ + u64 sequence; +}; + struct apple_clpc { struct device *dev; struct device *pmc; @@ -109,6 +130,7 @@ struct apple_clpc { unsigned long gpu_read_errors; struct dentry *debugfs; struct regmap *gpu_pmgr; + struct apple_clpc_gpu *gpu; u32 gpu_offset; /* Graphics powered at the last sample, since when, and that sample's time */ bool gpu_on; @@ -119,6 +141,117 @@ struct apple_clpc { unsigned long gpu_limited_at; }; +/* Protects publication and removal of the drvdata, never a GPU report. */ +static DEFINE_MUTEX(apple_clpc_clients_lock); + +static void apple_clpc_gpu_free(struct kref *ref) +{ + kfree(container_of(ref, struct apple_clpc_gpu, ref)); +} + +/** + * apple_clpc_gpu_set_active() - report GPU activity to the CLPC + * @gpu: request from apple_clpc_gpu_get(), or NULL + * @active: whether the GPU has work + * + * Safe in atomic context: it only takes the request's own spinlock. + */ +void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) +{ + unsigned long flags; + + if (!gpu) + return; + spin_lock_irqsave(&gpu->lock, flags); + gpu->active = active; + gpu->updated_ns = ktime_get_mono_fast_ns(); + gpu->sequence++; + spin_unlock_irqrestore(&gpu->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_active); + +/** + * apple_clpc_gpu_put() - clear and release a request from apple_clpc_gpu_get() + * @gpu: the request, or NULL + */ +void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) +{ + if (gpu) { + apple_clpc_gpu_set_active(gpu, false); + kref_put(&gpu->ref, apple_clpc_gpu_free); + } +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_put); + +/** + * apple_clpc_gpu_get() - get the activity input of the CLPC a GPU links to + * @dev: the GPU device + * @out: the request, or NULL when @dev names no CLPC + * + * The CLPC is named by the "apple,clpc" phandle of @dev. A device link orders + * it before the GPU. It has one such input, for one GPU. + * + * Return: 0, -EPROBE_DEFER while the CLPC has not probed, or another error. + */ +int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +{ + struct device_node *np; + struct platform_device *provider; + struct apple_clpc *clpc; + struct device_link *link; + int ret = -EPROBE_DEFER; + + *out = NULL; + np = of_parse_phandle(dev->of_node, "apple,clpc", 0); + if (!np) + return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; + if (!of_device_is_compatible(np, "apple,t8140-clpc")) { + of_node_put(np); + return -EINVAL; + } + provider = of_find_device_by_node(np); + of_node_put(np); + if (!provider) + return ret; + link = device_link_add(dev, &provider->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + put_device(&provider->dev); + return -ENOMEM; + } + mutex_lock(&apple_clpc_clients_lock); + clpc = dev_get_drvdata(&provider->dev); + if (clpc && clpc->gpu && refcount_read(&clpc->gpu->ref.refcount) == 1) { + kref_get(&clpc->gpu->ref); + *out = clpc->gpu; + ret = 0; + } + mutex_unlock(&apple_clpc_clients_lock); + put_device(&provider->dev); + return ret; +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_get); + +static bool apple_clpc_gpu_active(struct apple_clpc_gpu *gpu, u64 *updated, + u64 *sequence) +{ + unsigned long flags; + bool active; + + spin_lock_irqsave(&gpu->lock, flags); + active = gpu->active; + *updated = gpu->updated_ns; + *sequence = gpu->sequence; + spin_unlock_irqrestore(&gpu->lock, flags); + return active; +} + +static void apple_clpc_gpu_owner_put(void *data) +{ + struct apple_clpc_gpu *gpu = data; + + kref_put(&gpu->ref, apple_clpc_gpu_free); +} + /* cpufreq policies can appear after probe; pick up new ones as they do. */ static void apple_clpc_find_clusters(struct apple_clpc *clpc) { @@ -376,6 +509,13 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) else seq_puts(s, "never\n"); } + if (clpc->gpu) { + u64 updated, sequence; + bool active = apple_clpc_gpu_active(clpc->gpu, &updated, &sequence); + + seq_printf(s, "host_gpu: active %u request_ns %llu sequence %llu\n", + active, updated, sequence); + } for (i = 0; i < clpc->nr_clusters; i++) seq_printf(s, "cluster cpu%u: busy %u%% level %u%% -> dcs %u soc %u\n", clpc->clusters[i].policy_cpu, clpc->clusters[i].busy, @@ -421,6 +561,9 @@ static void apple_clpc_stop(void *data) { struct apple_clpc *clpc = data; + mutex_lock(&apple_clpc_clients_lock); + dev_set_drvdata(clpc->dev, NULL); + mutex_unlock(&apple_clpc_clients_lock); apple_clpc_halt(clpc); debugfs_remove_recursive(clpc->debugfs); apple_clpc_free_clusters(clpc); @@ -441,6 +584,18 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->dev = dev; clpc->pmc = dev->parent; clpc->hw = of_device_get_match_data(dev); + /* The GPU driver reports its submissions here, see apple_clpc_gpu_get(). */ + if (of_device_is_compatible(dev->of_node, "apple,t8140-clpc")) { + clpc->gpu = kzalloc_obj(*clpc->gpu); + if (!clpc->gpu) + return -ENOMEM; + kref_init(&clpc->gpu->ref); + spin_lock_init(&clpc->gpu->lock); + ret = devm_add_action_or_reset(dev, apple_clpc_gpu_owner_put, + clpc->gpu); + if (ret) + return ret; + } /* The property is optional: without it, CPU demand alone sets the floors. */ if (clpc->hw->gpu_demand) gpu = of_parse_phandle(dev->of_node, "apple,gpu-power-state", 0); @@ -506,7 +661,9 @@ static int apple_clpc_probe(struct platform_device *pdev) if (ret) return ret; + mutex_lock(&apple_clpc_clients_lock); platform_set_drvdata(pdev, clpc); + mutex_unlock(&apple_clpc_clients_lock); apple_clpc_queue(clpc, true); return 0; } @@ -583,6 +740,63 @@ static const struct apple_clpc_hw apple_clpc_hw_t8140 = { .pmp_agent = 0, }; +#if IS_BUILTIN(CONFIG_KUNIT) && !defined(MODULE) +#include + +static void apple_clpc_gpu_request_lifetime_test(struct kunit *test) +{ + struct apple_clpc_gpu *gpu = kzalloc_obj(*gpu); + u64 updated, sequence; + + KUNIT_ASSERT_NOT_NULL(test, gpu); + kref_init(&gpu->ref); + spin_lock_init(&gpu->lock); + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_active(gpu, &updated, &sequence)); + KUNIT_EXPECT_EQ(test, sequence, 0ull); + /* The CLPC and one GPU own references of their own. */ + kref_get(&gpu->ref); + apple_clpc_gpu_set_active(gpu, true); + KUNIT_EXPECT_TRUE(test, apple_clpc_gpu_active(gpu, &updated, &sequence)); + KUNIT_EXPECT_GT(test, updated, 0ull); + KUNIT_EXPECT_EQ(test, sequence, 1ull); + /* The GPU's put clears the input. */ + apple_clpc_gpu_put(gpu); + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_active(gpu, &updated, &sequence)); + KUNIT_EXPECT_EQ(test, sequence, 2ull); + kref_put(&gpu->ref, apple_clpc_gpu_free); +} + +static void apple_clpc_gpu_provider_removal_test(struct kunit *test) +{ + struct apple_clpc_gpu *gpu = kzalloc_obj(*gpu); + u64 updated, sequence; + + KUNIT_ASSERT_NOT_NULL(test, gpu); + kref_init(&gpu->ref); + spin_lock_init(&gpu->lock); + kref_get(&gpu->ref); + /* A removed CLPC drops its reference; the GPU's stays usable. */ + kref_put(&gpu->ref, apple_clpc_gpu_free); + apple_clpc_gpu_set_active(gpu, true); + KUNIT_EXPECT_TRUE(test, apple_clpc_gpu_active(gpu, &updated, &sequence)); + apple_clpc_gpu_set_active(gpu, false); + KUNIT_EXPECT_FALSE(test, apple_clpc_gpu_active(gpu, &updated, &sequence)); + apple_clpc_gpu_put(gpu); +} + +static struct kunit_case apple_clpc_cases[] = { + KUNIT_CASE(apple_clpc_gpu_request_lifetime_test), + KUNIT_CASE(apple_clpc_gpu_provider_removal_test), + {} +}; + +static struct kunit_suite apple_clpc_suite = { + .name = "apple-clpc-gpu", + .test_cases = apple_clpc_cases, +}; +kunit_test_suite(apple_clpc_suite); +#endif + static const struct of_device_id apple_clpc_of_match[] = { { .compatible = "apple,t8132-clpc", .data = &apple_clpc_hw_t8132 }, { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, diff --git a/include/linux/soc/apple/clpc.h b/include/linux/soc/apple/clpc.h new file mode 100644 index 00000000000000..37b2cfce5594a9 --- /dev/null +++ b/include/linux/soc/apple/clpc.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _LINUX_SOC_APPLE_CLPC_H +#define _LINUX_SOC_APPLE_CLPC_H + +#include +#include +#include + +struct apple_clpc_gpu; + +#if IS_REACHABLE(CONFIG_APPLE_CLPC) +int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); +void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu); +#else +static inline int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +{ + *out = NULL; + return device_property_present(dev, "apple,clpc") ? -ENODEV : 0; +} +static inline void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) {} +static inline void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) {} +#endif +#endif