From 54b635838d4c32c4b52fb822b283d19dec8fca39 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 03:17:29 -0400 Subject: [PATCH 1/4] cpufreq: apple: verify T8140 transitions on the next request Every T8140 frequency request spun until the cluster reported the new state, about 50 us per transition, and the same verification kept schedutil off the fast-switch path. Each change therefore woke the governor kthread, which runs above every normal task and then busy-waited on whichever CPU it landed on. Under light GPU work the P-cluster changes state about a thousand times a second: each GPU completion runs the mailbox interrupt thread, a real-time task that raises the cluster to its top state, and the cluster drops back once the thread sleeps. The spin cost 7% of a P-core and landed between a completion and the next submission. On the fast-switch path, check each request when the next one is issued instead. That request still waits for BUSY to clear and still stops all requests if the accepted state is not the one last written, so an unexpected owner is caught as before, one request later. With the wait gone the driver can switch from the scheduler path, and the governor thread is no longer used. Requests through ->target_index, from suspend and the static governors, still wait for their own transition. The fast-switch callback now resolves its table index from the frequency it is given instead of reading policy->cached_resolved_idx, which a concurrent limits update can rewrite between programming the cluster and reporting the frequency. Measured on Neo, same boot, alternating with the previous module (gpubench, us per submission, two rounds): compute, depth 2 58.8-59.7 -> 46.4-47.3 compute, depth 1 107.8-108.4 -> 102.6-102.7 render, depth 2 47.5-47.8 -> 45.8-45.9 render, depth 32 and compute, depth 64: unchanged (cherry picked from commit 9aada77348904f44f930e8341e63392326570cca) Signed-off-by: DjDeveloperr --- drivers/cpufreq/apple-soc-cpufreq.c | 48 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index d5172ddd346f3f..f0159407078e48 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -109,7 +109,10 @@ * running, the hrtimer only checks every 100 ms, and every second once the * warning has been given. A nohz_full CPU can run a task with its tick * stopped, and so without sampling, so a cluster with nohz_full CPUs - * samples from the hrtimer all the time. + * samples from the hrtimer all the time. Where schedutil switches frequency + * from the scheduler (fast switch), a lower limit reaches the cluster at its + * next scheduler frequency update rather than at once, which a nohz_full CPU + * running with its tick stopped may not have for a long time. * * A sample reads the effort through the PMP report driver; only when the * limit it maps to changes is a work item queued to update the QoS request, @@ -186,6 +189,7 @@ static int apple_fast_die_hp_state; struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + bool lazy_verify; /* Fail closed when the DT describes no fast die-temperature limit. */ bool needs_thermal_policy; u32 transition_timeout_us; @@ -239,12 +243,17 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { /* * T8140 uses 5-bit PS1 state IDs starting at 1. The command register reads - * back the last accepted request, not a measured frequency; every transition - * is verified, see apple_soc_cpufreq_set_target(). Voltage, PLL and firmware - * throttling configuration are left as the bootloader set them. + * back the last accepted request, not a measured frequency; each request is + * verified, see apple_soc_cpufreq_request(). Requests from the scheduler's + * fast-switch path are verified when the next one is issued rather than + * waited for, so schedutil can switch from the scheduler; requests through + * ->target_index (suspend, the static governors) are still waited for. + * Voltage, PLL and firmware throttling configuration are left as the + * bootloader set them. */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, + .lazy_verify = true, .needs_thermal_policy = true, .min_pstate = 1, .max_pstate = 31, @@ -354,8 +363,14 @@ static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) return 0; } -static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, - unsigned int index) +/* + * Requests the frequency table entry @index. With @wait, a SoC that verifies + * transitions waits for this one to be accepted; without it, the next request + * checks it: it waits for BUSY to clear and requires the accepted state to + * match. + */ +static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, + unsigned int index, bool wait) { struct apple_cpu_priv *priv = policy->driver_data; unsigned int pstate = policy->freq_table[index].driver_data; @@ -400,7 +415,7 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); - if (priv->info->verify_transition && + if (priv->info->verify_transition && wait && readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY) && FIELD_GET(APPLE_DVFS_CMD_PS1, reg) == pstate, @@ -417,13 +432,26 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, return 0; } +static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + return apple_soc_cpufreq_request(policy, index, true); +} + static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { - if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) + struct apple_cpu_priv *priv = policy->driver_data; + /* + * Resolve the index here: policy->cached_resolved_idx can be rewritten + * by a concurrent limits update. target_freq is already clamped. + */ + int idx = cpufreq_table_find_index_l(policy, target_freq, false); + + if (idx < 0 || apple_soc_cpufreq_request(policy, idx, !priv->info->lazy_verify) < 0) return 0; - return policy->freq_table[policy->cached_resolved_idx].frequency; + return policy->freq_table[idx].frequency; } static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, @@ -909,7 +937,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->cpuinfo.transition_latency = transition_latency; policy->dvfs_possible_from_any_cpu = true; - policy->fast_switch_possible = !info->verify_transition; + policy->fast_switch_possible = !info->verify_transition || info->lazy_verify; policy->suspend_freq = freq_table[0].frequency; return 0; From 7298b9bb7f0d1719c230a999d1c518aab45f2c55 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:15:30 -0400 Subject: [PATCH 2/4] cpufreq: apple: verify T8140 fast-switch requests from a timer The previous change verified a fast-switch request only when the next one was issued. That request still spun on BUSY in the scheduler for up to the transition timeout, the last request of a burst was never verified, and a request dropped on BUSY was never retried: schedutil does not ask again for the frequency it last asked for. Verify from a hard-irq hrtimer instead, serialised with the scheduler's requests and ->target_index by a per-policy raw spinlock: - A request that finds the cluster idle checks that the accepted state is the one last written, and is written at once. The timer checks the last request written the same way within 1 ms, with 0.5 ms of slack so that it can share an interrupt, so every request is verified, the last of a burst included. - A request that finds the cluster busy is kept as pending; it is neither spun on nor dropped. The timer writes it once the transition has finished, from about when it should have, and verifies it in turn. - A transition still busy past the transition timeout, or an accepted state other than the one written, latches transition_failed as before. The timer reports it, outside the scheduler. - fast_switch returns the requested frequency once a request is committed. ->target_index cancels the timer, drops any pending request and keeps its blocking verification; ->exit cancels the timer. Only lazy_verify SoCs (T8140) use the timer. Measured on T8140 against the previous change, in the same boot, alternating, three rounds (us per submit, medians): compute.light queue d1 102.67 -> 102.38 compute.light queue d2 46.70 -> 46.61 render.light queue d2 45.81 -> 45.76 with 5-6% more timer interrupts over those rows. Under mixed GPU and CPU load the policies made about 570 transitions/s with no DVFS errors; all-core load throttled to the same frequencies; the static governors and a pm_test=devices suspend/resume cycle ran without errors. A diagnostic build, not part of this change, showed the timer alone finding a refused write when no further request was issued, and a stuck BUSY on both policies. Signed-off-by: DjDeveloperr --- drivers/cpufreq/apple-soc-cpufreq.c | 283 +++++++++++++++++++++++++--- 1 file changed, 262 insertions(+), 21 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index f0159407078e48..3647d78427f742 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -71,6 +71,17 @@ #define APPLE_DVFS_TRANSITION_TIMEOUT 400 +/* + * Lazy verification: the timer checks the last request written this long + * after it, with this much slack so that it can share an interrupt with other + * timers. A request kept pending behind a busy transition is retried about + * when that transition should finish, and then at the short interval. + */ +#define APPLE_DVFS_VERIFY_LAZY_NS (1 * NSEC_PER_MSEC) +#define APPLE_DVFS_VERIFY_SLACK_NS (500 * NSEC_PER_USEC) +#define APPLE_DVFS_TRANSITION_NS (50 * NSEC_PER_USEC) +#define APPLE_DVFS_RETRY_NS (4 * NSEC_PER_USEC) + /* * Fast die-temperature limit * @@ -210,6 +221,22 @@ struct apple_cpu_priv { const struct apple_soc_cpufreq_info *info; bool transition_failed; unsigned int expected_pstate; + /* + * Lazy verification, see apple_soc_cpufreq_lazy_request(): the lock + * serialises the command register, expected_pstate and + * transition_failed between the scheduler's requests, the timer that + * verifies them and ->target_index. + */ + raw_spinlock_t lock; + struct hrtimer verify_timer; + /* When the last request was written, and when it times out. */ + ktime_t write_time; + ktime_t verify_deadline; + /* P-state requested while a transition was busy, or -1. */ + int pending_pstate; + /* A failure the verify timer has yet to report, and its command. */ + bool failure_unreported; + u64 failure_command; }; static struct cpufreq_driver apple_soc_cpufreq_driver; @@ -244,12 +271,11 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { /* * T8140 uses 5-bit PS1 state IDs starting at 1. The command register reads * back the last accepted request, not a measured frequency; each request is - * verified, see apple_soc_cpufreq_request(). Requests from the scheduler's - * fast-switch path are verified when the next one is issued rather than - * waited for, so schedutil can switch from the scheduler; requests through - * ->target_index (suspend, the static governors) are still waited for. - * Voltage, PLL and firmware throttling configuration are left as the - * bootloader set them. + * verified. Requests from the scheduler's fast-switch path are verified by a + * timer rather than waited for, so schedutil can switch from the scheduler, + * see apple_soc_cpufreq_lazy_request(); requests through ->target_index + * (suspend, the static governors) are still waited for. Voltage, PLL and + * firmware throttling configuration are left as the bootloader set them. */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, @@ -363,14 +389,25 @@ static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) return 0; } +/* Writes a request for @pstate into the command register value @reg. */ +static void apple_soc_cpufreq_write(struct apple_cpu_priv *priv, u64 reg, + unsigned int pstate) +{ + reg &= ~priv->info->ps1_mask; + reg |= pstate << priv->info->ps1_shift; + if (priv->info->has_ps2) + FIELD_MODIFY(APPLE_DVFS_CMD_PS2, ®, pstate); + reg |= APPLE_DVFS_CMD_SET; + + writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); +} + /* - * Requests the frequency table entry @index. With @wait, a SoC that verifies - * transitions waits for this one to be accepted; without it, the next request - * checks it: it waits for BUSY to clear and requires the accepted state to - * match. + * Requests the frequency table entry @index and, on a SoC that verifies + * transitions, waits for the cluster to accept it. */ static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, - unsigned int index, bool wait) + unsigned int index) { struct apple_cpu_priv *priv = policy->driver_data; unsigned int pstate = policy->freq_table[index].driver_data; @@ -407,15 +444,9 @@ static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, return 0; } - reg &= ~priv->info->ps1_mask; - reg |= pstate << priv->info->ps1_shift; - if (priv->info->has_ps2) - FIELD_MODIFY(APPLE_DVFS_CMD_PS2, ®, pstate); - reg |= APPLE_DVFS_CMD_SET; - - writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); + apple_soc_cpufreq_write(priv, reg, pstate); - if (priv->info->verify_transition && wait && + if (priv->info->verify_transition && readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY) && FIELD_GET(APPLE_DVFS_CMD_PS1, reg) == pstate, @@ -432,10 +463,210 @@ static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, return 0; } +/* + * Lazy verification (lazy_verify SoCs) + * + * A transition takes about 50 us. The scheduler's fast-switch path must not + * wait for one, nor drop a request: schedutil records the frequency it asked + * for and does not ask again for the same one. + * + * A request finding the cluster idle requires the accepted state to be the + * one last written, so verifying the previous request, and is written at + * once. The verify timer checks the last request written in the same way + * within APPLE_DVFS_VERIFY_LAZY_NS plus slack, so the last request of a burst + * is verified like every other; a transition still busy then is followed + * until it finishes or its timeout has passed. + * + * A request finding the cluster busy is kept as pending, replacing an older + * pending one. The timer writes it as soon as the transition has finished, + * after the same check, and then verifies it as above. + * + * A failure stops all further requests, as for a request that is waited for, + * and the timer reports it. + * + * Called with priv->lock held. + */ +static void apple_soc_cpufreq_fail(struct apple_cpu_priv *priv, u64 reg) +{ + priv->transition_failed = true; + priv->pending_pstate = -1; + /* The scheduler may hold a runqueue lock: the timer reports it. */ + priv->failure_unreported = true; + priv->failure_command = reg; +} + +/* Reports a failure recorded by apple_soc_cpufreq_fail(), once. */ +static void apple_soc_cpufreq_report(struct apple_cpu_priv *priv, u64 reg, + unsigned int expected) +{ + if (reg & APPLE_DVFS_CMD_BUSY) + dev_err(priv->cpu_dev, + "DVFS busy timeout, command=%#llx; stopping requests\n", reg); + else + dev_err(priv->cpu_dev, + "unexpected DVFS state %u (expected %u); stopping requests\n", + (unsigned int)FIELD_GET(APPLE_DVFS_CMD_PS1, reg), expected); +} + +/* + * Writes @pstate if it differs from the accepted state, given the idle + * command register @reg, and returns whether a transition is now in flight. + * Called with priv->lock held. + */ +static bool apple_soc_cpufreq_lazy_write(struct apple_cpu_priv *priv, u64 reg, + unsigned int pstate) +{ + if (FIELD_GET(APPLE_DVFS_CMD_PS1, reg) != priv->expected_pstate) { + apple_soc_cpufreq_fail(priv, reg); + return false; + } + if (pstate == priv->expected_pstate) + return false; + + apple_soc_cpufreq_write(priv, reg, pstate); + priv->expected_pstate = pstate; + priv->write_time = ktime_get(); + priv->verify_deadline = ktime_add_us(priv->write_time, + apple_soc_cpufreq_timeout(priv->info)); + return true; +} + +/* Called with priv->lock held. */ +static void apple_soc_cpufreq_verify_later(struct apple_cpu_priv *priv) +{ + hrtimer_start_range_ns(&priv->verify_timer, + ns_to_ktime(APPLE_DVFS_VERIFY_LAZY_NS), + APPLE_DVFS_VERIFY_SLACK_NS, HRTIMER_MODE_REL_HARD); +} + +static enum hrtimer_restart apple_soc_cpufreq_verify(struct hrtimer *timer) +{ + struct apple_cpu_priv *priv = container_of(timer, struct apple_cpu_priv, + verify_timer); + unsigned int expected; + bool report = false; + unsigned long flags; + u64 reg; + + raw_spin_lock_irqsave(&priv->lock, flags); + if (priv->transition_failed) + goto out; + + /* + * Re-armed with hrtimer_start(), never forwarded: a request may have + * queued the timer again while this callback waited for the lock. + */ + reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + if (reg & APPLE_DVFS_CMD_BUSY) { + ktime_t now = ktime_get(); + + if (ktime_after(now, priv->verify_deadline)) + apple_soc_cpufreq_fail(priv, reg); + else if (priv->pending_pstate >= 0) + hrtimer_start(timer, ns_to_ktime(APPLE_DVFS_RETRY_NS), + HRTIMER_MODE_REL_HARD); + else + hrtimer_start(timer, priv->verify_deadline, + HRTIMER_MODE_ABS_HARD); + } else if (priv->pending_pstate >= 0) { + unsigned int pstate = priv->pending_pstate; + + priv->pending_pstate = -1; + if (apple_soc_cpufreq_lazy_write(priv, reg, pstate)) + apple_soc_cpufreq_verify_later(priv); + } else if (FIELD_GET(APPLE_DVFS_CMD_PS1, reg) != priv->expected_pstate) { + apple_soc_cpufreq_fail(priv, reg); + } +out: + if (priv->failure_unreported) { + priv->failure_unreported = false; + report = true; + reg = priv->failure_command; + expected = priv->expected_pstate; + } + raw_spin_unlock_irqrestore(&priv->lock, flags); + if (report) + apple_soc_cpufreq_report(priv, reg, expected); + return HRTIMER_NORESTART; +} + +/* + * Requests the frequency table entry @index from the scheduler's fast-switch + * path without waiting, see "Lazy verification" above. + */ +static int apple_soc_cpufreq_lazy_request(struct cpufreq_policy *policy, + unsigned int index) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned int pstate = policy->freq_table[index].driver_data; + unsigned long flags; + int ret = 0; + u64 reg; + + if (pstate < priv->info->min_pstate || pstate > priv->info->max_pstate) + return -EINVAL; + + raw_spin_lock_irqsave(&priv->lock, flags); + if (priv->transition_failed) { + ret = -EIO; + goto out; + } + + reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + if (reg & APPLE_DVFS_CMD_BUSY) { + /* Retry about when the transition should have finished. */ + ktime_t now = ktime_get(); + ktime_t at = ktime_add_ns(priv->write_time, APPLE_DVFS_TRANSITION_NS); + + if (ktime_before(at, ktime_add_ns(now, APPLE_DVFS_RETRY_NS))) + at = ktime_add_ns(now, APPLE_DVFS_RETRY_NS); + priv->pending_pstate = pstate; + if (!hrtimer_active(&priv->verify_timer) && + ktime_after(now, priv->verify_deadline)) + priv->verify_deadline = + ktime_add_us(now, apple_soc_cpufreq_timeout(priv->info)); + if (!hrtimer_is_queued(&priv->verify_timer) || + ktime_after(hrtimer_get_softexpires(&priv->verify_timer), at)) + hrtimer_start(&priv->verify_timer, at, HRTIMER_MODE_ABS_HARD); + goto out; + } + + /* An idle cluster has accepted, or refused, the previous request. */ + priv->pending_pstate = -1; + if (apple_soc_cpufreq_lazy_write(priv, reg, pstate)) { + if (!hrtimer_is_queued(&priv->verify_timer)) + apple_soc_cpufreq_verify_later(priv); + } else if (priv->transition_failed) { + /* Let the timer report it, outside the scheduler. */ + hrtimer_start(&priv->verify_timer, 0, HRTIMER_MODE_REL_HARD); + ret = -EIO; + } +out: + raw_spin_unlock_irqrestore(&priv->lock, flags); + return ret; +} + static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) { - return apple_soc_cpufreq_request(policy, index, true); + struct apple_cpu_priv *priv = policy->driver_data; + unsigned long flags; + int ret; + + if (!priv->info->lazy_verify) + return apple_soc_cpufreq_request(policy, index); + + /* + * Waited-for requests (suspend, the static governors) replace any + * pending one; the lock keeps a late fast-switch request out until + * this one is accepted. + */ + hrtimer_cancel(&priv->verify_timer); + raw_spin_lock_irqsave(&priv->lock, flags); + priv->pending_pstate = -1; + ret = apple_soc_cpufreq_request(policy, index); + raw_spin_unlock_irqrestore(&priv->lock, flags); + return ret; } static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, @@ -448,9 +679,13 @@ static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, */ int idx = cpufreq_table_find_index_l(policy, target_freq, false); - if (idx < 0 || apple_soc_cpufreq_request(policy, idx, !priv->info->lazy_verify) < 0) + if (idx < 0) + return 0; + if (priv->info->lazy_verify ? apple_soc_cpufreq_lazy_request(policy, idx) : + apple_soc_cpufreq_request(policy, idx)) return 0; + /* A lazy request is committed: the cluster applies it within microseconds. */ return policy->freq_table[idx].frequency; } @@ -917,6 +1152,10 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) priv->cpu_dev = cpu_dev; priv->reg_base = reg_base; priv->info = info; + raw_spin_lock_init(&priv->lock); + hrtimer_setup(&priv->verify_timer, apple_soc_cpufreq_verify, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_HARD); + priv->pending_pstate = -1; policy->driver_data = priv; policy->freq_table = freq_table; @@ -961,6 +1200,8 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; + /* The governor has stopped: no request re-arms the verify timer. */ + hrtimer_cancel(&priv->verify_timer); /* Drain firmware-limit updates before releasing the OPP tables. */ apple_fast_die_exit(priv); From 979d4f190190c7181b82f6428ccace579397b622 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:43:03 -0400 Subject: [PATCH 3/4] cpufreq: apple: settle T8140 lazy requests before suspend A fast-switch request returns its frequency once committed, and the core records it in policy->cur, but on a lazy_verify SoC it may still be pending behind a busy transition, or be dropped when the verify timer finds that transition failed. cpufreq_generic_suspend() skips a suspend frequency equal to policy->cur, so suspend could leave the cluster at an older state, or miss a failure, without calling the driver. Give the driver its own ->suspend: on lazy_verify SoCs it cancels the verify timer, reports a failure the timer had yet to report, writes the pending request and waits for it, verifying it and the transition before it, and only then does the generic suspend. A latched failure fails the driver suspend, which the core logs. Waited-for requests share the draining step. They never run alongside fast-switch requests, so ->target_index no longer holds the raw spinlock, with interrupts off, across its two polls; a failure report the timer lost to the cancel is now printed; ->exit drains as well. Measured on T8140 against the previous change's base in the same boot, alternating, three rounds (us per submit, medians): compute.light queue d1 102.25 -> 102.12 compute.light queue d2 36.93 -> 36.47 render.light queue d2 48.24 -> 48.45 A pm_test=devices suspend cycle ran without DVFS or driver suspend errors. With a diagnostic build, not part of this change: a request left pending for the lowest frequency, equal to policy->cur and suspend_freq, was written and verified at suspend; a stuck BUSY that dropped the pending request, with the timer's report withheld, was reported at suspend and failed the driver suspend. Signed-off-by: DjDeveloperr --- drivers/cpufreq/apple-soc-cpufreq.c | 118 +++++++++++++++++++++------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 3647d78427f742..1cce6915655739 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -224,8 +224,9 @@ struct apple_cpu_priv { /* * Lazy verification, see apple_soc_cpufreq_lazy_request(): the lock * serialises the command register, expected_pstate and - * transition_failed between the scheduler's requests, the timer that - * verifies them and ->target_index. + * transition_failed between the scheduler's requests and the timer + * that verifies them. Waited-for requests cancel the timer first and + * do not take it. */ raw_spinlock_t lock; struct hrtimer verify_timer; @@ -403,19 +404,14 @@ static void apple_soc_cpufreq_write(struct apple_cpu_priv *priv, u64 reg, } /* - * Requests the frequency table entry @index and, on a SoC that verifies - * transitions, waits for the cluster to accept it. + * Requests @pstate and, on a SoC that verifies transitions, waits for the + * cluster to accept it, having waited for and verified the previous request. */ -static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, - unsigned int index) +static int apple_soc_cpufreq_set_pstate(struct apple_cpu_priv *priv, + unsigned int pstate) { - struct apple_cpu_priv *priv = policy->driver_data; - unsigned int pstate = policy->freq_table[index].driver_data; u64 reg; - /* OPP level is a hardware ID, not the frequency-table array index. */ - if (pstate < priv->info->min_pstate || pstate > priv->info->max_pstate) - return -EINVAL; if (priv->transition_failed) return -EIO; @@ -463,6 +459,20 @@ static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, return 0; } +/* Requests the frequency table entry @index, see apple_soc_cpufreq_set_pstate(). */ +static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, + unsigned int index) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned int pstate = policy->freq_table[index].driver_data; + + /* OPP level is a hardware ID, not the frequency-table array index. */ + if (pstate < priv->info->min_pstate || pstate > priv->info->max_pstate) + return -EINVAL; + + return apple_soc_cpufreq_set_pstate(priv, pstate); +} + /* * Lazy verification (lazy_verify SoCs) * @@ -484,6 +494,13 @@ static int apple_soc_cpufreq_request(struct cpufreq_policy *policy, * A failure stops all further requests, as for a request that is waited for, * and the timer reports it. * + * Waited-for requests (the initial frequency check, the static governors, + * suspend) never run alongside fast-switch requests: the core stops a + * governor that switches from the scheduler first. They cancel the timer, + * report a failure it had yet to report and replace a pending request, see + * apple_soc_cpufreq_drain(); suspend first writes the pending request, see + * apple_soc_cpufreq_suspend(). + * * Called with priv->lock held. */ static void apple_soc_cpufreq_fail(struct apple_cpu_priv *priv, u64 reg) @@ -646,27 +663,69 @@ static int apple_soc_cpufreq_lazy_request(struct cpufreq_policy *policy, return ret; } -static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, - unsigned int index) +/* + * Stops lazy verification ahead of a waited-for request: cancels the verify + * timer, reports a failure it had yet to report, and drops the pending + * request, returning its P-state or -1. No fast-switch request runs + * concurrently to queue the timer again. + */ +static int apple_soc_cpufreq_drain(struct apple_cpu_priv *priv) { - struct apple_cpu_priv *priv = policy->driver_data; + unsigned int expected; unsigned long flags; - int ret; - - if (!priv->info->lazy_verify) - return apple_soc_cpufreq_request(policy, index); + bool report; + int pstate; + u64 reg; - /* - * Waited-for requests (suspend, the static governors) replace any - * pending one; the lock keeps a late fast-switch request out until - * this one is accepted. - */ hrtimer_cancel(&priv->verify_timer); + raw_spin_lock_irqsave(&priv->lock, flags); + pstate = priv->pending_pstate; priv->pending_pstate = -1; - ret = apple_soc_cpufreq_request(policy, index); + report = priv->failure_unreported; + priv->failure_unreported = false; + reg = priv->failure_command; + expected = priv->expected_pstate; raw_spin_unlock_irqrestore(&priv->lock, flags); - return ret; + + if (report) + apple_soc_cpufreq_report(priv, reg, expected); + return pstate; +} + +static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct apple_cpu_priv *priv = policy->driver_data; + + /* A waited-for request replaces a pending one, see "Lazy verification". */ + if (priv->info->lazy_verify) + apple_soc_cpufreq_drain(priv); + + return apple_soc_cpufreq_request(policy, index); +} + +/* + * policy->cur holds the last fast-switch request, which on a lazy_verify SoC + * may still be pending, or lost to a failure that stopped requests. The core + * skips a suspend frequency equal to policy->cur, so settle the cluster + * first: write and verify the pending request, or report the failure. + */ +static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) +{ + struct apple_cpu_priv *priv = policy->driver_data; + + if (priv->info->lazy_verify) { + int pstate = apple_soc_cpufreq_drain(priv); + int ret; + + ret = apple_soc_cpufreq_set_pstate(priv, pstate >= 0 ? pstate : + priv->expected_pstate); + if (ret) + return ret; + } + + return cpufreq_generic_suspend(policy); } static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, @@ -1200,8 +1259,11 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; - /* The governor has stopped: no request re-arms the verify timer. */ - hrtimer_cancel(&priv->verify_timer); + /* + * The governor has stopped and the policy goes away: stop the verify + * timer, report a failure it had yet to report, drop a pending request. + */ + apple_soc_cpufreq_drain(priv); /* Drain firmware-limit updates before releasing the OPP tables. */ apple_fast_die_exit(priv); @@ -1227,7 +1289,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .fast_switch = apple_soc_cpufreq_fast_switch, .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, - .suspend = cpufreq_generic_suspend, + .suspend = apple_soc_cpufreq_suspend, }; static int __init apple_soc_cpufreq_module_init(void) From 29ce152d297c0a77bbe759e63f9dfab66381f805 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 21:36:35 -0400 Subject: [PATCH 4/4] cpufreq: apple: apply a lowered T8140 fast die limit at once The fast die-temperature limit is a FREQ_QOS_MAX request. T8140 now switches frequency from the scheduler, where schedutil only records a lower limit and applies it at its next frequency update, which a nohz_full CPU running a task with its tick stopped may not have for a long time. Through ->target_index, which T8140 used before lazy verification, the core applies the limit at once. When a sample changes the limit of a lazy_verify cluster, record the fastest P-state at or below it and move the cluster to the P-state the scheduler last asked for, at most that one, through the same locked path as fast-switch requests. Later fast-switch requests are clamped to it while the limit holds, and return the frequency actually requested. Going back to the scheduler's request also covers a limit that rises again before the QoS request was updated: the request is then left as it was, and schedutil does not ask again. Waited-for requests own the cluster from the drain to the next fast-switch request, and the core applies the limit to them, so the samples leave the cluster alone then. The drain now marks this before it cancels the verify timer, so that no sample queues the timer again. Measured on T8140 with a diagnostic build that forces a 2196 MHz limit while a spinner runs on a P-core: the limit is written inside the sample that sets it, where the scheduler path took about 1 ms, and 0.02 to 2.5 ms with nohz_full on that CPU. A drop and restore within one sample, with the QoS request unchanged, returns the cluster to the scheduler's request. Under a real thermal load the frequency stayed at or below the limit in every sample. Static governor switches, unbinding and a pm_test=devices cycle were clean, and light GPU submission costs are unchanged. Other SoCs are unchanged. Signed-off-by: DjDeveloperr --- drivers/cpufreq/apple-soc-cpufreq.c | 151 ++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 29 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 1cce6915655739..d3af6f015b328c 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -121,9 +121,10 @@ * warning has been given. A nohz_full CPU can run a task with its tick * stopped, and so without sampling, so a cluster with nohz_full CPUs * samples from the hrtimer all the time. Where schedutil switches frequency - * from the scheduler (fast switch), a lower limit reaches the cluster at its - * next scheduler frequency update rather than at once, which a nohz_full CPU - * running with its tick stopped may not have for a long time. + * from the scheduler (fast switch), it picks up a lower limit only at its + * next frequency update, which a nohz_full CPU running with its tick stopped + * may not have for a long time, so the sample applies the limit to the + * cluster itself, see apple_soc_cpufreq_lazy_cap(). * * A sample reads the effort through the PMP report driver; only when the * limit it maps to changes is a work item queued to update the QoS request, @@ -224,9 +225,10 @@ struct apple_cpu_priv { /* * Lazy verification, see apple_soc_cpufreq_lazy_request(): the lock * serialises the command register, expected_pstate and - * transition_failed between the scheduler's requests and the timer - * that verifies them. Waited-for requests cancel the timer first and - * do not take it. + * transition_failed between the scheduler's requests, the timer that + * verifies them and the fast die-temperature samples. Waited-for + * requests cancel the timer first and do not take it; the samples + * then leave the cluster to them. */ raw_spinlock_t lock; struct hrtimer verify_timer; @@ -235,6 +237,15 @@ struct apple_cpu_priv { ktime_t verify_deadline; /* P-state requested while a transition was busy, or -1. */ int pending_pstate; + /* + * Highest P-state, and its frequency, that the fast die-temperature + * limit allows, the P-state the scheduler last asked for, and whether + * waited-for requests own the cluster, see apple_soc_cpufreq_lazy_cap(). + */ + unsigned int cap_pstate; + unsigned int cap_freq; + unsigned int asked_pstate; + bool waited; /* A failure the verify timer has yet to report, and its command. */ bool failure_unreported; u64 failure_command; @@ -608,26 +619,16 @@ static enum hrtimer_restart apple_soc_cpufreq_verify(struct hrtimer *timer) } /* - * Requests the frequency table entry @index from the scheduler's fast-switch - * path without waiting, see "Lazy verification" above. + * Requests @pstate without waiting, see "Lazy verification" above. + * Called with priv->lock held. */ -static int apple_soc_cpufreq_lazy_request(struct cpufreq_policy *policy, - unsigned int index) +static int apple_soc_cpufreq_lazy_set(struct apple_cpu_priv *priv, + unsigned int pstate) { - struct apple_cpu_priv *priv = policy->driver_data; - unsigned int pstate = policy->freq_table[index].driver_data; - unsigned long flags; - int ret = 0; u64 reg; - if (pstate < priv->info->min_pstate || pstate > priv->info->max_pstate) - return -EINVAL; - - raw_spin_lock_irqsave(&priv->lock, flags); - if (priv->transition_failed) { - ret = -EIO; - goto out; - } + if (priv->transition_failed) + return -EIO; reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); if (reg & APPLE_DVFS_CMD_BUSY) { @@ -645,7 +646,7 @@ static int apple_soc_cpufreq_lazy_request(struct cpufreq_policy *policy, if (!hrtimer_is_queued(&priv->verify_timer) || ktime_after(hrtimer_get_softexpires(&priv->verify_timer), at)) hrtimer_start(&priv->verify_timer, at, HRTIMER_MODE_ABS_HARD); - goto out; + return 0; } /* An idle cluster has accepted, or refused, the previous request. */ @@ -656,11 +657,69 @@ static int apple_soc_cpufreq_lazy_request(struct cpufreq_policy *policy, } else if (priv->transition_failed) { /* Let the timer report it, outside the scheduler. */ hrtimer_start(&priv->verify_timer, 0, HRTIMER_MODE_REL_HARD); - ret = -EIO; + return -EIO; } -out: + return 0; +} + +/* + * Requests the frequency table entry @index from the scheduler's fast-switch + * path, at most the fast die-temperature limit, and returns the frequency + * requested, or 0 on failure. The request is committed: the cluster applies + * it within microseconds. + */ +static unsigned int apple_soc_cpufreq_lazy_request(struct cpufreq_policy *policy, + unsigned int index) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned int pstate = policy->freq_table[index].driver_data; + unsigned int freq = policy->freq_table[index].frequency; + unsigned long flags; + int ret; + + if (pstate < priv->info->min_pstate || pstate > priv->info->max_pstate) + return 0; + + raw_spin_lock_irqsave(&priv->lock, flags); + priv->waited = false; + priv->asked_pstate = pstate; + if (pstate > priv->cap_pstate) { + pstate = priv->cap_pstate; + freq = priv->cap_freq; + } + ret = apple_soc_cpufreq_lazy_set(priv, pstate); + raw_spin_unlock_irqrestore(&priv->lock, flags); + return ret ? 0 : freq; +} + +/* + * Applies a new fast die-temperature limit, whose highest P-state is @cap at + * @freq, to a lazy_verify cluster at once. schedutil picks up a lower limit + * only at its next frequency update, which a nohz_full CPU running with its + * tick stopped may not have for a long time. The cluster is moved to the + * P-state the scheduler last asked for, at most @cap, and later requests are + * clamped to @cap. That also restores the request when the limit rises + * again before schedutil has seen it fall: the QoS request is then left + * unchanged, so schedutil does not ask again. + * + * Waited-for requests own the cluster from apple_soc_cpufreq_drain() to the + * next fast-switch request, and the core applies the limit to them. + */ +static void apple_soc_cpufreq_lazy_cap(struct apple_cpu_priv *priv, + unsigned int cap, unsigned int freq) +{ + unsigned int requested, pstate; + unsigned long flags; + + raw_spin_lock_irqsave(&priv->lock, flags); + priv->cap_pstate = cap; + priv->cap_freq = freq; + requested = priv->pending_pstate >= 0 ? priv->pending_pstate : + priv->expected_pstate; + pstate = min(priv->asked_pstate, cap); + if (!priv->waited && pstate != requested) + apple_soc_cpufreq_lazy_set(priv, pstate); raw_spin_unlock_irqrestore(&priv->lock, flags); - return ret; } /* @@ -677,6 +736,11 @@ static int apple_soc_cpufreq_drain(struct apple_cpu_priv *priv) int pstate; u64 reg; + /* First, so that no fast die-temperature sample queues the timer again. */ + raw_spin_lock_irqsave(&priv->lock, flags); + priv->waited = true; + raw_spin_unlock_irqrestore(&priv->lock, flags); + hrtimer_cancel(&priv->verify_timer); raw_spin_lock_irqsave(&priv->lock, flags); @@ -740,11 +804,11 @@ static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, if (idx < 0) return 0; - if (priv->info->lazy_verify ? apple_soc_cpufreq_lazy_request(policy, idx) : - apple_soc_cpufreq_request(policy, idx)) + if (priv->info->lazy_verify) + return apple_soc_cpufreq_lazy_request(policy, idx); + if (apple_soc_cpufreq_request(policy, idx)) return 0; - /* A lazy request is committed: the cluster applies it within microseconds. */ return policy->freq_table[idx].frequency; } @@ -836,6 +900,31 @@ static void apple_fast_die_arm(struct apple_fast_die *fd) HRTIMER_MODE_REL_SOFT); } +/* + * Applies @limit to a lazy_verify cluster at once, as its fastest state at or + * below the limit, see apple_soc_cpufreq_lazy_cap(). + */ +static void apple_fast_die_cap(struct apple_fast_die *fd, unsigned int limit) +{ + struct cpufreq_policy *policy = fd->policy; + struct apple_cpu_priv *priv = policy->driver_data; + struct cpufreq_frequency_table *pos; + unsigned int cap = 0, freq = 0; + + if (!priv->info->lazy_verify) + return; + + cpufreq_for_each_valid_entry(pos, policy->freq_table) { + if (pos->frequency <= limit && pos->driver_data > cap) { + cap = pos->driver_data; + freq = pos->frequency; + } + } + /* Limits are frequency-table entries, see apple_fast_die_fit(). */ + if (freq) + apple_soc_cpufreq_lazy_cap(priv, cap, freq); +} + /* Take a sample and queue an update if the limit changes. Call with @lock held. */ static void apple_fast_die_sample(struct apple_fast_die *fd) { @@ -863,6 +952,7 @@ static void apple_fast_die_sample(struct apple_fast_die *fd) if (limit != fd->limit) { WRITE_ONCE(fd->limit, limit); + apple_fast_die_cap(fd, limit); schedule_work(&fd->work); } } @@ -1036,6 +1126,7 @@ static int apple_fast_die_init(struct cpufreq_policy *policy, fd->limit); if (ret < 0) goto err_free; + apple_fast_die_cap(fd, fd->limit); priv->fast_die = fd; @@ -1215,6 +1306,8 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) hrtimer_setup(&priv->verify_timer, apple_soc_cpufreq_verify, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); priv->pending_pstate = -1; + priv->cap_pstate = info->max_pstate; + priv->waited = true; policy->driver_data = priv; policy->freq_table = freq_table;