diff --git a/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi b/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi index 5775b05487876..d6f24cbe59c94 100644 --- a/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi +++ b/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi @@ -33,6 +33,11 @@ sep = &sep; }; + /* Suspend-to-RAM at EL2 through aura's S2R mailbox; without the mailbox it stays out. */ + t6041-sleep { + compatible = "apple,t6041-sleep"; + }; + chosen { #address-cells = <2>; #size-cells = <2>; diff --git a/arch/arm64/boot/dts/apple/t6041.dtsi b/arch/arm64/boot/dts/apple/t6041.dtsi index 22a215fe7126f..5fca6b7688051 100644 --- a/arch/arm64/boot/dts/apple/t6041.dtsi +++ b/arch/arm64/boot/dts/apple/t6041.dtsi @@ -445,3 +445,17 @@ &{/opp-table-e/opp07} { opp-microwatt = <531000>; }; + +/* + * What the SEP's DART gets around a sleep, as offset and value: the writes T8140's has + * (t8140.dtsi), which the J616C has slept with. The SEP driver saves the DART with them + * before it tells the enclave to sleep at suspend. + */ +&sep_dart { + apple,sleep-prepare = <0x80 0xa102>, <0x704 0xffffffff>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x318 0x1>; + apple,sleep-restore-late = <0x80 0x8000>, <0x228 0x3f3f3f3f>, + <0x22c 0x1010101>; +}; diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index dc24c3e5057a0..7e97876b22b0d 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -5,6 +5,7 @@ * Copyright (C) The Asahi Linux Contributors */ +#include #include #include #include @@ -3168,6 +3169,19 @@ static int bcm4377_resume(struct device *dev) bcm4377->wake_armed = false; } + /* + * A real sleep took the function's power: the firmware is gone, and every resume command + * would only time out (two times 2 s inside the system resume, then the rebind anyway). + * Skip them and rebind from the work queue; probe boots the firmware from scratch. + */ + if (pm_resume_via_firmware() && !bcm4377->dma_window_base) { + bcm4377->quiesced = false; + bcm4377->resume_powered = false; + WRITE_ONCE(bcm4377->setup_retry_blocked, false); + bcm4377_queue_rebind(bcm4377, "power lost in the sleep"); + return 0; + } + if (bcm4377->quiesced) { iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index a24b557119b35..f05e98085dd65 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1964,6 +1964,9 @@ static int apple_platform_suspend(struct device *dev) if (apple && apple->dark_held) return 0; + if (pm_suspend_via_firmware()) + dcp_t6041_s2r_all(false); + if (apple && !apple->aborted) { apple->s2r_skipped = apple_drm_firmware_stopped(apple); if (!apple->s2r_skipped) @@ -2009,6 +2012,9 @@ static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + /* Nothing if this suspend never took the display down. */ + dcp_t6041_s2r_all(true); + if (!apple || apple->aborted || apple->s2r_skipped) return 0; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 90939cb1cb2cc..73fe3ca3d8a78 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -479,6 +479,8 @@ struct apple_dcp { */ bool ext_failsafe; bool ext_dead; + /* T6041 S2R: this external DCP's firmware stop ran in a sleep of this boot */ + bool t6041_ext_seen; /* * A T6030 external processor in the main DRM device: the kernel starts * its firmware on first use, its modes come from the attached display. diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7fb810d0320d0..6990b5339ce02 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -8837,6 +8838,551 @@ static struct platform_driver apple_platform_driver = { }, }; +/* + * T6041 suspend-to-RAM: each display firmware is stopped from Linux before the sleep, from the + * display subsystem's own suspend, and the internal one is restarted warm after the wake. + */ +static int dcp_stop_poller(void *data) +{ + while (!kthread_should_stop()) { + apple_rtkit_poll(data); + usleep_range(500, 1000); + } + return 0; +} + +/* + * The fixed waits of the display stop and restart, as run on 11 Nov (lit on every wake): after + * the pipe power-on before the mode set, before the power words go back to their boot state, + * between the firmware stop and the power words, and after the power words. + */ +#define DCP_S2R_SETTLE_MS 200 +#define DCP_S2R_WORDS_MS 100 +#define DCP_S2R_STOP_MS 300 +#define DCP_S2R_DOWN_MS 100 + +static int dcp_t6041_stop_one(struct device *dev, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + bool external = *(bool *)data; + struct task_struct *poller; + u32 status; + int ret; + + /* dcp->external is only set on T6030. */ + if (!dcp || IS_ERR_OR_NULL(dcp->rtk) || + strstarts(dev_name(dev), "402e") == external) + return 0; + /* + * An external DCP stopped for an earlier sleep stays down, with its domains forced off by + * the power word step: reading its registers again hung the second sleep of a boot (11 Nov). + */ + if (external) { + /* Also one that was never running: the power word step took its domains down too. */ + if (READ_ONCE(dcp->ext_dead) || dcp->t6041_ext_seen) + return 0; + dcp->t6041_ext_seen = true; + /* + * This pipe's display DART too goes dark for good, now, while its block is still on: + * the power word step forces the external display domains off and nothing raises + * them, but the framebuffers are mapped through all three display DARTs, so every + * later unmap went to flush a DART behind a dead block and the CPU doing it never came + * back (login or sddm restart after a wake, last line DRM_IOCTL_MODE_CLOSEFB; 11 Nov). + * Marked dark, the flush only publishes to memory. + */ + dcp_s2r_find_darts(dcp); + if (dcp->s2r_disp_dart) + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, false, true); + } + if (!apple_rtkit_is_running(dcp->rtk)) + return 0; + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_pause_presentation(dcp, true); + iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + dcp_s2r_find_darts(dcp); + if (!external) { + bool idle; + + if (dcp->s2r_disp_dart) + apple_dart_display_keep(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, true); + if (dcp->s2r_dcp_dart) + apple_dart_display_keep(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true); + /* + * As linux#1477 dcp_t6041_s2r_stop: the pipe off through the firmware, then wait for + * its calls to settle. No av_service_disconnect: the firmware answered it with an + * unplug (HPD removed), and after the warm restart it refused every mode set with + * "HPD is 0 link_status is 0" (10 Nov). No display sleep transition after the + * power-off either. + */ + _dcp_poweroff(dcp); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 200000, false, dcp)) + dev_warn(dev, "display calls still in flight after the power-off\n"); + msleep(100); + /* The firmware's D572(0), only answered here: its display DART streams off now. */ + if (dcp->s2r_disp_dart) + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, + false, false); + usleep_range(5000, 6000); + } + iomfb_s2r_500_v26_6_desktop(dcp); + usleep_range(4000, 5000); + ret = apple_rtkit_idle(dcp->rtk); + if (ret) { + poller = kthread_run(dcp_stop_poller, dcp->rtk, "dcpstop"); + ret = apple_rtkit_idle(dcp->rtk); + if (!IS_ERR(poller)) + kthread_stop(poller); + } + if (ret) { + dev_warn(dev, "the firmware didn't go idle for the sleep: %d\n", ret); + return 0; + } + apple_rtkit_poll(dcp->rtk); + readl_poll_timeout(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS, status, + (status & (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) == + (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED), 100, 50000); + /* + * QUIESCED and SLEEP after idle: T23 reached STOPPED (0x6e) only this way. Not for the + * internal DCP: linux#1477 stops it at the idle (A500, idle, mailbox off, run bit, bit 5) + * and its warm restart then works; after a shutdown ours crashed in the restart (10 Nov, + * data abort at 0xffffffff00000040 right after the first shmem_setup). + */ + if (external) + apple_rtkit_shutdown(dcp->rtk); + WRITE_ONCE(dcp->crashed, true); + WRITE_ONCE(dcp->ext_dead, true); + WRITE_ONCE(dcp->s2r_fw_down, true); + apple_rtkit_mailbox_off(dcp->rtk); + dcp_s2r_cpu_stop(dcp); + dcp_s2r_stop_streams(dcp); + msleep(20); + + return 0; +} + +/* + * The power words of each display CPU down, after its firmware stopped. The fabric rejects + * writes to a DCP's CPU word until bit 4 of its slice in the protection table at 0x500578000 is + * clear: ADT clock-protection-slice-index, 0x100 + 0x40 * index. Then macOS's order on the Neo. + */ +static u32 dcp_t6041_child_saved[11]; +static bool dcp_t6041_child_valid; + +static void dcp_t6041_word_try(const char *name, u32 slice, unsigned long cpu, unsigned long fe, + unsigned long sys, unsigned long pg_a, unsigned long pg_b) +{ + void __iomem *gapf = ioremap(0x500578000UL + slice, 4); + void __iomem *rcpu = ioremap(cpu, 4), *rfe = ioremap(fe, 4), *rsys = ioremap(sys, 4); + void __iomem *ra = ioremap(pg_a, 4), *rb = ioremap(pg_b, 4), *rfab = NULL; + bool subs_main = cpu < 0x502800000UL; + u32 val, fab = 0, cur; + + if (!gapf || !rcpu || !rfe || !rsys || !ra || !rb) + goto out; + val = readl(gapf); + writel(val & ~BIT(4), gapf); + msleep(1); + val = readl(rcpu); + writel(val & ~(BIT(28) | GENMASK(9, 8)), rcpu); + msleep(1); + val = readl(rcpu); + /* + * Everything macOS's sleep check wants off under this CPU, last first: DSC, PPP, GPL, + * GP1, GP0, DPEXT, DBE. iBoot leaves them on over a DCP Linux never started. + */ + { + static const u8 ext_subs[] = { 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0 }; + static const u8 main_subs[] = { 0x58, 0x50, 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0 }; + const u8 *subs = subs_main ? main_subs : ext_subs; + int i; + + if (subs_main) { + for (i = 0; i < 11; i++) { + void __iomem *sub = ioremap(cpu + 8 * (i + 1), 4); + + if (sub) { + dcp_t6041_child_saved[i] = readl(sub); + iounmap(sub); + } + } + dcp_t6041_child_valid = true; + } + for (i = 0; subs[i]; i++) { + void __iomem *sub = ioremap(cpu + subs[i], 4); + + if (!sub) + continue; + val = readl(sub); + if (val & BIT(28)) { + val &= ~(BIT(28) | GENMASK(9, 8)); + writel(val, sub); + } + writel((val & ~GENMASK(3, 0)) | GENMASK(9, 8), sub); + iounmap(sub); + } + val = readl(rcpu); + } + /* + * Two things the restart sets that boot did not have, undone before the lowering: bit 0 of + * both words of bridge 4 (boot 0x10000 / 0x101330, after a restart 0x10001 / 0x101331) and the + * display voter count at PMC 0x5048a9c14 (boot 0, after a restart 1). With them left, the + * second lowering of a boot ends with FE 0x200 and SYS 0x800 (bit 11) instead of 0x300 / 0x300. + */ + if (subs_main) { + void __iomem *br = ioremap(0x403000100UL, 0x10); + void __iomem *cnt = ioremap(0x5048a9c14UL, 4); + + if (br && cnt) { + writel(readl(br + 0x4) & ~1U, br + 0x4); + writel(readl(br + 0x8) & ~1U, br + 0x8); + writel(0, cnt); + } + if (br) + iounmap(br); + if (cnt) + iounmap(cnt); + } + /* + * The internal display's SYS hangs off fab6_soc, which gates itself as soon as nothing + * under it is on. Lowered while the fabric is already going (second stop of a boot, the + * external pipes long off), SYS latches PARENT_OFF (reads 0x800, FE 0x200), the sleep + * engine waits on that bit and the chip resets. So: the fabric on by hand until SYS is down. + */ + if (subs_main) { + rfab = ioremap(0x502800480UL, 4); + if (rfab) { + fab = readl(rfab); + writel((fab & ~(BIT(28) | GENMASK(9, 8) | GENMASK(3, 0))) | 0xf, rfab); + if (readl_poll_timeout(rfab, cur, (cur & 0xf0) == 0xf0, 50, 20000)) + pr_warn("dcp %s: fab6_soc not on: %#x\n", name, cur); + } + } + writel(3, ra); + writel(0, rb); + writel((val & ~GENMASK(3, 0)) | GENMASK(9, 8), rcpu); + val = readl(rfe); + if (val & BIT(28)) + writel(val & ~(BIT(28) | GENMASK(9, 8)), rfe); + writel((readl(rfe) & ~GENMASK(3, 0)) | GENMASK(9, 8), rfe); + val = readl(rsys); + if (val & BIT(28)) + writel(val & ~(BIT(28) | GENMASK(9, 8)), rsys); + writel((readl(rsys) & ~GENMASK(3, 0)) | GENMASK(9, 8), rsys); + msleep(50); + if (rfab) { + if (readl_poll_timeout(rsys, cur, !(cur & (0xf0 | BIT(11))), 50, 50000)) + pr_warn("dcp %s: sys still %#x with fab6_soc held\n", name, cur); + /* Back to automatic, as it was; the flag bits are write-one-to-clear, left alone. */ + writel((readl(rfab) & ~(GENMASK(9, 8) | BIT(28))) | (fab & BIT(28)), rfab); + msleep(5); + iounmap(rfab); + } +out: + if (gapf) + iounmap(gapf); + if (rcpu) + iounmap(rcpu); + if (rfe) + iounmap(rfe); + if (rsys) + iounmap(rsys); + if (ra) + iounmap(ra); + if (rb) + iounmap(rb); +} + +/* + * After a real wake: the internal DCP back, as linux#1477 does it for the M4 Pro + * (t6040_disp_back, dcp_t6041_s2r_domains_on, dcp_t6041_s2r_resume, the warm restart and the + * mode set of dcp_hib_recommit): DISP_SYS on, the retention pair (A := 0, B := 1), DISP_FE on, + * the GAPF protection off, DISP_CPU on, its 11 children as the stop found them, the PMC voter, + * both DARTs' streams, the idled firmware restarted, the pipe powered and the mode set again. + */ +static int dcp_t6041_recommit(struct apple_dcp *dcp) +{ + struct apple_crtc *acrtc = READ_ONCE(dcp->crtc); + struct drm_modeset_acquire_ctx ctx; + struct drm_atomic_commit *state; + struct drm_crtc_state *cs; + struct drm_crtc *crtc; + int ret; + + iomfb_poweron_v26_6_desktop(dcp); + /* Room for the firmware to bring its link and hotplug state back before the mode set. */ + msleep(DCP_S2R_SETTLE_MS); + if (!acrtc) + return 0; + crtc = &acrtc->base; + DRM_MODESET_LOCK_ALL_BEGIN(crtc->dev, ctx, 0, ret); + if (!crtc->state->active) { + ret = 0; + goto unlock; + } + state = drm_atomic_commit_alloc(crtc->dev); + if (!state) { + ret = -ENOMEM; + goto unlock; + } + state->acquire_ctx = &ctx; + cs = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(cs)) { + ret = PTR_ERR(cs); + goto put; + } + cs->mode_changed = true; + ret = drm_atomic_add_affected_connectors(state, crtc); + if (!ret) + ret = drm_atomic_add_affected_planes(state, crtc); + if (!ret) + ret = drm_atomic_commit(state); +put: + drm_atomic_commit_put(state); +unlock: + DRM_MODESET_LOCK_ALL_END(crtc->dev, ctx, ret); + return ret; +} + +void dcp_backlight_s2r_restore(struct apple_dcp *dcp); + +static int dcp_t6041_start_one(struct device *dev, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + void __iomem *ps, *pg, *gapf, *pmc; + u32 v, cur, to = 0; + int ret, i; + + if (!dcp || IS_ERR_OR_NULL(dcp->rtk) || !strstarts(dev_name(dev), "402e")) + return 0; + ps = ioremap(0x502280000UL, 0x14000); + pg = ioremap(0x5022a1000UL, 0x100); + gapf = ioremap(0x500578540UL, 4); + pmc = ioremap(0x5048a9c14UL, 8); + if (!ps || !pg || !gapf || !pmc) { + dev_err(dev, "cannot map the display power words after the sleep\n"); + goto unmap; + } + + v = readl(ps + 0x258) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + /* + * As macOS's wake writes them: target on with both flag bits clear (0xf). Raised with the + * flags kept (0x30f) the word reads 0x3ff, a later write can't clear them, and the next + * lowering then latches bit 11 on SYS (0x800) and leaves FE at 0x200: the sleep engine waits + * on that bit and the chip resets (second sleep of a boot, 11 Nov). + */ + writel((v & ~0x30fU) | 0xf, ps + 0x258); + if (readl_poll_timeout(ps + 0x258, cur, (cur & 0xf0) == 0xf0 && !(cur & BIT(11)), 50, 10000)) + to |= BIT(0); + writel(0, pg + 0x1c); + writel(1, pg + 0x60); + /* + * Bridge 4 stays as boot has it (bit 0 of both words clear, 0x10000 / 0x101330), unlike + * macOS's wake: with it set, the driver's own display power-off after a wake froze the whole + * machine right after "dcp_poweroff() done" (login after a wake, sddm restart; 11 Nov). + */ + v = readl(ps + 0x368) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel((v & ~0x30fU) | 0xf, ps + 0x368); + if (readl_poll_timeout(ps + 0x368, cur, (cur & 0xf0) == 0xf0, 50, 10000)) + to |= BIT(1); + writel(readl(gapf) & ~BIT(4), gapf); + v = readl(ps + 0x10000); + writel((v & ~(0x30fU | BIT(28))) | 0xf, ps + 0x10000); + mb(); + if (readl_poll_timeout(ps + 0x10000, cur, (cur & 0xf0) == 0xf0 && !(cur & BIT(11)), 50, 10000)) + to |= BIT(2); + for (i = 0; i < 11 && dcp_t6041_child_valid; i++) { + void __iomem *cw = ps + 0x10000 + 8 * (i + 1); + u32 want = dcp_t6041_child_saved[i]; + + if (!(want & 0xf) || (readl(cw) & 0xf0)) + continue; + writel(want, cw); + if (readl_poll_timeout(cw, cur, cur & 0xf0, 50, 2000)) + to |= BIT(8 + i); + } + dcp_pmc_set(dcp, pmc, true); + if (to) + dev_warn(dev, "display domains not on after the sleep: %#x\n", to); +unmap: + if (ps) + iounmap(ps); + if (pg) + iounmap(pg); + if (gapf) + iounmap(gapf); + if (pmc) + iounmap(pmc); + if (!ps || !pg || !gapf || !pmc) + return 0; + + dcp_s2r_find_darts(dcp); + if (dcp->s2r_disp_dart) { + apple_dart_display_keep(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, false); + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, true, false); + } + if (dcp->s2r_dcp_dart) { + apple_dart_display_keep(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, false); + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + } + dcp_s2r_put_darts(dcp); + + WRITE_ONCE(dcp->crashed, false); + WRITE_ONCE(dcp->ext_dead, false); + ret = apple_rtkit_power_cycled_return(dcp->rtk, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (ret) { + dev_err(dev, "the firmware's warm restart after the sleep failed: %d\n", ret); + WRITE_ONCE(dcp->crashed, true); + return 0; + } + WRITE_ONCE(dcp->s2r_fw_down, false); + /* + * Before the mode set, not after it: the commit's swap is what hands the firmware the + * console's framebuffer, and fbcon draws into that memory without committing again. With + * presentation still paused the swap was dropped and the panel stayed black with everything + * else up (10 Nov). + */ + iomfb_pause_presentation(dcp, false); + ret = dcp_t6041_recommit(dcp); + if (ret) + dev_warn(dev, "mode set after the sleep failed: %d\n", ret); + if (!READ_ONCE(dcp->crashed)) + iomfb_pause_presentation(dcp, false); + dcp_backlight_s2r_restore(dcp); + return 0; +} + +static bool dcp_t6041_down, dcp_t6041_words; + +/* + * The internal display's SYS, FE and CPU power words as they were before the stop (from boot: + * SYS and FE with no flags, the CPU word 0x1000030f, automatic). The restart leaves the CPU word + * manual (0x0f0003ff) and SYS/FE with both flags, and the first display power-off after such a + * wake hung the machine (11 Nov, run 6). Put back once the display runs again. + */ +static u32 dcp_t6041_boot_ps[3]; + +static void dcp_t6041_boot_ps_io(bool restore) +{ + static const u32 off[3] = { 0x258, 0x368, 0x10000 }; + void __iomem *ps = ioremap(0x502280000UL, 0x14000); + int i; + + if (!ps) + return; + for (i = 0; i < 3; i++) { + if (!restore) + dcp_t6041_boot_ps[i] = readl(ps + off[i]); + else if (dcp_t6041_boot_ps[i]) + writel(dcp_t6041_boot_ps[i] & ~0xf0U, ps + off[i]); + } + iounmap(ps); +} + +/* @np's first DART, dark if its driver is bound. */ +static void dcp_t6041_dart_dark(struct device_node *np) +{ + struct device_node *dart = np ? of_parse_phandle(np, "iommus", 0) : NULL; + struct platform_device *pdev = dart ? of_find_device_by_node(dart) : NULL; + + of_node_put(dart); + if (!pdev) + return; + if (device_is_bound(&pdev->dev)) + apple_dart_power_dark(&pdev->dev, true); + put_device(&pdev->dev); +} + +/* + * Every external chain's DARTs, dark before their domains go off. The display subsystem maps + * every framebuffer through the display DARTs, and an unmap after the wake (an sddm restart, a + * logout) flushed one behind a dead block and stopped the machine (12 Nov). By node, so a chain + * whose DCP never bound, which the display subsystem skips as unusable, is covered too, and so + * is every chain a board enables, not only the first two. + */ +static void dcp_t6041_ext_darts_dark(void) +{ + struct device_node *np, *piodma; + + for_each_compatible_node(np, NULL, "apple,t6041-dcpext") { + if (!of_device_is_available(np)) + continue; + piodma = of_get_child_by_name(np, "piodma"); + dcp_t6041_dart_dark(piodma); + of_node_put(piodma); + dcp_t6041_dart_dark(np); + } +} + +enum dcp_t6041_step { DCP_S2R_FW_STOP, DCP_S2R_WORDS_DOWN, DCP_S2R_START }; + +/* Each step once per sleep. */ +static int dcp_t6041_s2r_step(enum dcp_t6041_step step) +{ + bool external = true; + int ret; + + if (step == DCP_S2R_START) { + if (!dcp_t6041_down) + return 0; + dcp_t6041_down = false; + dcp_t6041_words = false; + ret = driver_for_each_device(&apple_platform_driver.driver, NULL, NULL, + dcp_t6041_start_one); + msleep(DCP_S2R_WORDS_MS); + dcp_t6041_boot_ps_io(true); + return ret; + } + if (step == DCP_S2R_WORDS_DOWN) { + if (dcp_t6041_words || !dcp_t6041_down) + return 0; + dcp_t6041_words = true; + dcp_t6041_ext_darts_dark(); + /* Never started, but the complex counts them. */ + dcp_t6041_word_try("ext3", 0x740, 0x5028002e8UL, 0x5028002e0UL, 0x5028002d8UL, + 0x50282dfb4UL, 0x50282dfd8UL); + dcp_t6041_word_try("ext2", 0xd80, 0x5028004f0UL, 0x5028004e8UL, 0x5028004e0UL, + 0x50282db34UL, 0x50282db58UL); + dcp_t6041_word_try("ext1", 0xd00, 0x5028004a0UL, 0x502800498UL, 0x502800490UL, + 0x50282d6b4UL, 0x50282d6d8UL); + dcp_t6041_word_try("ext0", 0x840, 0x502800350UL, 0x502800348UL, 0x502800340UL, + 0x50282d234UL, 0x50282d258UL); + dcp_t6041_word_try("main", 0x540, 0x502290000UL, 0x502280368UL, 0x502280258UL, + 0x5022a101cUL, 0x5022a1060UL); + return 0; + } + if (dcp_t6041_down) + return 0; + dcp_t6041_down = true; + dcp_t6041_boot_ps_io(false); + ret = driver_for_each_device(&apple_platform_driver.driver, NULL, &external, + dcp_t6041_stop_one); + external = false; + return ret ?: driver_for_each_device(&apple_platform_driver.driver, NULL, &external, + dcp_t6041_stop_one); +} + +/* T6041: the display down for a real sleep and back after it. */ +void dcp_t6041_s2r_all(bool start) +{ + if (!of_machine_is_compatible("apple,t6041")) + return; + if (start) { + dcp_t6041_s2r_step(DCP_S2R_START); + return; + } + if (dcp_t6041_down) + return; + dcp_t6041_s2r_step(DCP_S2R_FW_STOP); + msleep(DCP_S2R_STOP_MS); + dcp_t6041_s2r_step(DCP_S2R_WORDS_DOWN); + msleep(DCP_S2R_DOWN_MS); +} + int __init dcp_register(void) { return platform_driver_register(&apple_platform_driver); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index f3222b1e8523e..e65e8a9692308 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -140,6 +140,7 @@ int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); bool dcp_s2r_firmware_stopped(struct platform_device *pdev); +void dcp_t6041_s2r_all(bool start); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 00f29157241f8..38bd72726e9be 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -408,6 +408,34 @@ void dcp_backlight_finish_fade(struct apple_dcp *dcp) dcp_backlight_kick(dcp); } +/* + * After the S2R display restart: ask for the user's level again and send it, whatever the + * backlight core's flags say at that moment. The class device resumes on its own schedule: + * when it runs after the restart the panel stays dark until then (suspended and core_blank + * still set at the mode set, 11 Nov), and when it runs before, its request goes to a firmware + * that isn't up yet and nothing sends it again (the dark lid wakes). + */ +void dcp_backlight_s2r_restore(struct apple_dcp *dcp) +{ + struct backlight_device *bd = READ_ONCE(dcp->brightness.bl_dev); + unsigned long flags; + + if (!dcp_backlight_desktop(dcp) || !bd) + return; + spin_lock_irqsave(&dcp->backlight.lock, flags); + /* + * A level sent to the firmware that the stop then took down never gets its answer: the + * state stayed in_flight for good and nothing was sent again (dark lid wake of 20:30, + * "sent 0 ... dirty 1 in_flight 1"). The restarted firmware knows nothing of it. + */ + dcp->backlight.state.in_flight = false; + dcp->backlight.state.retries = 0; + dcp_bl_request(&dcp->backlight.state, bd->props.brightness, false, false); + dcp->backlight.state.dirty = true; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + dcp_backlight_kick(dcp); +} + int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b38a0669dfb0e..f19dfac5953a1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2626,6 +2626,40 @@ int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_call } #endif +#if DCP_FW_DESKTOP_26(DCP_FW_VER) +DCP_THUNK_IN(dcp_unnamed_500, dcpep_unnamed_500, u32); + +static void dcp_s2r_500_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); +} + +/* Sleep test: H17P's A500, same number and bytes as this firmware's neighbours. */ +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + u32 val = 0xaaaa0000; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + kref_get(&cookie->refcount); + + dcp_unnamed_500(dcp, false, &val, dcp_s2r_500_done, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} +#endif + static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { /* H17P does not confirm that scanout has stopped. */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 93e8895e9fb48..4ecfcc4640183 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -389,6 +389,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); +#if DCP_FW_DESKTOP_26(DCP_FW_VER) +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp); +#endif #if DCP_FW_H17(DCP_FW_VER) int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp); int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob); diff --git a/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c b/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c index c2d0bee01e05f..d56a03a078f68 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c @@ -44,6 +44,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A469", iomfbep_abort_swaps_dcp), IOMFB_METHOD("A473", dcpep_set_power_state), IOMFB_METHOD("A474", dcpep_pipe_query_474), + IOMFB_METHOD("A500", dcpep_unnamed_500), /* A364 get_mode_info and A420/A421 gamma: see iomfb_template.c */ }; @@ -234,9 +235,11 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [576] = trampoline_nop, [581] = trampoline_nop, [582] = trampoline_true, /* create_default_fb_surface */ + [583] = trampoline_nop, /* clear_default_surface */ [588] = trampoline_nop, /* resize_default_fb_surface_gated */ [590] = trampoline_swap_complete, [592] = trampoline_swap_complete_intent_gated, + [597] = trampoline_zero, /* sleep test: (0, 4), purpose unknown */ [593] = trampoline_abort_swap_ap_gated, /* a four-byte padded notification; only its first byte is read */ [594] = trampoline_enable_backlight_message_ap_gated, @@ -323,9 +326,11 @@ static const struct dcp_cb_min_v26_6_desktop cb_min[IOMFB_MAX_CB] = { [576] = CB_MIN(0, 0), [581] = CB_MIN(0, 0), [582] = CB_MIN(0, sizeof(u8)), + [583] = CB_MIN(0, 0), [588] = CB_MIN(0, 0), [590] = CB_MIN(sizeof(struct DCP_FW_NAME(dc_swap_complete_resp)), 0), [592] = CB_MIN(sizeof(dcp_swap_intent_t), 0), + [597] = CB_MIN(0, sizeof(u32)), [593] = CB_MIN(sizeof(u32), 0), [594] = CB_MIN(sizeof(u8), 0), [595] = CB_MIN(0, 0), diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c index 6207c99613172..6a6b22bbb7759 100644 --- a/drivers/gpu/drm/asahi/g17/sleep.c +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -56,12 +56,30 @@ int asahi_g17_pmp_request(struct device *dev, bool enable) return ret; } +/* + * The node of the GPU's power state word. Where the GPU's first power domain is its PMP + * report entry (a small index for a reg, under the PMP), as in the J616C device tree the + * bring-up runs, the word is that entry's own domain. + */ +static struct device_node *asahi_g17_ps_node(struct device *dev) +{ + struct device_node *np, *next; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np && of_device_is_compatible(np, "apple,t6000-pmp-v2-report-entry")) { + next = of_parse_phandle(np, "power-domains", 0); + of_node_put(np); + np = next; + } + return np; +} + static int asahi_g17_ps(struct device *dev, struct regmap **regmap, u32 *offset) { struct device_node *np; int ret; - np = of_parse_phandle(dev->of_node, "power-domains", 0); + np = asahi_g17_ps_node(dev); if (!np) return -EINVAL; ret = of_property_read_u32(np, "reg", offset); @@ -296,7 +314,7 @@ struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, } ret = -ENODEV; - np = of_parse_phandle(dev->of_node, "power-domains", 0); + np = asahi_g17_ps_node(dev); if (!np) goto err; if (!of_property_read_u32(np, "reg", &offset) && np->parent) diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 98a36216528e2..f5394f8b153de 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -155,6 +155,8 @@ const CONFIG_25G83: Config = Config { reserved_slots: &[], chip: Some(&CHIP_25G83), firmware: fw::variant::Firmware::Identified(&[BUILD_25G83]), + // The GPU is restarted after a real sleep, as on the 32-core part. + sleep_restart: true, ..BASE }; diff --git a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs index e23b419364a78..7872412f21775 100644 --- a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs +++ b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs @@ -60,6 +60,8 @@ pub(super) const CONFIG: Config = Config { // hang this GPU's firmware on grids of more than ten workgroups. compute_blocks: 96, }, + // The GPU is restarted after a real sleep, as on T8140. + sleep_restart: true, ..t6041::BASE }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 2c4350a0f29fe..5b2d7c0c75b56 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -6887,6 +6887,56 @@ EXPORT_SYMBOL_GPL(apple_dart_gapf_ready); * Stopping again after a sleep that never reached the commit does nothing: the * domains may be off by then. */ +/* + * T6041: a display DART is locked, so the driver neither saves nor restores its TCR / TTBR, and + * after a real sleep the panel stayed black with everything else up. Keep the words of @streams + * before the sleep (save), and after it write back what differs (!save). Returns how many words + * were written back. + */ +int apple_dart_display_keep(struct device *dev, u32 streams, bool save) +{ + static struct { struct apple_dart *dart; u32 tcr[32]; u32 ttbr[32][4]; } keep[2]; + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long bits = streams; + unsigned int sid, idx, k, n = 0; + + if (!dart) + return -ENODEV; + for (k = 0; k < 2 && keep[k].dart && keep[k].dart != dart; k++) + ; + if (k == 2) + return -ENOSPC; + if (!save && keep[k].dart != dart) + return -ENOENT; + keep[k].dart = dart; + for_each_set_bit(sid, &bits, 32) { + u32 tcr = readl(dart->regs + DART_TCR(dart, sid)); + + if (sid >= dart->num_streams) + break; + if (save) + keep[k].tcr[sid] = tcr; + for (idx = 0; idx < dart->hw->ttbr_count && idx < 4; idx++) { + u32 ttbr = readl(dart->regs + DART_TTBR(dart, sid, idx)); + + if (save) { + keep[k].ttbr[sid][idx] = ttbr; + continue; + } + if (ttbr != keep[k].ttbr[sid][idx]) { + writel(keep[k].ttbr[sid][idx], dart->regs + DART_TTBR(dart, sid, idx)); + n++; + } + } + if (!save && tcr != keep[k].tcr[sid]) { + writel(keep[k].tcr[sid], dart->regs + DART_TCR(dart, sid)); + n++; + } + } + return n; +} +EXPORT_SYMBOL_GPL(apple_dart_display_keep); + int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush) { struct apple_dart *dart = dev_get_drvdata(dev); diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 393596ff6231b..8d70b320b0fd7 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -341,6 +341,18 @@ static const struct aic_info aic3_t8140_info __initconst = { .fw_suspend = true, }; +/* + * T6041 (M4 Max): the AICv3 of T8122, with the state its sleep driver's firmware + * suspend loses saved and restored around it. + */ +static const struct aic_info aic3_t6041_info __initconst = { + .version = 3, + .irq_cfg = AIC3_IRQ_CFG, + .fast_ipi = true, + .local_fast_ipi = true, + .fw_suspend = true, +}; + static const struct aic_info aic3_t8142_info __initconst = { .version = 3, .irq_cfg = AIC3_IRQ_CFG, @@ -358,6 +370,10 @@ static const struct of_device_id aic_info_match[] = { .compatible = "apple,t8142-aic3", .data = &aic3_t8142_info, }, + { + .compatible = "apple,t6041-aic3", + .data = &aic3_t6041_info, + }, { .compatible = "apple,t8103-aic", .data = &aic1_local_fipi_info, diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 7c525b6c7b2e1..42b620ee7dc58 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -759,6 +759,18 @@ static int apple_smc_sleep_commit(struct apple_smc *smc) { int ret; + /* + * T6041: macOS's last SMC writes before MBSE (trace of its sleep under aura's hypervisor): + * WKTP = 2 (AppleSMCEmbedded, going to sleep), and pmVC = 0x100, AppleT6041PMGR's + * enableVddCio with every port's CIO rail off. + */ + if (of_machine_is_compatible("apple,t6041")) { + int r1 = apple_smc_write_u8(smc, SMC_KEY(WKTP), 2); + int r2 = apple_smc_write_u32(smc, SMC_KEY(pmVC), 0x101); + + if (r1 || r2) + dev_err(smc->dev, "WKTP %d, pmVC %d\n", r1, r2); + } apple_smc_write_mbse(smc, SMC_KEY(slpa)); /* Device interrupts are off here. */ mdelay(SMC_SLEEP_ARM_MS); diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f0892f4986b94..4d04df8e96366 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2454,6 +2454,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) return ERR_PTR(ret); anv->adminq.is_adminq = true; platform_set_drvdata(pdev, anv); + /* Its resume waits on the coprocessor (0.2 to 0.9 s after a real sleep): not in the others' way. */ + device_enable_async_suspend(&pdev->dev); anv->hw = of_device_get_match_data(&pdev->dev); if (!anv->hw) { @@ -3169,6 +3171,10 @@ static const struct apple_nvme_hw apple_nvme_t6040_hw = { .needs_ioq_registers = true, .lsq_in_nvmmu = true, .max_queue_depth = 64, + /* ANS takes part in a firmware sleep, whole endpoint map echoed. */ + .sleep_traced = true, + .sleep_flush_max_nsid = 1, + .sleep_shn = NVME_CC_SHN_MASK, .quirks = NVME_QUIRK_NO_APST, }; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 8395a73bb8396..1272bffb059e9 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -73,6 +73,11 @@ static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); +/* A real sleep takes a conventional host's power: its ports are set up again as at probe. */ +static bool s2r_conventional = true; +module_param(s2r_conventional, bool, 0644); +MODULE_PARM_DESC(s2r_conventional, "Set a conventional host's ports up again after a real S2R"); + /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -533,6 +538,7 @@ struct apple_pcie { const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; + struct list_head hib_fns; struct completion event; struct irq_fwspec fwspec; struct irq_domain *msi_domain; @@ -561,6 +567,9 @@ struct apple_pcie_port { struct gpio_desc *perst_gpio; struct gpio_desc *pwren_gpio; int idx; + /* Conventional port: link up when the sleep began, and found down after it */ + bool hib_link_up; + bool hib_cold; bool started; bool needs_stop; bool link_failed; @@ -4760,6 +4769,9 @@ static int apple_pcie_probe(struct platform_device *pdev) struct device *piodma_supplier = NULL; int ret; + /* Setting the ports up again after a real sleep takes ~0.75 s: resume in parallel. */ + device_enable_async_suspend(&pdev->dev); + hw = of_device_get_match_data(dev); if (!hw) return -ENODEV; @@ -4882,6 +4894,7 @@ static int apple_pcie_probe(struct platform_device *pdev) mutex_init(&pcie->lock); INIT_LIST_HEAD(&pcie->ports); + INIT_LIST_HEAD(&pcie->hib_fns); /* Resolve PM dependencies before publishing the PCI hierarchy. */ if (pcie->hw->tunneled) { @@ -5677,6 +5690,10 @@ static void apple_pcie_stop_for_sleep(struct device *dev) static void apple_pcie_neo_save_functions(struct apple_pcie *pcie); +static bool apple_pcie_s2r_conventional(struct apple_pcie *pcie); +static void apple_pcie_hib_save(struct apple_pcie *pcie); +static void apple_pcie_hib_restore_noirq(struct apple_pcie *pcie, const char *what); + /* * The J716c radio drivers save PCI state without entering D3. The PCI core * keeps the root port in D0 as well. Its host and DART have no switchable @@ -5775,6 +5792,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) return ret; } if (!pcie->hw->tunneled) { + if (apple_pcie_s2r_conventional(pcie) && pm_suspend_via_firmware()) + apple_pcie_hib_save(pcie); if (pm_suspend_no_platform() && pcie->num_pds) return apple_pcie_hold_domains(pcie); return 0; @@ -6173,8 +6192,11 @@ static int apple_pcie_resume_noirq(struct device *dev) } return ret; } - if (!pcie->hw->tunneled) + if (!pcie->hw->tunneled) { + if (apple_pcie_s2r_conventional(pcie) && pm_resume_via_firmware()) + apple_pcie_hib_restore_noirq(pcie, "S2R"); return 0; + } if (pcie->resume_failed) return -EIO; /* @@ -6241,9 +6263,194 @@ static int apple_pcie_resume_noirq(struct device *dev) return ret; } +/* + * A conventional host (the internal ports with the radio and the SD reader) loses its power + * in a real sleep: the endpoints sit unpowered or in reset behind a link that is down. The + * resume then does what probe did. From aurora-silicon/linux#1477 (0ab1a6603). + */ +struct apple_pcie_hib_fn { + struct list_head entry; + struct pci_dev *pdev; + struct pci_saved_state *state; +}; + +static bool apple_pcie_s2r_conventional(struct apple_pcie *pcie) +{ + return READ_ONCE(s2r_conventional) && !pcie->hw->tunneled && !pcie->managed_sids && + !pcie->piodma_supplier && !pcie->hw->root_bus_only; +} + +/* What only the port registers hold. */ +static void apple_pcie_hib_save(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i; + + mutex_lock(&pcie->lock); + list_for_each_entry(port, &pcie->ports, entry) { + port->hib_link_up = apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP; + port->hib_cold = false; + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + } + mutex_unlock(&pcie->lock); +} + +static void apple_pcie_hib_free_fns(struct apple_pcie *pcie) +{ + struct apple_pcie_hib_fn *fn, *tmp; + + list_for_each_entry_safe(fn, tmp, &pcie->hib_fns, entry) { + list_del(&fn->entry); + kfree(fn->state); + pci_dev_put(fn->pdev); + kfree(fn); + } +} + +/* + * The PCI core's noirq resume spends the functions' saved state on a link that is down + * ("device inaccessible"). A copy taken first goes back once the link is. + */ +static int apple_pcie_hib_keep_fn(struct pci_dev *pdev, void *data) +{ + struct apple_pcie *pcie = data; + struct apple_pcie_port *port; + struct apple_pcie_hib_fn *fn; + + if (pci_is_root_bus(pdev->bus) || !pdev->state_saved) + return 0; + port = apple_pcie_get_port(pdev); + if (!port || !port->hib_cold) + return 0; + fn = kzalloc(sizeof(*fn), GFP_ATOMIC); + if (!fn) + return 0; + fn->state = pci_store_saved_state(pdev); + if (!fn->state) { + kfree(fn); + return 0; + } + fn->pdev = pci_dev_get(pdev); + list_add_tail(&fn->entry, &pcie->hib_fns); + return 0; +} + +/* Before the PCI core restores the functions; nothing is written to a port in this phase. */ +static void apple_pcie_hib_restore_noirq(struct apple_pcie *pcie, const char *what) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + bool cold = false; + + apple_pcie_hib_free_fns(pcie); + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->hib_link_up) + continue; + if (apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP) + continue; + port->hib_cold = true; + cold = true; + dev_info(pcie->dev, "%s: %pOF lost its link, setting it up again\n", + what, port->np); + } + if (cold && bridge->bus) + pci_walk_bus(bridge->bus, apple_pcie_hib_keep_fn, pcie); +} + +static void apple_pcie_hib_rid2sid(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + int i; + + mutex_lock(&pcie->lock); + for (i = 0; i < port->sid_map_sz; i++) + apple_pcie_rid2sid_write(port, i, test_bit(i, port->sid_map) ? + port->saved_rid2sid[i] : 0); + mutex_unlock(&pcie->lock); +} + +/* Probe's bring-up of a conventional port, with the state this kernel had programmed. */ +static int apple_pcie_hib_port_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + ret = apple_pcie_setup_link(pcie, port, port->np); + if (ret) + return ret; + if (!pcie->hw->retain_clocks) { + if (pcie->hw->port_refclk) + rmw_clear(PORT_REFCLK_CGDIS, port->base + pcie->hw->port_refclk); + else if (port->phy) + rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + } + /* All port interrupts masked and cleared, MSI routing as at probe. */ + apple_pcie_tunnel_restore_irq_hw(port); + apple_pcie_hib_rid2sid(pcie, port); + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, + stat & PORT_LINKSTS_UP, 100, link_up_timeout * 1000); + if (ret) { + dev_err(pcie->dev, "S2R: %pOF status %#x link %#x after training\n", port->np, + apple_pcie_port_readl(port, PORT_STATUS), stat); + return ret; + } + apple_pcie_port_writel(port, BIT(PORT_INT_LINK_UP), PORT_INTSTAT); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + /* 100 ms after the link before configuration requests (PCIe r5.0, 6.6.1). */ + msleep(100); + return 0; +} + +/* The normal phase, after the SMC (the PWREN GPIOs are its) and before the functions. */ +static int apple_pcie_resume(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + struct apple_pcie_port *port; + struct apple_pcie_hib_fn *fn; + int ret; + + if (!apple_pcie_s2r_conventional(pcie) || !pm_resume_via_firmware()) + return 0; + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->hib_link_up) + continue; + if (!port->hib_cold) { + apple_pcie_hib_rid2sid(pcie, port); + continue; + } + ret = apple_pcie_hib_port_up(pcie, port); + if (ret) { + dev_err(dev, "S2R: %pOF link not back (%d), its functions stay unreachable\n", + port->np, ret); + port->hib_cold = false; + continue; + } + dev_info(dev, "S2R: %pOF link back, PWREN and PERST# cycled as at probe\n", + port->np); + } + list_for_each_entry(fn, &pcie->hib_fns, entry) { + port = apple_pcie_get_port(fn->pdev); + if (!port || !port->hib_cold) + continue; + if (pci_load_saved_state(fn->pdev, fn->state)) + pci_warn(fn->pdev, "S2R: saved configuration not taken back\n"); + else + pci_restore_state(fn->pdev); + } + apple_pcie_hib_free_fns(pcie); + list_for_each_entry(port, &pcie->ports, entry) + port->hib_cold = false; + return 0; +} + static const struct dev_pm_ops apple_pcie_pm_ops = { .suspend_noirq = apple_pcie_suspend_noirq, .resume_noirq = apple_pcie_resume_noirq, + .resume = apple_pcie_resume, }; static const struct of_device_id apple_pcie_of_match[] = { diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 3e8ab5a9b933b..e070094ab67fd 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -324,6 +324,15 @@ static void apple_gpio_irq_unmask(struct irq_data *data) gpiochip_enable_irq(gc, data->hwirq); apple_gpio_set_reg(pctl, data->hwirq, REG_GPIOx_MODE, FIELD_PREP(REG_GPIOx_MODE, irqtype)); + /* + * On T6041 a level interrupt fired again at once every time it was unmasked although its + * line was inactive (J616c speaker amps, pin 55: mode 3, data 1, ~140 interrupts a second + * from boot until the core disabled the line as unhandled). Switching the mode back from + * "off" leaves the status bit set. Clear it once more: a level that really is active sets + * it again. + */ + if (irqd_get_trigger_type(data) & IRQ_TYPE_LEVEL_MASK) + apple_gpio_irq_ack(data); } static unsigned int apple_gpio_irq_startup(struct irq_data *data) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index f0f6080e11a5e..a720ea04f32af 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -108,6 +108,10 @@ config APPLE_SOC_SLEEP each by its match data: today T8140. On every other SoC the driver doesn't bind, and s2idle stays the only system sleep. + T6041 (M4 Max) has a driver of its own, t6041-sleep, which binds + to an "apple,t6041-sleep" node and uses this driver's coprocessor + stop. + config APPLE_SOC_SLEEP_KUNIT_TEST bool "KUnit tests for the Apple SoC sleep match data" if !KUNIT_ALL_TESTS depends on APPLE_SOC_SLEEP && KUNIT=y diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 5065bb7a1324a..013f7f83b131f 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o obj-$(CONFIG_APPLE_SOC_SLEEP) += apple-sleep.o apple-sleep-y = sleep.o +obj-$(CONFIG_APPLE_SOC_SLEEP) += t6041-sleep.o obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 2347073ddcdd7..21ff31399e3bb 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -1983,8 +1983,10 @@ impl AopData { /// (one to one when the IOVA is the physical address, translated /// otherwise), so they are only checked here. Every page has to be mapped /// where the DT says, the external code (`__ETEXT`) at `ec0p` and any - /// other segment (`__OS_LOG`) at its physical address. - fn check_fw_regions(dev: &device::Device, ec0p: u64) -> Result<()> { + /// other segment (`__OS_LOG`) at its physical address. With `above_ec0p` + /// (27.2's firmware, which adds a segment at `ec0p + 0x20000` on T6041), a + /// segment above `ec0p` is translated too. + fn check_fw_regions(dev: &device::Device, ec0p: u64, above_ec0p: bool) -> Result<()> { // SAFETY: `dev` is a live device being probed. let np = unsafe { (*dev.as_raw()).of_node }; if np.is_null() { @@ -2029,7 +2031,7 @@ impl AopData { return Err(EINVAL); } etext = true; - } else if iova != phys { + } else if iova != phys && !(above_ec0p && iova > ec0p) { dev_err!( dev, "Firmware memory-region {} at IOVA {:#x}, neither EC0p {:#x} nor its physical address", @@ -2820,6 +2822,9 @@ struct AopHwConfig { ncal: Option, alig: u64, aopt: u64, + /// The `ED0p` boot argument, or `None` to keep the firmware's own value; with it, a + /// firmware segment above `ec0p` is accepted at a translated IOVA too. + ed0p: Option, /// Complete the firmware's boot arguments before starting it. patch_bootargs: bool, /// The firmware speaks EPIC with version 4 sub-headers. @@ -2853,6 +2858,7 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { ncal: Some(0), aopt: 1, alig: 128, + ed0p: None, patch_bootargs: true, epic_v4: false, setup_port: false, @@ -2873,6 +2879,7 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { ncal: None, aopt: 0, alig: 128, + ed0p: None, patch_bootargs: true, epic_v4: false, setup_port: false, @@ -2892,6 +2899,7 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { ncal: None, aopt: 0, alig: 64, + ed0p: None, patch_bootargs: true, epic_v4: false, setup_port: false, @@ -2910,6 +2918,7 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { ncal: Some(0), aopt: 0, alig: 64, + ed0p: None, patch_bootargs: true, epic_v4: false, setup_port: false, @@ -2933,6 +2942,7 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { ncal: Some(1), aopt: 0, alig: 64, + ed0p: None, patch_bootargs: true, epic_v4: false, setup_port: false, @@ -2946,6 +2956,15 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { bootargs_word_access: true, halt_on_failed_start: true, }; +/// T6041 (J616C): as T6040, with the AFK sleep and wake order macOS 26.6.2 uses on the J616C +/// (trace, 8 Nov 2026: sleep to these endpoints in this order, then the AP idle notice), and +/// `ED0p` at `ec0p + 0x20000` for the 27.2 firmware's segment above `ec0p`. +const HW_CFG_T6041: AopHwConfig = AopHwConfig { + sleep_endpoints: &[0x2b, 0x2a, 0x24, 0x20, 0x27, 0x26, 0x28, 0x25, 0x21, 0x29, 0x23, 0x22], + wake_endpoints: &[0x22, 0x23, 0x29, 0x21, 0x25, 0x28, 0x26, 0x27, 0x20, 0x24, 0x2a, 0x2b], + ed0p: Some(0x0100_00000000 + 0x20000), + ..HW_CFG_T6040 +}; /// T8132 (J713): as T6040. The bootloader hands the AOP over the same way, with /// the DART's DAPF programmed and its windows reserved alongside the firmware /// segments. @@ -2960,6 +2979,7 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { ncal: Some(0), aopt: 0, alig: 0, + ed0p: None, patch_bootargs: false, epic_v4: true, setup_port: true, @@ -2977,6 +2997,7 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { /// ADT aop-exclave-mailbox) is not reachable from the AP: both bus-error. const HW_CFG_T8152: AopHwConfig = AopHwConfig { ec0p: 0, + ed0p: None, ncal: None, aopt: 0, alig: 0, @@ -3001,6 +3022,7 @@ const HW_CFG_T8152: AopHwConfig = AopHwConfig { /// firmware runs with the boot arguments iBoot left. const HW_CFG_T8142: AopHwConfig = AopHwConfig { ec0p: 0, + ed0p: None, ncal: None, aopt: 0, alig: 0, @@ -3078,7 +3100,7 @@ kernel::of_device_table!( (of::DeviceId::new(c_str!("apple,t6000-aop")), &HW_CFG_T6000), (of::DeviceId::new(c_str!("apple,t6020-aop")), &HW_CFG_T6020), (of::DeviceId::new(c_str!("apple,t8140-aop")), &HW_CFG_T8140), - (of::DeviceId::new(c_str!("apple,t6041-aop")), &HW_CFG_T6040), + (of::DeviceId::new(c_str!("apple,t6041-aop")), &HW_CFG_T6041), (of::DeviceId::new(c_str!("apple,t8132-aop")), &HW_CFG_T8132), (of::DeviceId::new(c_str!("apple,t6050-aop")), &HW_CFG_T6050), (of::DeviceId::new(c_str!("apple,t8142-aop")), &HW_CFG_T8142), @@ -3117,7 +3139,7 @@ impl platform::Driver for AopDriver { let asc_mmio = asc_mmio.as_view().relaxed(); if cfg.fw_regions { // Before anything is written to the AOP: no mapping, no start. - AopData::check_fw_regions(pdev.as_ref(), cfg.ec0p)?; + AopData::check_fw_regions(pdev.as_ref(), cfg.ec0p, cfg.ed0p.is_some())?; } let data = AopData::new(pdev, cfg)?; // Whatever fails below leaves the AOP running and its children @@ -3132,6 +3154,7 @@ impl platform::Driver for AopDriver { (from_fourcc(b"nCal"), cfg.ncal), (from_fourcc(b"alig"), Some(cfg.alig)), (from_fourcc(b"AOPt"), Some(cfg.aopt)), + (from_fourcc(b"ED0p"), cfg.ed0p), ], cfg.bootargs_word_access, )?; diff --git a/drivers/soc/apple/display-gate.c b/drivers/soc/apple/display-gate.c index ba2542e0e4a36..2e9cc1589968c 100644 --- a/drivers/soc/apple/display-gate.c +++ b/drivers/soc/apple/display-gate.c @@ -3094,7 +3094,39 @@ static const struct gate_staged *t6041_max_stage(unsigned int i) /* Applied, they stay: the live tree refers to them. Only the KUnit test reverts them. */ static struct of_changeset t6041_max_cs[ARRAY_SIZE(t6041_max_stages)]; -/* Enable every node of stage @i and put its display DART into the display subsystem's iommus. */ +/* + * Whether @np needs a power domain that stays disabled: neither enabled nor one of the nodes + * stage @st enables. + */ +static bool __init t6041_max_needs_off_domain(const struct gate_staged *st, struct device_node *np) +{ + struct of_phandle_args args; + bool off = false; + unsigned int n; + int i; + + for (i = 0; !off && !of_parse_phandle_with_args(np, "power-domains", "#power-domain-cells", + i, &args); i++) { + off = !of_device_is_available(args.np); + for (n = 0; off && n < st->nr; n++) { + struct device_node *dn = of_find_node_by_path(st->nodes[n]); + + off = dn != args.np; + of_node_put(dn); + } + of_node_put(args.np); + } + return off; +} + +/* + * Enable every node of stage @i and put its display DART into the display subsystem's iommus. + * The in-tree trees give the chains no DISPEXT power domain: they are on as iBoot left them. A + * tree that does describe them (the J616c's bring-up EL2 tree: dispextN_fe, and PMP report + * entries above them, all disabled) loses the property on the chain's nodes here, or each + * device would wait for a domain provider that never comes and give up at the deferred probe + * timeout, and the display subsystem with them. + */ static int __init t6041_max_apply(unsigned int i, struct device_node *display) { const struct gate_staged *st = t6041_max_stage(i); @@ -3110,6 +3142,9 @@ static int __init t6041_max_apply(unsigned int i, struct device_node *display) for (n = 0; !ret && n < st->nr; n++) { np = of_find_node_by_path(st->nodes[n]); ret = np ? of_changeset_update_prop_string(cs, np, "status", "okay") : -ENODEV; + if (!ret && t6041_max_needs_off_domain(st, np)) + ret = of_changeset_remove_property(cs, np, + of_find_property(np, "power-domains", NULL)); of_node_put(np); } if (!ret) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 28ec03e7e9fb7..02e56f7548a96 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -38,6 +38,9 @@ extern "C" { fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; fn sep_pm_clear_dev_disable(dev: *mut c_void) -> c_int; fn sep_pm_dart_restore(dev: *mut c_void); + fn sep_pm_dart_prepare(dev: *mut c_void) -> c_int; + fn sep_pm_wait_gated(dev: *mut c_void, timeout_ms: c_uint) -> c_int; + fn sep_pm_platform_commit() -> bool; fn sep_pm_commit_prepare( dev: *mut c_void, mbox: *mut c_void, @@ -128,6 +131,31 @@ pub(crate) fn commit_finish(dev: &kernel::device::Device) -> Option { } /// Puts the enclave's DART back if it lost its state, before the doorbell. +/// How long the enclave gets to gate its domain once it has answered (told at suspend). +const GATE_TIMEOUT_MS: u32 = 5000; + +/// Whether a sleep core commit sends the staged SLEEP; if not, it is sent at device suspend. +pub(crate) fn platform_commit() -> bool { + // SAFETY: only reads the sleep core's state. + unsafe { sep_pm_platform_commit() } +} + +/// Whether the enclave gated itself within the bound. +pub(crate) fn wait_gated(dev: &kernel::device::Device) -> bool { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_wait_gated(dev.as_raw().cast(), GATE_TIMEOUT_MS) == 0 } +} + +/// Saves the enclave's DART while it can be read, before the enclave is told. +pub(crate) fn dart_prepare(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_dart_prepare(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) +} + pub(crate) fn dart_restore(dev: &kernel::device::Device) { // SAFETY: `dev` is the bound SEP device. unsafe { sep_pm_dart_restore(dev.as_raw().cast()) }; diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index c1d415c0bd744..5f5ee20bd7db1 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -155,6 +155,39 @@ static struct device *sep_pm_dart(struct device *dev) return pdev ? &pdev->dev : NULL; } +/* + * Saves the enclave's DART while it can be read, before the enclave is told at suspend. Only + * a DART with apple,sleep-restore in its node is saved here and then left alone by its own + * callbacks. Without it, the DART's noirq suspend would read its registers after the enclave + * slept and its power domain went off, which hangs the suspend: refuse instead. + */ +int sep_pm_dart_prepare(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + int ret; + + if (!dart) + return -ENODEV; + if (!apple_dart_sleep_listed(dart)) { + dev_err(dart, "no apple,sleep-restore list: the enclave can't sleep at suspend\n"); + put_device(dart); + return -EOPNOTSUPP; + } + ret = apple_dart_sleep_prepare(dart); + put_device(dart); + + return ret; +} + +/* + * Whether a sleep core commit sends the staged SLEEP (T8140). Without one, a firmware suspend + * (T6041's own sleep driver) tells the enclave at device suspend instead. + */ +bool sep_pm_platform_commit(void) +{ + return apple_sleep_commit_stops_coprocs(); +} + void sep_pm_dart_restore(void *ptr) { struct device *dart = sep_pm_dart(ptr); diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index ff2c1bf973e04..c3adf46eb0642 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -868,7 +868,9 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: true, coreanalytics: Ep7::Always, - sram_backup: None, + // 26.6.2 firmware (J616C): __DATA at 0x500761000, 0x7d000 bytes; the 27.2 firmware has it at + // 0x62000. Kept across a real sleep. + sram_backup: Some((0x61000, 0x7d000)), }; /// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 9987fcd18dbab..084444c6b6384 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -216,6 +216,8 @@ const RNG_MAX_WORDS_PER_READ: usize = 16; // already being written can still go out once the sleep has begun; see // `send_entropy_request()`. const RNG_SLEEP_DRAIN_MS: u32 = proto::CONTROL_TIMEOUT_MS + 1000; +/// How long a full mailbox may take to drain before a sleep is refused. +const MAILBOX_DRAIN_MS: u32 = 100; extern "C" { fn sep_cancel_work_sync(work: *mut c_void); @@ -666,6 +668,11 @@ struct SepData { // The mailbox's receive interrupt is held off for the enclave's sleep. sleep_rx_held: Atomic, + /// Told at device suspend (no sleep core commit, T6041) rather than staged for the commit. + sleep_at_suspend: Atomic, + /// The enclave didn't answer a sleep request or a wake this boot: no deep sleep until a + /// reboot, as a commit over a running enclave resets the SoC. + sleep_unanswered: Atomic, #[pin] ool_xarm: Mutex>, @@ -1162,6 +1169,8 @@ impl SepData { phase: Atomic::new(PHASE_ATTACH), sleep_gate: Atomic::new(pm::SleepGate::Open as u32), sleep_rx_held: Atomic::new(false), + sleep_at_suspend: Atomic::new(false), + sleep_unanswered: Atomic::new(false), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), @@ -1458,9 +1467,19 @@ impl SepData { /// Prepare a firmware-driven suspend without sending SLEEP. The platform /// owns the irreversible request, after every returnable refusal. pub(crate) fn sleep_enter(&self) -> Result<()> { + // T6041: its sleep driver has no sleep core commit, so the enclave is told now. + if !pm::platform_commit() { + return self.sleep_enter_at_suspend(); + } match self.sleep_gate() { // A failed earlier wake is not repaired by another precommit attempt. - pm::SleepGate::Lost => return Err(EIO), + pm::SleepGate::Lost => { + dev_err!( + self.dev, + "deep sleep refused: the enclave did not come back from a sleep earlier this boot\n" + ); + return Err(EBUSY); + } pm::SleepGate::Open => {} _ => return Err(EBUSY), } @@ -1472,6 +1491,14 @@ impl SepData { ); return Err(EBUSY); } + // Refused here, at once, rather than at the commit after its send wait. + if !self.mailbox_drains() { + dev_err!( + self.dev, + "deep sleep refused: the enclave isn't taking messages (its mailbox stays full)\n" + ); + return Err(EBUSY); + } self.set_sleep_gate(pm::SleepGate::Draining); let idle = control::wait_idle( @@ -1501,6 +1528,178 @@ impl SepData { Ok(()) } + /// Without a sleep core commit (T6041): tell the enclave at device suspend, seconds before + /// the platform's commit, which answers, saves its state and gates its own domain (waited + /// for in suspend_noirq). A deep sleep commits only over an enclave that is asleep. + fn sleep_enter_at_suspend(&self) -> Result<()> { + if self.sleep_unanswered.load(Relaxed) { + dev_err!( + self.dev, + "deep sleep refused: the enclave did not answer a sleep request or wake earlier this boot\n" + ); + return Err(EBUSY); + } + match self.sleep_gate() { + // Gated already at an earlier, refused sleep: it stays asleep until a real wake. + pm::SleepGate::Lost if self.sleep_at_suspend.load(Relaxed) => return Ok(()), + pm::SleepGate::Lost => { + dev_err!( + self.dev, + "deep sleep refused: the enclave did not come back from a sleep earlier this boot\n" + ); + return Err(EBUSY); + } + pm::SleepGate::Open => {} + _ => return Err(EBUSY), + } + // One that didn't come up still runs: a commit over it resets the SoC. + if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { + dev_err!( + self.dev, + "deep sleep refused: the enclave did not finish coming up, so it can't be put to sleep\n" + ); + return Err(EBUSY); + } + if !self.mailbox_drains() { + dev_err!( + self.dev, + "deep sleep refused: the enclave isn't taking messages (its mailbox stays full)\n" + ); + return Err(EBUSY); + } + + self.set_sleep_gate(pm::SleepGate::Draining); + let idle = control::wait_idle( + proto::CONTROL_TIMEOUT_MS, + 10, + || self.control.lock().requests_in_flight(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_err!(self.dev, "sleep refused: a request is still waiting for its reply\n"); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(EBUSY); + } + + if let Err(e) = pm::dart_prepare(&self.dev) { + dev_err!(self.dev, "sleep refused: its DART could not be saved ({:?})\n", e); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(e); + } + + self.sleep_at_suspend.store(true, Relaxed); + match self.sleep_request() { + Ok(()) => { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.set_recv_enabled(false); + } + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Told); + dev_dbg!(self.dev, "the enclave answered the sleep request\n"); + Ok(()) + } + Err(e) => { + dev_err!(self.dev, "sleep refused: no answer to the sleep request ({:?})\n", e); + self.sleep_unanswered.store(true, Relaxed); + // A request that timed out may still have been taken. + self.set_sleep_gate(pm::SleepGate::Told); + self.sleep_exit(); + Err(e) + } + } + } + + /// Asks the enclave to go to sleep (told at suspend). The request is sent past the gate. + fn sleep_request(&self) -> Result<()> { + let op = proto::op_sleep(); + let tag = proto::TAG_SLEEP; + let idx = self.control.lock().alloc_sleep()?; + let msg = proto::encode_control(&op, tag); + + let sent = match self.mbox.lock().as_ref() { + Some(mbox) => Self::ring(mbox, msg), + None => Err(ENODEV), + }; + if let Err(e) = sent { + self.control.lock().abandon(idx); + return Err(e); + } + match self.control_wait(&op, idx, tag)? { + Some(_) => Ok(()), + None => Err(ETIMEDOUT), + } + } + + /// The wake of an enclave told at suspend: a real wake starts it, a refused sleep finds + /// it gated (it stays asleep until the next real sleep) or asks whether it still answers. + fn sleep_exit_at_suspend(&self) { + let gate = self.sleep_gate(); + if gate == pm::SleepGate::Draining { + self.set_sleep_gate(pm::SleepGate::Open); + return; + } + if !gate.needs_wake() { + return; + } + // SAFETY: only reads a flag. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if !real && gate == pm::SleepGate::Lost { + return; + } + let answered = if real { + self.sleep_kick() + } else if pm::wait_gated(&self.dev) { + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + dev_warn!( + self.dev, + "the enclave stays asleep until the next real sleep or a reboot\n" + ); + return; + } else { + dev_warn!(self.dev, "the enclave never gated its power domain\n"); + self.sleep_probe() + }; + match answered { + Ok(()) => { + self.sleep_at_suspend.store(false, Relaxed); + self.set_sleep_gate(pm::SleepGate::Open); + // One line per sleep it was told of: nothing else says it came back. + dev_info!(self.dev, "the enclave is back from its sleep request\n"); + } + Err(e) => { + dev_err!(self.dev, "the enclave did not come back ({:?}); it stays unused\n", e); + self.sleep_unanswered.store(true, Relaxed); + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + } + } + } + + /// Whether an enclave that was told and never gated still answers. + fn sleep_probe(&self) -> Result<()> { + pm::dart_restore(&self.dev); + self.sleep_release_rx(); + let op = proto::op_nop(0); + let (idx, tag) = self.control.lock().alloc()?; + let msg = proto::encode_control(&op, tag); + let sent = match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, msg) + } + None => Err(ENODEV), + }; + if let Err(e) = sent { + self.control.lock().abandon(idx); + return Err(e); + } + match self.control_wait(&op, idx, tag)? { + Some(_) => Ok(()), + None => Err(ETIMEDOUT), + } + } + /// Stage without sending; the platform's commit owns SLEEP with IRQs off. fn sleep_leave_for_commit(&self) -> Result<()> { let request = proto::encode_control(&proto::op_sleep(), proto::TAG_SLEEP); @@ -1519,6 +1718,20 @@ impl SepData { Ok(()) } + /// Whether the enclave takes messages: its receive FIFO drains within `MAILBOX_DRAIN_MS`. A + /// FIFO that stays full means it stopped reading, and a sleep request would only time out. + fn mailbox_drains(&self) -> bool { + for _ in 0..MAILBOX_DRAIN_MS / 5 { + match self.mbox.lock().as_ref() { + Some(mbox) if mbox.send_ready() => return true, + Some(_) => {} + None => return false, + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(5)); + } + false + } + /// Holds off the receive interrupt, whose handler reads the mailbox. fn sleep_hold_rx(&self) { if !self.sleep_rx_held.xchg(true, Relaxed) { @@ -1539,6 +1752,10 @@ impl SepData { /// A refused precommit attempt was never sent; reopen normal admission. /// A sent request can be restarted only after actual physical sleep/wake. pub(crate) fn sleep_exit(&self) { + if self.sleep_at_suspend.load(Relaxed) { + self.sleep_exit_at_suspend(); + return; + } // Staged only: an unsent refusal releases RX without a restart. match pm::commit_finish(&self.dev) { Some(true) => self.set_sleep_gate(pm::SleepGate::Told), diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index aa5112b1d5577..5c22bf70c678a 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -33,6 +33,8 @@ int sep_pm_keep_domains(void *dev); int sep_pm_wait_gated(void *dev, unsigned int timeout_ms); int sep_pm_clear_dev_disable(void *dev); void sep_pm_dart_restore(void *dev); +int sep_pm_dart_prepare(void *dev); +bool sep_pm_platform_commit(void); int sep_pm_commit_prepare(void *dev, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask); int sep_pm_commit_finish(void *dev); extern const struct dev_pm_ops sep_pm_ops; diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 6e97b81b9e05d..6bfc28d66f8f5 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -1059,6 +1059,44 @@ static void apple_sleep_coprocs_stop(struct apple_sleep *s, bool first) dsb(sy); } +/* + * For a SoC whose sleep has its own driver at EL2 (T6041): both passes of the stop above, + * the coprocessors registered first, then the rest. A negative return of a last stop ends it. + */ +int apple_sleep_coprocs_stop_all(void) +{ + int pass, i, ret; + + for (pass = 1; pass >= 0; pass--) { + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != pass || c->stopped) + continue; + if (c->last_stop) { + ret = c->last_stop(c->data); + if (ret < 0) + return ret; + } + apple_sleep_coproc_run_off(c); + } + } + dsb(sy); + return 0; +} + +/* For T6041: whether the coprocessor behind the storage PCIe link (ANS) was stopped. */ +bool apple_sleep_coprocs_ans_stopped(void) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].pcie_ans && + apple_sleep_coprocs[i].stopped) + return true; + return false; +} + /* MPIDR's cluster is affinity 2, its core affinity 0, as 0.0.n and 1.1.n on T8140. */ static unsigned int sleep_cpu_cluster(u64 mpidr) { diff --git a/drivers/soc/apple/t6041-apcie-tunables.h b/drivers/soc/apple/t6041-apcie-tunables.h new file mode 100644 index 0000000000000..ba3550109b68c --- /dev/null +++ b/drivers/soc/apple/t6041-apcie-tunables.h @@ -0,0 +1,336 @@ +/* ANS storage-PCIe tunables of the J616C, from its ADT /arm-io/apcie0 (apcie,t6040): {offset, mask, value}. Generated (NOTES 10 Nov). */ +struct t6041_tun { u32 off, mask, val; }; +static const struct t6041_tun t6041_tun_phy[] = { + { 0x8000, 0x4000000, 0x0 }, + { 0x20000, 0x10000000, 0x0 }, + { 0x24000, 0x10000000, 0x0 }, + { 0x28000, 0x10000000, 0x0 }, + { 0x2c000, 0x10000000, 0x0 }, +}; +static const struct t6041_tun t6041_tun_axi2af[] = { + { 0x0, 0x800000ff, 0x80000022 }, + { 0x4, 0x80ff00ff, 0x80220003 }, + { 0x8, 0x80ff00ff, 0x80220003 }, + { 0xc, 0x80ff00ff, 0x80220003 }, + { 0x10, 0x80ff00ff, 0x80220003 }, + { 0x14, 0x80ff00ff, 0x80220003 }, + { 0x18, 0x80ff00ff, 0x80220003 }, + { 0x1c, 0x800000ff, 0x8000001c }, + { 0x20, 0x800000ff, 0x80000088 }, + { 0x24, 0x80ff00ff, 0x8088000c }, + { 0x28, 0x80ff00ff, 0x8088000c }, + { 0x2c, 0x80ff00ff, 0x80200008 }, + { 0x30, 0x80ff00ff, 0x8088000c }, + { 0x34, 0x80ff00ff, 0x8088000c }, + { 0x38, 0x800000ff, 0x8000001c }, + { 0x3c, 0x800000ff, 0x8000001c }, + { 0x40, 0x800000ff, 0x8000001c }, + { 0x100, 0xff, 0xff }, + { 0x108, 0xffff0006, 0x100000 }, + { 0x304, 0x3f3f0000, 0x8080000 }, + { 0x308, 0xfffff, 0x1681 }, + { 0x30c, 0xfffff, 0x1680 }, + { 0x310, 0x70000fff, 0x1e0 }, + { 0x314, 0xf0000fff, 0x800001e0 }, + { 0x318, 0x70000fff, 0x5a0 }, + { 0x31c, 0xf0000fff, 0x800005a0 }, + { 0x320, 0x70000fff, 0x5a0 }, + { 0x324, 0xf0000fff, 0x800005a0 }, + { 0x328, 0x70000fff, 0x5a0 }, + { 0x32c, 0x70000fff, 0x5a0 }, + { 0x330, 0x70000fff, 0x5a0 }, + { 0x334, 0x70000fff, 0x5a0 }, + { 0x338, 0x70000fff, 0x5a0 }, + { 0x33c, 0x70000fff, 0x5a0 }, + { 0x340, 0x70000fff, 0x5a0 }, + { 0x348, 0xffffffff, 0xffff }, + { 0x34c, 0xffffffff, 0xffff }, + { 0x408, 0xffff0020, 0x1f0020 }, + { 0x40c, 0xfffff, 0x1681 }, + { 0x410, 0xfffff, 0x1680 }, + { 0x414, 0x70000fff, 0x1e0 }, + { 0x418, 0x70000fff, 0x1e0 }, + { 0x41c, 0x70000fff, 0x5a0 }, + { 0x420, 0x70000fff, 0x5a0 }, + { 0x424, 0x70000fff, 0x5a0 }, + { 0x428, 0x70000fff, 0x5a0 }, + { 0x42c, 0x70000fff, 0x5a0 }, + { 0x430, 0x70000fff, 0x5a0 }, + { 0x434, 0x70000fff, 0x5a0 }, + { 0x438, 0x70000fff, 0x5a0 }, + { 0x43c, 0x70000fff, 0x5a0 }, + { 0x440, 0x70000fff, 0x5a0 }, + { 0x444, 0x70000fff, 0x5a0 }, + { 0x948, 0x7ff07ff, 0x200000 }, + { 0xa48, 0x7ff07ff, 0xa0100 }, + { 0xb48, 0x7ff07ff, 0x800080 }, + { 0xe00, 0xffffffff, 0x7fffffff }, + { 0x4008, 0x1, 0x1 }, + { 0x400c, 0x1, 0x1 }, + { 0x4014, 0x1, 0x0 }, + { 0x4018, 0x1, 0x0 }, + { 0x401c, 0x1, 0x1 }, + { 0x10f8, 0x1, 0x1 }, + { 0x11f8, 0x1, 0x1 }, + { 0x12f8, 0x1, 0x1 }, + { 0x13f8, 0x1, 0x1 }, + { 0x14f8, 0x1, 0x1 }, + { 0x10fc, 0x1, 0x1 }, + { 0x11fc, 0x1, 0x1 }, + { 0x12fc, 0x1, 0x1 }, + { 0x13fc, 0x1, 0x1 }, + { 0x14fc, 0x1, 0x1 }, + { 0x20028, 0xfe2f, 0x0 }, + { 0x2002c, 0x7, 0x0 }, + { 0x20030, 0x7f, 0x0 }, + { 0x20034, 0xff, 0x0 }, + { 0x20058, 0x10b, 0x0 }, +}; +static const struct t6041_tun t6041_tun_phy_ip_lanes[] = { + { 0xa000, 0xf, 0xa }, + { 0xa008, 0xf, 0xa }, + { 0xa00c, 0xf, 0xa }, + { 0xa010, 0xf, 0xa }, + { 0xa014, 0xf, 0xa }, + { 0xa038, 0xffff, 0x271 }, + { 0xa040, 0x800, 0x0 }, + { 0xa04c, 0x3fff, 0xbff }, + { 0xa050, 0x1, 0x1 }, + { 0xa068, 0x7, 0x1 }, + { 0xa070, 0xfff038e0, 0x55b00820 }, + { 0xa084, 0xffffffff, 0x3e8f4240 }, + { 0xa088, 0xfff, 0xfff }, + { 0xa09c, 0xfc00000, 0x5400000 }, + { 0xa0b8, 0xff03e000, 0xaa03e000 }, + { 0xa0bc, 0x60fff, 0x40faa }, + { 0xa0c0, 0x3ff80000, 0x14500000 }, + { 0xa0c4, 0xffff, 0x7777 }, + { 0xa0e0, 0x18000000, 0x8000000 }, + { 0xa0e8, 0x1ff000, 0x2000 }, + { 0xa10c, 0x3fffffff, 0xf4240 }, + { 0xa11c, 0x40000, 0x40000 }, + { 0xa178, 0x3f00000, 0x300000 }, + { 0xa1b4, 0x3ff00, 0x1500 }, + { 0xa1bc, 0x3ff00, 0x1500 }, + { 0xa1c4, 0x1, 0x1 }, + { 0xa1cc, 0x1, 0x1 }, + { 0xa1e8, 0xf, 0x0 }, + { 0xa1b0, 0xffffffff, 0x98924 }, + { 0xa1b8, 0xffffffff, 0x98924 }, + { 0x12000, 0xf, 0xa }, + { 0x12008, 0xf, 0xa }, + { 0x1200c, 0xf, 0xa }, + { 0x12010, 0xf, 0xa }, + { 0x12014, 0xf, 0xa }, + { 0x12038, 0xffff, 0x271 }, + { 0x12040, 0x800, 0x0 }, + { 0x1204c, 0x3fff, 0xbff }, + { 0x12050, 0x1, 0x1 }, + { 0x12068, 0x7, 0x1 }, + { 0x12070, 0xfff038e0, 0x55b00820 }, + { 0x12084, 0xffffffff, 0x3e8f4240 }, + { 0x12088, 0xfff, 0xfff }, + { 0x1209c, 0xfc00000, 0x5400000 }, + { 0x120b8, 0xff03e000, 0xaa03e000 }, + { 0x120bc, 0x60fff, 0x40faa }, + { 0x120c0, 0x3ff80000, 0x14500000 }, + { 0x120c4, 0xffff, 0x7777 }, + { 0x120e0, 0x18000000, 0x8000000 }, + { 0x120e8, 0x1ff000, 0x2000 }, + { 0x1210c, 0x3fffffff, 0xf4240 }, + { 0x1211c, 0x40000, 0x40000 }, + { 0x12178, 0x3f00000, 0x300000 }, + { 0x121b4, 0x3ff00, 0x1500 }, + { 0x121bc, 0x3ff00, 0x1500 }, + { 0x121c4, 0x1, 0x1 }, + { 0x121cc, 0x1, 0x1 }, + { 0x121e8, 0xf, 0x0 }, + { 0x121b0, 0xffffffff, 0x98924 }, + { 0x121b8, 0xffffffff, 0x98924 }, + { 0x1a000, 0xf, 0xa }, + { 0x1a008, 0xf, 0xa }, + { 0x1a00c, 0xf, 0xa }, + { 0x1a010, 0xf, 0xa }, + { 0x1a014, 0xf, 0xa }, + { 0x1a038, 0xffff, 0x271 }, + { 0x1a040, 0x800, 0x0 }, + { 0x1a04c, 0x3fff, 0xbff }, + { 0x1a050, 0x1, 0x1 }, + { 0x1a068, 0x7, 0x1 }, + { 0x1a070, 0xfff038e0, 0x55b00820 }, + { 0x1a084, 0xffffffff, 0x3e8f4240 }, + { 0x1a088, 0xfff, 0xfff }, + { 0x1a09c, 0xfc00000, 0x5400000 }, + { 0x1a0b8, 0xff03e000, 0xaa03e000 }, + { 0x1a0bc, 0x60fff, 0x40faa }, + { 0x1a0c0, 0x3ff80000, 0x14500000 }, + { 0x1a0c4, 0xffff, 0x7777 }, + { 0x1a0e0, 0x18000000, 0x8000000 }, + { 0x1a0e8, 0x1ff000, 0x2000 }, + { 0x1a10c, 0x3fffffff, 0xf4240 }, + { 0x1a11c, 0x40000, 0x40000 }, + { 0x1a178, 0x3f00000, 0x300000 }, + { 0x1a1b4, 0x3ff00, 0x1500 }, + { 0x1a1bc, 0x3ff00, 0x1500 }, + { 0x1a1c4, 0x1, 0x1 }, + { 0x1a1cc, 0x1, 0x1 }, + { 0x1a1e8, 0xf, 0x0 }, + { 0x1a1b0, 0xffffffff, 0x98924 }, + { 0x1a1b8, 0xffffffff, 0x98924 }, + { 0x22000, 0xf, 0xa }, + { 0x22008, 0xf, 0xa }, + { 0x2200c, 0xf, 0xa }, + { 0x22010, 0xf, 0xa }, + { 0x22014, 0xf, 0xa }, + { 0x22038, 0xffff, 0x271 }, + { 0x22040, 0x800, 0x0 }, + { 0x2204c, 0x3fff, 0xbff }, + { 0x22050, 0x1, 0x1 }, + { 0x22068, 0x7, 0x1 }, + { 0x22070, 0xfff038e0, 0x55b00820 }, + { 0x22084, 0xffffffff, 0x3e8f4240 }, + { 0x22088, 0xfff, 0xfff }, + { 0x2209c, 0xfc00000, 0x5400000 }, + { 0x220b8, 0xff03e000, 0xaa03e000 }, + { 0x220bc, 0x60fff, 0x40faa }, + { 0x220c0, 0x3ff80000, 0x14500000 }, + { 0x220c4, 0xffff, 0x7777 }, + { 0x220e0, 0x18000000, 0x8000000 }, + { 0x220e8, 0x1ff000, 0x2000 }, + { 0x2210c, 0x3fffffff, 0xf4240 }, + { 0x2211c, 0x40000, 0x40000 }, + { 0x22178, 0x3f00000, 0x300000 }, + { 0x221b4, 0x3ff00, 0x1500 }, + { 0x221bc, 0x3ff00, 0x1500 }, + { 0x221c4, 0x1, 0x1 }, + { 0x221cc, 0x1, 0x1 }, + { 0x221e8, 0xf, 0x0 }, + { 0x221b0, 0xffffffff, 0x98924 }, + { 0x221b8, 0xffffffff, 0x98924 }, + { 0xb02c, 0x100000, 0x100000 }, + { 0xb058, 0x18, 0x18 }, + { 0xb070, 0x30, 0x20 }, + { 0xb078, 0x3c000000, 0x2c000000 }, + { 0x1302c, 0x100000, 0x100000 }, + { 0x13058, 0x18, 0x18 }, + { 0x13070, 0x30, 0x20 }, + { 0x13078, 0x3c000000, 0x2c000000 }, + { 0x1b02c, 0x100000, 0x100000 }, + { 0x1b058, 0x18, 0x18 }, + { 0x1b070, 0x30, 0x20 }, + { 0x1b078, 0x3c000000, 0x2c000000 }, + { 0x2302c, 0x100000, 0x100000 }, + { 0x23058, 0x18, 0x18 }, + { 0x23070, 0x30, 0x20 }, + { 0x23078, 0x3c000000, 0x2c000000 }, + { 0x9000, 0x40000000, 0x0 }, + { 0x9008, 0x10, 0x0 }, + { 0x903c, 0x1800000, 0x1800000 }, + { 0x90d8, 0x3fff, 0x68d }, + { 0x90dc, 0x3fff, 0x408 }, + { 0x91e0, 0x8, 0x8 }, + { 0x9280, 0xf00000, 0xc00000 }, + { 0x9280, 0xff000f, 0xc80000 }, + { 0x9288, 0x3, 0x3 }, + { 0x928c, 0x3, 0x3 }, + { 0x11000, 0x40000000, 0x0 }, + { 0x11008, 0x10, 0x0 }, + { 0x1103c, 0x1800000, 0x1800000 }, + { 0x110d8, 0x3fff, 0x68d }, + { 0x110dc, 0x3fff, 0x408 }, + { 0x111e0, 0x8, 0x8 }, + { 0x11280, 0xf00000, 0xc00000 }, + { 0x11280, 0xff000f, 0xc80006 }, + { 0x11288, 0x3, 0x3 }, + { 0x1128c, 0x3, 0x3 }, + { 0x19000, 0x40000000, 0x0 }, + { 0x19008, 0x10, 0x0 }, + { 0x1903c, 0x1800000, 0x1800000 }, + { 0x190d8, 0x3fff, 0x68d }, + { 0x190dc, 0x3fff, 0x408 }, + { 0x191e0, 0x8, 0x8 }, + { 0x19280, 0xf00000, 0xc00000 }, + { 0x19288, 0x3, 0x3 }, + { 0x21000, 0x40000000, 0x0 }, + { 0x21008, 0x10, 0x0 }, + { 0x2103c, 0x1800000, 0x1800000 }, + { 0x210d8, 0x3fff, 0x68d }, + { 0x210dc, 0x3fff, 0x408 }, + { 0x211e0, 0x8, 0x8 }, + { 0x21280, 0xf00000, 0xc00000 }, + { 0x21288, 0x3, 0x3 }, + { 0xd008, 0xc, 0xc }, + { 0xd014, 0x7efc, 0x408 }, + { 0x15008, 0xc, 0xc }, + { 0x15014, 0x7efc, 0x408 }, + { 0x1d008, 0xc, 0xc }, + { 0x1d014, 0x7efc, 0x408 }, + { 0x25008, 0xc, 0xc }, + { 0x25014, 0x7efc, 0x408 }, + { 0xc01c, 0xe0, 0xa0 }, + { 0xc080, 0x60000000, 0x40000000 }, + { 0xc08c, 0x3f0000, 0x3e0000 }, + { 0xc0a0, 0x1f80000, 0x380000 }, + { 0xc0ec, 0x1, 0x0 }, + { 0x1401c, 0xe0, 0xa0 }, + { 0x14080, 0x60000000, 0x40000000 }, + { 0x1408c, 0x3f0000, 0x3e0000 }, + { 0x140a0, 0x1f80000, 0x380000 }, + { 0x140ec, 0x1, 0x0 }, + { 0x1c01c, 0xe0, 0xa0 }, + { 0x1c080, 0x60000000, 0x40000000 }, + { 0x1c08c, 0x3f0000, 0x3e0000 }, + { 0x1c0a0, 0x1f80000, 0x380000 }, + { 0x1c0ec, 0x1, 0x0 }, + { 0x2401c, 0xe0, 0xa0 }, + { 0x24080, 0x60000000, 0x40000000 }, + { 0x2408c, 0x3f0000, 0x3e0000 }, + { 0x240a0, 0x1f80000, 0x380000 }, + { 0x240ec, 0x1, 0x0 }, + { 0xe404, 0x40000000, 0x40000000 }, + { 0x16404, 0x40000000, 0x40000000 }, + { 0x1e404, 0x40000000, 0x40000000 }, + { 0x26404, 0x40000000, 0x40000000 }, +}; +static const struct t6041_tun t6041_tun_phy_ip_pll[] = { + { 0x90, 0x1, 0x1 }, + { 0xa4, 0x1, 0x1 }, + { 0x1000, 0x3f0000, 0x230000 }, + { 0x102c, 0x3fffffc, 0x454444 }, + { 0x1204, 0x7c, 0x54 }, + { 0x501c, 0x4, 0x0 }, + { 0x5080, 0x3c00000, 0x1400000 }, + { 0x5088, 0xf0000, 0x60000 }, + { 0x508c, 0x3c00000, 0x1400000 }, + { 0x5094, 0xf0000, 0x60000 }, + { 0x50a0, 0xf0000, 0x50000 }, + { 0x50ac, 0xf0000, 0x50000 }, + { 0x50b8, 0xf0000, 0x50000 }, + { 0x5200, 0xa, 0xa }, + { 0x5224, 0xc00, 0x800 }, + { 0x5230, 0x1f, 0x8 }, + { 0x5240, 0x7d8007f, 0x4000049 }, + { 0x52b0, 0x8700000, 0x300000 }, + { 0x52e8, 0x60000, 0x40000 }, + { 0x5300, 0xffffff, 0x78078 }, + { 0x5320, 0x6, 0x2 }, + { 0x608c, 0x3c3c000, 0x1818000 }, + { 0x6224, 0xc00, 0x800 }, + { 0x6228, 0xf00, 0xb00 }, + { 0x6230, 0x1f, 0xf }, + { 0x624c, 0x7f, 0x1e }, + { 0x62b0, 0x700000, 0x300000 }, + { 0x62e8, 0x60000, 0x40000 }, +}; +static const struct t6041_tun t6041_tun_cio3pll[] = { + { 0x0, 0xa0b, 0xa01 }, + { 0x24, 0xc00, 0x800 }, + { 0x28, 0xf00, 0xb00 }, + { 0x38, 0x20, 0x0 }, + { 0x4c, 0xff, 0x96 }, + { 0xe8, 0xe0000, 0x20000 }, + { 0x100, 0xffffff, 0xb40b4 }, +}; +static const struct t6041_tun t6041_tun_clkgen[] = { + { 0x0, 0x3e0, 0x220 }, +}; diff --git a/drivers/soc/apple/t6041-macos-regs.h b/drivers/soc/apple/t6041-macos-regs.h new file mode 100644 index 0000000000000..be04f86a7ea3b --- /dev/null +++ b/drivers/soc/apple/t6041-macos-regs.h @@ -0,0 +1,350 @@ +/* Generated from images/macos-precommit.json: macOS 26.6.2 at its S2R commit (T6041). ps = a named PS word. */ +static const struct { u64 pa; u32 val; u8 ps; } t6041_macos_regs[] = { + { 0x288e80000, 0x1f0002ff, 1 }, /* gpx */ + { 0x288e80100, 0xf000300, 1 }, /* afr */ + { 0x288e80110, 0xf000300, 1 }, /* gfx */ + { 0x50020c000, 0x4333, 0 }, /* soc_perf */ + { 0x50020c400, 0x7666, 0 }, /* dcs_perf */ + { 0x502280000, 0x4000200, 1 }, /* ispsens0 */ + { 0x502280100, 0x200, 1 }, /* apcie_gp */ + { 0x502280108, 0x200, 1 }, /* apcie_st0 */ + { 0x502280110, 0x200, 1 }, /* apcie_st1 */ + { 0x502280118, 0x1400034f, 1 }, /* fab3_soc */ + { 0x502280120, 0xf0002ff, 1 }, /* sio */ + { 0x502280128, 0x1000030f, 1 }, /* fab4_soc */ + { 0x502280130, 0xf0002ff, 1 }, /* sio_cpu */ + { 0x502280138, 0x200, 1 }, /* fpwm0 */ + { 0x502280140, 0x200, 1 }, /* fpwm1 */ + { 0x502280148, 0x200, 1 }, /* fpwm2 */ + { 0x502280150, 0x200, 1 }, /* i2c0 */ + { 0x502280158, 0x200, 1 }, /* i2c1 */ + { 0x502280160, 0x200, 1 }, /* i2c2 */ + { 0x502280168, 0x200, 1 }, /* i2c3 */ + { 0x502280170, 0x200, 1 }, /* i2c4 */ + { 0x502280178, 0x200, 1 }, /* i2c5 */ + { 0x502280180, 0x200, 1 }, /* i2c6 */ + { 0x502280188, 0x200, 1 }, /* i2c7 */ + { 0x502280190, 0x200, 1 }, /* i2c8 */ + { 0x502280198, 0xff, 1 }, /* spi_p */ + { 0x5022801a0, 0xff, 1 }, /* uart_p */ + { 0x5022801a8, 0x2ff, 1 }, /* dpa_p */ + { 0x5022801b0, 0xff, 1 }, /* aes */ + { 0x5022801b8, 0x200, 1 }, /* spi0 */ + { 0x5022801c0, 0x200, 1 }, /* spi1 */ + { 0x5022801c8, 0x200, 1 }, /* spi2 */ + { 0x5022801d0, 0x200, 1 }, /* spi3 */ + { 0x5022801d8, 0x200, 1 }, /* spi4 */ + { 0x5022801e0, 0x200, 1 }, /* spi5 */ + { 0x5022801e8, 0x200, 1 }, /* qspi */ + { 0x5022801f0, 0xff, 1 }, /* uart_n */ + { 0x5022801f8, 0xff, 1 }, /* uart0 */ + { 0x502280200, 0xff, 1 }, /* uart1 */ + { 0x502280208, 0xff, 1 }, /* uart2 */ + { 0x502280210, 0xff, 1 }, /* uart3 */ + { 0x502280218, 0x2ff, 1 }, /* uart4 */ + { 0x502280220, 0x2ff, 1 }, /* uart5 */ + { 0x502280228, 0x2ff, 1 }, /* uart6 */ + { 0x502280230, 0x200, 1 }, /* dpa0 */ + { 0x502280238, 0x200, 1 }, /* dpa1 */ + { 0x502280240, 0x200, 1 }, /* dpa2 */ + { 0x502280248, 0x200, 1 }, /* dpa3 */ + { 0x502280250, 0x200, 1 }, /* dpa4 */ + { 0x502280258, 0x300, 1 }, /* disp_sys */ + { 0x502280260, 0x1000030f, 1 }, /* amcc4 */ + { 0x502280268, 0x1000030f, 1 }, /* amcc5 */ + { 0x502280270, 0x1000030f, 1 }, /* amcc6 */ + { 0x502280278, 0x1000030f, 1 }, /* amcc7 */ + { 0x502280280, 0x1000030f, 1 }, /* amcc12 */ + { 0x502280288, 0x1000030f, 1 }, /* amcc13 */ + { 0x502280290, 0x1000030f, 1 }, /* amcc14 */ + { 0x502280298, 0x1000030f, 1 }, /* amcc15 */ + { 0x5022802a0, 0x1000030f, 1 }, /* amcc20 */ + { 0x5022802a8, 0x1000030f, 1 }, /* amcc21 */ + { 0x5022802b0, 0x1000030f, 1 }, /* amcc22 */ + { 0x5022802b8, 0x1000030f, 1 }, /* amcc23 */ + { 0x5022802e0, 0x1400034f, 1 }, /* dcs_04 */ + { 0x5022802e8, 0x1f00034f, 1 }, /* dcs_05 */ + { 0x5022802f0, 0x1400034f, 1 }, /* dcs_06 */ + { 0x5022802f8, 0x1400034f, 1 }, /* dcs_07 */ + { 0x502280300, 0x1400034f, 1 }, /* dcs_12 */ + { 0x502280308, 0x1400034f, 1 }, /* dcs_13 */ + { 0x502280310, 0x1400034f, 1 }, /* dcs_14 */ + { 0x502280318, 0x1400034f, 1 }, /* dcs_15 */ + { 0x502280320, 0x1400034f, 1 }, /* dcs_20 */ + { 0x502280328, 0x1400034f, 1 }, /* dcs_21 */ + { 0x502280330, 0x1400034f, 1 }, /* dcs_22 */ + { 0x502280338, 0x1000030f, 1 }, /* dcs_23 */ + { 0x502280368, 0x300, 1 }, /* disp_fe */ + { 0x502280370, 0xf000300, 1 }, /* ans */ + { 0x502280378, 0x300, 1 }, /* isp_sys */ + { 0x502280380, 0xf000300, 1 }, /* apcie_sys_st0 */ + { 0x502280388, 0xf000300, 1 }, /* apcie_sys_st1 */ + { 0x502280390, 0xf000300, 1 }, /* apcie_sys_gp */ + { 0x502280398, 0xf000200, 1 }, /* apcie_phy_sw */ + { 0x5022803a0, 0xf000300, 1 }, /* jpg */ + { 0x5022803a8, 0x300, 1 }, /* prores */ + { 0x5022803b0, 0xf000300, 1 }, /* msr1 */ + { 0x5022803b8, 0xf000300, 1 }, /* msr1_ase_core */ + { 0x5022803c0, 0x300, 1 }, /* ane_sys */ + { 0x502280c00, 0x1000070f, 1 }, /* sep */ + { 0x50228c000, 0x300, 1 }, /* ane_mpm */ + { 0x50228c008, 0xf000300, 1 }, /* ane_cpu */ + { 0x50228c010, 0x300, 1 }, /* ane_td */ + { 0x50228c018, 0x300, 1 }, /* ane_base */ + { 0x502290000, 0xf000300, 1 }, /* disp_cpu */ + { 0x502800000, 0x1f0, 1 }, /* ecpu_0 */ + { 0x502800008, 0x1f0, 1 }, /* ecpu_1 */ + { 0x502800010, 0x1f0, 1 }, /* ecpu_2 */ + { 0x502800018, 0x1f0, 1 }, /* ecpu_3 */ + { 0x502800020, 0x1f0, 1 }, /* pcpu0_0 */ + { 0x502800028, 0x1f0, 1 }, /* pcpu0_1 */ + { 0x502800030, 0x1f0, 1 }, /* pcpu0_2 */ + { 0x502800038, 0x1f0, 1 }, /* pcpu0_3 */ + { 0x502800040, 0x1f0, 1 }, /* pcpu0_4 */ + { 0x502800048, 0x100, 1 }, /* pcpu0_5 */ + { 0x502800050, 0x1f0, 1 }, /* pcpu1_0 */ + { 0x502800058, 0x1f0, 1 }, /* pcpu1_1 */ + { 0x502800060, 0x1f0, 1 }, /* pcpu1_2 */ + { 0x502800068, 0x1f0, 1 }, /* pcpu1_3 */ + { 0x502800070, 0x1f0, 1 }, /* pcpu1_4 */ + { 0x502800078, 0x100, 1 }, /* pcpu1_5 */ + { 0x502800080, 0x1f0, 1 }, /* ecpm */ + { 0x502800088, 0x1f0, 1 }, /* pcpm0 */ + { 0x502800090, 0x1f0, 1 }, /* pcpm1 */ + { 0x502800100, 0x1f0002ff, 1 }, /* pms */ + { 0x502800108, 0x2ff, 1 }, /* dwi */ + { 0x502800110, 0x1f0000ff, 1 }, /* afi */ + { 0x502800118, 0xff, 1 }, /* aic */ + { 0x502800120, 0x1400024f, 1 }, /* pmc */ + { 0x502800128, 0xff, 1 }, /* pms_c1ppt */ + { 0x502800130, 0xff, 1 }, /* soc_dpe */ + { 0x502800138, 0x200, 1 }, /* pmgr_soc_ocla */ + { 0x502800140, 0x200, 1 }, /* pms_fpwm0 */ + { 0x502800148, 0x200, 1 }, /* pms_fpwm1 */ + { 0x502800150, 0x200, 1 }, /* pms_fpwm2 */ + { 0x502800158, 0x200, 1 }, /* pms_fpwm3 */ + { 0x502800160, 0x200, 1 }, /* pms_fpwm4 */ + { 0x502800168, 0x1000030f, 1 }, /* fab0_soc */ + { 0x502800170, 0xf000300, 1 }, /* avd_sys */ + { 0x502800178, 0x1f0000ff, 1 }, /* afc */ + { 0x502800180, 0x1000030f, 1 }, /* amcc0 */ + { 0x502800188, 0x1400034f, 1 }, /* dcs_00 */ + { 0x502800190, 0x300, 1 }, /* rom */ + { 0x502800198, 0x1000030f, 1 }, /* amcc1 */ + { 0x5028001a0, 0x1400034f, 1 }, /* dcs_01 */ + { 0x5028001a8, 0x1000030f, 1 }, /* amcc2 */ + { 0x5028001b0, 0x1400034f, 1 }, /* dcs_02 */ + { 0x5028001b8, 0x1000030f, 1 }, /* amcc3 */ + { 0x5028001c0, 0x1400034f, 1 }, /* dcs_03 */ + { 0x5028001c8, 0x1000030f, 1 }, /* amcc8 */ + { 0x5028001d0, 0x1400034f, 1 }, /* dcs_08 */ + { 0x5028001d8, 0x1000030f, 1 }, /* amcc9 */ + { 0x5028001e0, 0x1400034f, 1 }, /* dcs_09 */ + { 0x5028001e8, 0x1000030f, 1 }, /* amcc10 */ + { 0x5028001f0, 0x1400034f, 1 }, /* dcs_10 */ + { 0x5028001f8, 0x1000030f, 1 }, /* amcc11 */ + { 0x502800200, 0x1f0003ff, 1 }, /* dcs_11 */ + { 0x502800208, 0x1000030f, 1 }, /* amcc16 */ + { 0x502800210, 0x1400034f, 1 }, /* dcs_16 */ + { 0x502800218, 0x1000030f, 1 }, /* amcc17 */ + { 0x502800220, 0x1400034f, 1 }, /* dcs_17 */ + { 0x502800228, 0x1000030f, 1 }, /* amcc18 */ + { 0x502800230, 0x1400034f, 1 }, /* dcs_18 */ + { 0x502800238, 0x1000030f, 1 }, /* amcc19 */ + { 0x502800240, 0x1000030f, 1 }, /* dcs_19 */ + { 0x502800278, 0xf000300, 1 }, /* venc0_sys */ + { 0x502800288, 0x1f0002ff, 1 }, /* fab_gw_0 */ + { 0x502800290, 0x1000020f, 1 }, /* fab_gw_1 */ + { 0x502800298, 0x1000020f, 1 }, /* fab_afr */ + { 0x5028002a0, 0xf000300, 1 }, /* venc1_sys */ + { 0x5028002a8, 0x300, 1 }, /* venc1_dma */ + { 0x5028002b0, 0x300, 1 }, /* venc1_pipe4 */ + { 0x5028002b8, 0x300, 1 }, /* venc1_pipe5 */ + { 0x5028002c0, 0x300, 1 }, /* venc1_me0 */ + { 0x5028002c8, 0x300, 1 }, /* venc1_me1 */ + { 0x5028002d0, 0x1000030f, 1 }, /* fab1_soc */ + { 0x5028002d8, 0x300, 1 }, /* dispext3_sys */ + { 0x5028002e0, 0x300, 1 }, /* dispext3_fe */ + { 0x5028002e8, 0xf000300, 1 }, /* dispext3_cpu */ + { 0x502800328, 0x300, 1 }, /* scodec */ + { 0x502800330, 0x300, 1 }, /* scodec_streaming */ + { 0x502800338, 0x1f0003ff, 1 }, /* fab2_soc */ + { 0x502800340, 0x300, 1 }, /* dispext0_sys */ + { 0x502800348, 0x300, 1 }, /* dispext0_fe */ + { 0x502800350, 0xf000300, 1 }, /* dispext0_cpu */ + { 0x502800390, 0x3ff, 1 }, /* atc0_common */ + { 0x502800398, 0x300, 1 }, /* atc0_cio */ + { 0x5028003a0, 0x300, 1 }, /* atc0_pcie */ + { 0x5028003a8, 0x300, 1 }, /* atc0_cio_pcie */ + { 0x5028003b0, 0x300, 1 }, /* atc0_cio_usb */ + { 0x5028003b8, 0x300, 1 }, /* atc1_common */ + { 0x5028003c0, 0x300, 1 }, /* atc1_cio */ + { 0x5028003c8, 0x300, 1 }, /* atc1_pcie */ + { 0x5028003d0, 0x300, 1 }, /* atc1_cio_pcie */ + { 0x5028003d8, 0x300, 1 }, /* atc1_cio_usb */ + { 0x5028003e0, 0x300, 1 }, /* atc2_common */ + { 0x5028003e8, 0x300, 1 }, /* atc2_cio */ + { 0x5028003f0, 0x300, 1 }, /* atc2_pcie */ + { 0x5028003f8, 0x300, 1 }, /* atc2_cio_pcie */ + { 0x502800400, 0x300, 1 }, /* atc2_cio_usb */ + { 0x502800408, 0x300, 1 }, /* atc3_common */ + { 0x502800410, 0x1f0f03ff, 1 }, /* pmp */ + { 0x502800418, 0x1f0003ff, 1 }, /* pms_sram */ + { 0x502800420, 0x300, 1 }, /* atc3_cio */ + { 0x502800428, 0x300, 1 }, /* atc3_pcie */ + { 0x502800430, 0x300, 1 }, /* atc3_cio_pcie */ + { 0x502800438, 0x300, 1 }, /* atc3_cio_usb */ + { 0x502800440, 0x1f0003ff, 1 }, /* fab5_soc */ + { 0x502800448, 0x1f0042ff, 1 }, /* sbr */ + { 0x502800450, 0xff, 1 }, /* gpio */ + { 0x502800458, 0x200, 1 }, /* msg */ + { 0x502800468, 0x300, 1 }, /* ap_tmm */ + { 0x502800470, 0xf000300, 1 }, /* msr0 */ + { 0x502800478, 0xf000300, 1 }, /* msr0_ase_core */ + { 0x502800480, 0x1000030f, 1 }, /* fab6_soc */ + { 0x502800488, 0x1000030f, 1 }, /* fab7_soc */ + { 0x502800490, 0x300, 1 }, /* dispext1_sys */ + { 0x502800498, 0x300, 1 }, /* dispext1_fe */ + { 0x5028004a0, 0xf000300, 1 }, /* dispext1_cpu */ + { 0x5028004e0, 0x300, 1 }, /* dispext2_sys */ + { 0x5028004e8, 0x300, 1 }, /* dispext2_fe */ + { 0x5028004f0, 0xf000300, 1 }, /* dispext2_cpu */ + { 0x502808000, 0x300, 1 }, /* venc0_dma */ + { 0x502808008, 0x300, 1 }, /* venc0_pipe4 */ + { 0x502808010, 0x300, 1 }, /* venc0_pipe5 */ + { 0x502808018, 0x300, 1 }, /* venc0_me0 */ + { 0x502808020, 0x300, 1 }, /* venc0_me1 */ + { 0x502820000, 0x10, 0 }, + { 0x502820004, 0x10, 0 }, + { 0x502820008, 0x10, 0 }, + { 0x50282000c, 0x0, 0 }, + { 0x502820010, 0x0, 0 }, + { 0x502820014, 0x0, 0 }, + { 0x50282c000, 0x80000000, 0 }, + { 0x50282c040, 0x80000000, 0 }, + { 0x50282c0a0, 0x80000000, 0 }, + { 0x50282c100, 0x80000000, 0 }, + { 0x50282c108, 0x80000000, 0 }, + { 0x50282c110, 0x80000000, 0 }, + { 0x502888000, 0xffef, 0 }, + { 0x502888004, 0xffef, 0 }, + { 0x502888008, 0x0, 0 }, + { 0x50288800c, 0x0, 0 }, + { 0x502888010, 0x0, 0 }, + { 0x502888014, 0x0, 0 }, + { 0x502888018, 0x0, 0 }, + { 0x50288801c, 0x0, 0 }, + { 0x5048a9c14, 0x0, 0 }, + { 0x5048a9c18, 0x0, 0 }, + { 0x508300000, 0x1f0000ff, 1 }, /* debug_gated */ + { 0x508300058, 0xff, 1 }, /* nub_spmi_ahb */ + { 0x508300060, 0x1400024f, 1 }, /* nub_spmi0 */ + { 0x508300068, 0x1400024f, 1 }, /* nub_spmi_a0 */ + { 0x508300070, 0x1400024f, 1 }, /* nub_spmi_a1 */ + { 0x508300078, 0x1400024f, 1 }, /* nub_spmi_a2 */ + { 0x508300080, 0x200, 1 }, /* nub_spi0 */ + { 0x508300088, 0x1f0002ff, 1 }, /* nub_aon */ + { 0x508300090, 0x2ff, 1 }, /* nub_gpio */ + { 0x508300098, 0x200, 1 }, /* nub_ocla */ + { 0x5083000a0, 0x2ff, 1 }, /* atc0_common_dp */ + { 0x5083000a8, 0x200, 1 }, /* atc1_common_dp */ + { 0x5083000b0, 0x200, 1 }, /* atc2_common_dp */ + { 0x5083000b8, 0x200, 1 }, /* atc3_common_dp */ + { 0x5083000c0, 0x1400024f, 1 }, /* nub_spmi1 */ + { 0x5083000c8, 0xff, 1 }, /* atc0_usb_aon */ + { 0x5083000d0, 0x300, 1 }, /* atc1_usb_aon */ + { 0x5083000d8, 0x300, 1 }, /* atc2_usb_aon */ + { 0x5083000e0, 0x300, 1 }, /* atc3_usb_aon */ + { 0x5083000e8, 0x1400024f, 1 }, /* nub_spmi2 */ + { 0x5083000f0, 0x1400024f, 1 }, /* nub_spmi3 */ + { 0x5083000f8, 0x1400024f, 1 }, /* nub_spmi4 */ + { 0x508300100, 0x1f00030f, 1 }, /* nub_fabric */ + { 0x508300108, 0x1000030f, 1 }, /* nub_sram */ + { 0x508300110, 0x1f0000ff, 1 }, /* debug_switch */ + { 0x508300118, 0x2ff, 1 }, /* atc0_usb */ + { 0x508300120, 0x300, 1 }, /* atc1_usb */ + { 0x508300140, 0x300, 1 }, /* atc2_usb */ + { 0x508300148, 0x300, 1 }, /* atc3_usb */ + { 0x508300150, 0x3ff, 1 }, /* atc0_phymxwrap */ + { 0x508300158, 0x300, 1 }, /* atc1_phymxwrap */ + { 0x508300160, 0x300, 1 }, /* atc2_phymxwrap */ + { 0x508300168, 0x300, 1 }, /* atc3_phymxwrap */ + { 0x508360000, 0x0, 0 }, + { 0x508360004, 0x1, 0 }, + { 0x508360008, 0x1, 0 }, + { 0x50836000c, 0x0, 0 }, + { 0x508360010, 0x0, 0 }, + { 0x508360014, 0x0, 0 }, + { 0x508360018, 0x0, 0 }, + { 0x50836001c, 0x0, 0 }, + { 0x508360020, 0x0, 0 }, + { 0x508360024, 0x0, 0 }, + { 0x508360028, 0x0, 0 }, + { 0x50836002c, 0x100, 0 }, + { 0x508360030, 0x3, 0 }, + { 0x508360034, 0x100, 0 }, + { 0x508360038, 0x0, 0 }, + { 0x50836003c, 0x100, 0 }, + { 0x508360040, 0x100, 0 }, + { 0x508360044, 0x3, 0 }, + { 0x508360048, 0x10100, 0 }, + { 0x50836004c, 0x10100, 0 }, + { 0x508360050, 0x100, 0 }, + { 0x508360054, 0x100, 0 }, + { 0x508360058, 0x1, 0 }, + { 0x50836005c, 0x0, 0 }, + { 0x508360060, 0x0, 0 }, + { 0x508360064, 0x0, 0 }, + { 0x508360068, 0x0, 0 }, + { 0x50836006c, 0x0, 0 }, + { 0x508360070, 0x0, 0 }, + { 0x508360074, 0x0, 0 }, + { 0x508360078, 0x0, 0 }, + { 0x50836007c, 0x0, 0 }, + { 0x508360080, 0x0, 0 }, + { 0x508360084, 0x0, 0 }, + { 0x508360088, 0x0, 0 }, + { 0x50836008c, 0x0, 0 }, + { 0x508360090, 0x0, 0 }, + { 0x508360094, 0x0, 0 }, + { 0x508360098, 0x0, 0 }, + { 0x50836009c, 0x0, 0 }, + { 0x5083600a0, 0x0, 0 }, + { 0x5083600a4, 0x0, 0 }, + { 0x5083600a8, 0x0, 0 }, + { 0x5083600ac, 0x0, 0 }, + { 0x5083600b0, 0x0, 0 }, + { 0x5083600b4, 0x0, 0 }, + { 0x5083600b8, 0x0, 0 }, + { 0x5083600bc, 0x0, 0 }, + { 0x5083600c0, 0x0, 0 }, + { 0x5083600c4, 0x0, 0 }, + { 0x5083600c8, 0x0, 0 }, + { 0x5083600cc, 0x0, 0 }, + { 0x5083600d0, 0x0, 0 }, + { 0x5083600d4, 0x0, 0 }, + { 0x5083600d8, 0x0, 0 }, + { 0x5083600dc, 0x0, 0 }, + { 0x5083600e0, 0x0, 0 }, + { 0x5083600e4, 0x0, 0 }, + { 0x5083600e8, 0x0, 0 }, + { 0x5083600ec, 0x0, 0 }, + { 0x5083600f0, 0x0, 0 }, + { 0x5083600f4, 0x0, 0 }, + { 0x5083600f8, 0x0, 0 }, + { 0x5083600fc, 0x0, 0 }, + { 0x508360100, 0x880ff04, 0 }, + { 0x508360104, 0x80, 0 }, + { 0x508360108, 0x3, 0 }, + { 0x50836010c, 0x1, 0 }, + { 0x508360110, 0x0, 0 }, + { 0x508360114, 0x0, 0 }, + { 0x508360118, 0x0, 0 }, + { 0x50836011c, 0x0, 0 }, + { 0x508360120, 0x0, 0 }, + { 0x508908214, 0x10000, 0 }, + { 0x504800000, 0x0, 0 }, /* pmc_voter0 (macOS 0 per dj) */ + { 0x504804000, 0x0, 0 }, /* pmc_voter1 (macOS 0 per dj) */ + { 0x504808000, 0x0, 0 }, /* pmc_voter2 (macOS 0 per dj) */ + { 0x50480c000, 0x0, 0 }, /* pmc_voter3 (macOS 0 per dj) */ +}; diff --git a/drivers/soc/apple/t6041-ps-order.h b/drivers/soc/apple/t6041-ps-order.h new file mode 100644 index 0000000000000..848087e604e8c --- /dev/null +++ b/drivers/soc/apple/t6041-ps-order.h @@ -0,0 +1,229 @@ +/* PS rows of t6041_macos_regs, parents before children (t6040-pmgr.dtsi power-domains; T6041-only words by their T6040 twin), CPU and CPM words left out. Generated by the snippet in NOTES 10 Nov. */ +static const u16 t6041_ps_order[] = { + 0, /* gpx, depth 0 */ + 1, /* afr, depth 0 */ + 5, /* ispsens0, depth 0 */ + 6, /* apcie_gp, depth 0 */ + 7, /* apcie_st0, depth 0 */ + 8, /* apcie_st1, depth 0 */ + 111, /* pms, depth 0 */ + 112, /* dwi, depth 0 */ + 113, /* afi, depth 0 */ + 118, /* pmgr_soc_ocla, depth 0 */ + 126, /* afc, depth 0 */ + 153, /* fab_gw_0, depth 0 */ + 154, /* fab_gw_1, depth 0 */ + 155, /* fab_afr, depth 0 */ + 236, /* debug_gated, depth 0 */ + 237, /* nub_spmi_ahb, depth 0 */ + 239, /* nub_spmi_a0, depth 0 */ + 240, /* nub_spmi_a1, depth 0 */ + 241, /* nub_spmi_a2, depth 0 */ + 242, /* nub_spi0, depth 0 */ + 243, /* nub_aon, depth 0 */ + 244, /* nub_gpio, depth 0 */ + 245, /* nub_ocla, depth 0 */ + 249, /* atc3_common_dp, depth 0 */ + 251, /* atc0_usb_aon, depth 0 */ + 252, /* atc1_usb_aon, depth 0 */ + 253, /* atc2_usb_aon, depth 0 */ + 254, /* atc3_usb_aon, depth 0 */ + 258, /* nub_fabric, depth 0 */ + 260, /* debug_switch, depth 0 */ + 2, /* gfx, depth 1 */ + 9, /* fab3_soc, depth 1 */ + 11, /* fab4_soc, depth 1 */ + 50, /* amcc4, depth 1 */ + 51, /* amcc5, depth 1 */ + 52, /* amcc6, depth 1 */ + 53, /* amcc7, depth 1 */ + 54, /* amcc12, depth 1 */ + 55, /* amcc13, depth 1 */ + 56, /* amcc14, depth 1 */ + 57, /* amcc15, depth 1 */ + 58, /* amcc20, depth 1 */ + 59, /* amcc21, depth 1 */ + 60, /* amcc22, depth 1 */ + 61, /* amcc23, depth 1 */ + 114, /* aic, depth 1 */ + 115, /* pmc, depth 1 */ + 116, /* pms_c1ppt, depth 1 */ + 117, /* soc_dpe, depth 1 */ + 119, /* pms_fpwm0, depth 1 */ + 120, /* pms_fpwm1, depth 1 */ + 121, /* pms_fpwm2, depth 1 */ + 122, /* pms_fpwm3, depth 1 */ + 123, /* pms_fpwm4, depth 1 */ + 124, /* fab0_soc, depth 1 */ + 127, /* amcc0, depth 1 */ + 130, /* amcc1, depth 1 */ + 132, /* amcc2, depth 1 */ + 134, /* amcc3, depth 1 */ + 136, /* amcc8, depth 1 */ + 138, /* amcc9, depth 1 */ + 140, /* amcc10, depth 1 */ + 142, /* amcc11, depth 1 */ + 144, /* amcc16, depth 1 */ + 146, /* amcc17, depth 1 */ + 148, /* amcc18, depth 1 */ + 150, /* amcc19, depth 1 */ + 162, /* fab1_soc, depth 1 */ + 168, /* fab2_soc, depth 1 */ + 188, /* pmp, depth 1 */ + 189, /* pms_sram, depth 1 */ + 194, /* fab5_soc, depth 1 */ + 201, /* fab6_soc, depth 1 */ + 202, /* fab7_soc, depth 1 */ + 238, /* nub_spmi0, depth 1 */ + 250, /* nub_spmi1, depth 1 */ + 255, /* nub_spmi2, depth 1 */ + 256, /* nub_spmi3, depth 1 */ + 257, /* nub_spmi4, depth 1 */ + 259, /* nub_sram, depth 1 */ + 10, /* sio, depth 2 */ + 49, /* disp_sys, depth 2 */ + 62, /* dcs_04, depth 2 */ + 63, /* dcs_05, depth 2 */ + 64, /* dcs_06, depth 2 */ + 65, /* dcs_07, depth 2 */ + 66, /* dcs_12, depth 2 */ + 67, /* dcs_13, depth 2 */ + 68, /* dcs_14, depth 2 */ + 69, /* dcs_15, depth 2 */ + 70, /* dcs_20, depth 2 */ + 71, /* dcs_21, depth 2 */ + 72, /* dcs_22, depth 2 */ + 73, /* dcs_23, depth 2 */ + 75, /* ans, depth 2 */ + 76, /* isp_sys, depth 2 */ + 79, /* apcie_sys_gp, depth 2 */ + 81, /* jpg, depth 2 */ + 83, /* msr1, depth 2 */ + 85, /* ane_sys, depth 2 */ + 86, /* sep, depth 2 */ + 125, /* avd_sys, depth 2 */ + 128, /* dcs_00, depth 2 */ + 129, /* rom, depth 2 */ + 131, /* dcs_01, depth 2 */ + 133, /* dcs_02, depth 2 */ + 135, /* dcs_03, depth 2 */ + 137, /* dcs_08, depth 2 */ + 139, /* dcs_09, depth 2 */ + 141, /* dcs_10, depth 2 */ + 143, /* dcs_11, depth 2 */ + 145, /* dcs_16, depth 2 */ + 147, /* dcs_17, depth 2 */ + 149, /* dcs_18, depth 2 */ + 151, /* dcs_19, depth 2 */ + 152, /* venc0_sys, depth 2 */ + 156, /* venc1_sys, depth 2 */ + 163, /* dispext3_sys, depth 2 */ + 166, /* scodec, depth 2 */ + 169, /* dispext0_sys, depth 2 */ + 172, /* atc0_common, depth 2 */ + 177, /* atc1_common, depth 2 */ + 182, /* atc2_common, depth 2 */ + 187, /* atc3_common, depth 2 */ + 195, /* sbr, depth 2 */ + 199, /* msr0, depth 2 */ + 203, /* dispext1_sys, depth 2 */ + 206, /* dispext2_sys, depth 2 */ + 12, /* sio_cpu, depth 3 */ + 13, /* fpwm0, depth 3 */ + 14, /* fpwm1, depth 3 */ + 15, /* fpwm2, depth 3 */ + 16, /* i2c0, depth 3 */ + 17, /* i2c1, depth 3 */ + 18, /* i2c2, depth 3 */ + 19, /* i2c3, depth 3 */ + 20, /* i2c4, depth 3 */ + 21, /* i2c5, depth 3 */ + 22, /* i2c6, depth 3 */ + 23, /* i2c7, depth 3 */ + 24, /* i2c8, depth 3 */ + 25, /* spi_p, depth 3 */ + 26, /* uart_p, depth 3 */ + 27, /* dpa_p, depth 3 */ + 28, /* aes, depth 3 */ + 74, /* disp_fe, depth 3 */ + 77, /* apcie_sys_st0, depth 3 */ + 78, /* apcie_sys_st1, depth 3 */ + 82, /* prores, depth 3 */ + 84, /* msr1_ase_core, depth 3 */ + 87, /* ane_mpm, depth 3 */ + 88, /* ane_cpu, depth 3 */ + 89, /* ane_td, depth 3 */ + 157, /* venc1_dma, depth 3 */ + 164, /* dispext3_fe, depth 3 */ + 167, /* scodec_streaming, depth 3 */ + 170, /* dispext0_fe, depth 3 */ + 173, /* atc0_cio, depth 3 */ + 174, /* atc0_pcie, depth 3 */ + 178, /* atc1_cio, depth 3 */ + 179, /* atc1_pcie, depth 3 */ + 183, /* atc2_cio, depth 3 */ + 184, /* atc2_pcie, depth 3 */ + 190, /* atc3_cio, depth 3 */ + 191, /* atc3_pcie, depth 3 */ + 196, /* gpio, depth 3 */ + 197, /* msg, depth 3 */ + 198, /* ap_tmm, depth 3 */ + 200, /* msr0_ase_core, depth 3 */ + 204, /* dispext1_fe, depth 3 */ + 207, /* dispext2_fe, depth 3 */ + 209, /* venc0_dma, depth 3 */ + 261, /* atc0_usb, depth 3 */ + 262, /* atc1_usb, depth 3 */ + 263, /* atc2_usb, depth 3 */ + 264, /* atc3_usb, depth 3 */ + 265, /* atc0_phymxwrap, depth 3 */ + 266, /* atc1_phymxwrap, depth 3 */ + 267, /* atc2_phymxwrap, depth 3 */ + 268, /* atc3_phymxwrap, depth 3 */ + 29, /* spi0, depth 4 */ + 30, /* spi1, depth 4 */ + 31, /* spi2, depth 4 */ + 32, /* spi3, depth 4 */ + 33, /* spi4, depth 4 */ + 34, /* spi5, depth 4 */ + 35, /* qspi, depth 4 */ + 36, /* uart_n, depth 4 */ + 37, /* uart0, depth 4 */ + 38, /* uart1, depth 4 */ + 39, /* uart2, depth 4 */ + 40, /* uart3, depth 4 */ + 41, /* uart4, depth 4 */ + 42, /* uart5, depth 4 */ + 43, /* uart6, depth 4 */ + 44, /* dpa0, depth 4 */ + 45, /* dpa1, depth 4 */ + 46, /* dpa2, depth 4 */ + 47, /* dpa3, depth 4 */ + 48, /* dpa4, depth 4 */ + 80, /* apcie_phy_sw, depth 4 */ + 90, /* ane_base, depth 4 */ + 91, /* disp_cpu, depth 4 */ + 158, /* venc1_pipe4, depth 4 */ + 159, /* venc1_pipe5, depth 4 */ + 160, /* venc1_me0, depth 4 */ + 165, /* dispext3_cpu, depth 4 */ + 171, /* dispext0_cpu, depth 4 */ + 175, /* atc0_cio_pcie, depth 4 */ + 176, /* atc0_cio_usb, depth 4 */ + 180, /* atc1_cio_pcie, depth 4 */ + 181, /* atc1_cio_usb, depth 4 */ + 185, /* atc2_cio_pcie, depth 4 */ + 186, /* atc2_cio_usb, depth 4 */ + 192, /* atc3_cio_pcie, depth 4 */ + 193, /* atc3_cio_usb, depth 4 */ + 205, /* dispext1_cpu, depth 4 */ + 208, /* dispext2_cpu, depth 4 */ + 210, /* venc0_pipe4, depth 4 */ + 211, /* venc0_pipe5, depth 4 */ + 212, /* venc0_me0, depth 4 */ + 246, /* atc0_common_dp, depth 4 */ + 247, /* atc1_common_dp, depth 4 */ + 248, /* atc2_common_dp, depth 4 */ + 161, /* venc1_me1, depth 5 */ + 213, /* venc0_me1, depth 5 */ +}; diff --git a/drivers/soc/apple/t6041-sleep.c b/drivers/soc/apple/t6041-sleep.c new file mode 100644 index 0000000000000..b2bc88529d01b --- /dev/null +++ b/drivers/soc/apple/t6041-sleep.c @@ -0,0 +1,1473 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T6041 (M4 Max) system sleep, at EL2 with no monitor below. + * + * It runs macOS 26.6.2's last steps before the S2R commit, in macOS's order and with its + * values, as traced from macOS sleeping under aura's hypervisor, and commits; aura's S2R + * mailbox brings the boot core back to cpu_resume on the wake, and the driver restores what + * the sleep took. + * + * Needs the kernel at EL2 (CTXR window) and the "aurora,rvbar-spin-table" CPU enable method, + * so the other cores can leave the kernel. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_BITS (BIT(0) | BIT(63)) + +/* Physical blocks (ADT /arm-io/pmgr reg entries on J616C). */ +#define PA_PS 0x502800000UL /* pmgr[37] and on: PS words */ +#define PA_EAST 0x502280000UL /* pmgr[2]: East PS words */ +#define PA_CLUSTER 0x502820000UL /* pmgr[44] */ +#define PA_PWRGATE 0x50282c000UL /* pmgr[38] */ +#define PA_MISC_CORES 0x502888000UL /* pmgr[40] + 0x8000 */ +#define PA_WDT 0x50836c000UL +#define PA_AON_MISC 0x508330000UL +#define PA_COMMIT 0x508904000UL /* pmgr[55] + 0x4000 */ +#define PA_CLKGEN 0x500038000UL +#define PA_SPMI0 0x509014000UL /* nub-spmi0 */ +#define PA_KA_PMGR 0x5029dc000UL /* pmgr[0] + 0x1dc000: keep-alive table */ +#define PA_KA_EAST 0x5022d0000UL /* pmgr[2] + 0x50000: keep-alive table */ +#define KA_WORDS 32 +#define KA_STRIDE 0x44 + +#define PS_AUTO_ENABLE BIT(28) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) + +#define SPMI_STATUS 0x0 +#define SPMI_CMD 0x4 +#define SPMI_RX_EMPTY BIT(24) +#define SPMI_NOTIFY_MASK 0x40 +#define SPMI_IRQ_ACK 0x80 +#define PMU_SID 0xb + +static const u64 t6041_macos_off[] = { + 0x508300168, + 0x508300160, + 0x508300158, + 0x508300148, + 0x508300140, + 0x508300120, + 0x5083000e0, + 0x5083000d8, + 0x5083000d0, + 0x5083000b8, + 0x5083000b0, + 0x5083000a8, + 0x508300098, + 0x508300080, + 0x502808020, + 0x502808018, + 0x502808010, + 0x502808008, + 0x502808000, + 0x502800478, + 0x502800470, + 0x502800468, + 0x502800458, + 0x502800438, + 0x502800430, + 0x502800428, + 0x502800420, + 0x502800408, + 0x502800400, + 0x5028003f8, + 0x5028003f0, + 0x5028003e8, + 0x5028003e0, + 0x5028003d8, + 0x5028003d0, + 0x5028003c8, + 0x5028003c0, + 0x5028003b8, + 0x5028003b0, + 0x5028003a8, + 0x5028003a0, + 0x502800398, + 0x502800330, + 0x502800328, + 0x5028002c8, + 0x5028002c0, + 0x5028002b8, + 0x5028002b0, + 0x5028002a8, + 0x5028002a0, + 0x502800278, + 0x502800190, + 0x502800170, + 0x502800160, + 0x502800158, + 0x502800150, + 0x502800148, + 0x502800140, + 0x502800138, + 0x50228c018, + 0x50228c010, + 0x50228c008, + 0x50228c000, + 0x5022803c0, + 0x5022803b8, + 0x5022803b0, + 0x5022803a8, + 0x5022803a0, + 0x502280398, + 0x502280390, + 0x502280388, + 0x502280380, + 0x502280378, + 0x502280370, + 0x502280250, + 0x502280248, + 0x502280240, + 0x502280238, + 0x502280230, + 0x5022801e8, + 0x5022801e0, + 0x5022801d8, + 0x5022801d0, + 0x5022801c8, + 0x5022801c0, + 0x5022801b8, + 0x502280190, + 0x502280188, + 0x502280180, + 0x502280178, + 0x502280170, + 0x502280168, + 0x502280160, + 0x502280158, + 0x502280150, + 0x502280148, + 0x502280140, + 0x502280138, + 0x502280110, + 0x502280108, + 0x502280100, + 0x502280000, + 0x288e80110, + 0x288e80100, +}; + +/* aura's struct aura_s2r_mailbox (aura PR 117). */ +#define T6041_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ +struct t6041_s2r_mailbox { + u64 magic; + u64 boot_mpidr; + u64 resume_pa; + u64 resumes; +}; + +struct t6041_sleep { + struct device *dev; + struct t6041_s2r_mailbox *mbox; + void __iomem *ps, *east, *cluster, *pwrgate, *misc, *wdt, *aon, *commit, *clkgen, *spmi; + void __iomem *off[ARRAY_SIZE(t6041_macos_off)]; + void __iomem *ps_aon, *pmc; + void __iomem *ka[2]; /* keep-alive tables, 1 from iBoot, 0 at macOS's commit */ + void __iomem *spmi_a0; /* 0x509188000: nub-spmi-a0, notify mask written by macOS at -113 ms */ + unsigned int ngroups; /* AMCCs of four planes: 8 on the full T6041, 6 on the binned one */ + void __iomem *amcc[32]; + /* DCS channel blocks (ADT mcc reg 20+): 0x220200000 + 0x2000000 * g + 0x200000 * c */ + void __iomem *dcs[32]; + /* AMCC blocks (ADT mcc reg 12+): 0x220000000 + 0x2000000 * g, planes at + 0x40000 * p */ + void __iomem *amccb[8]; + void __iomem *wake; /* 0x508004000: SLP_DDR +0, wake masks +0x0c..+0x2c */ + void __iomem *perf; /* 0x50020c000: SoC perf +0, DCS perf +0x400 */ + void __iomem *axi2af[15]; + void __iomem *ptd; /* 0x5003d0000: the PMP's PTD update page (ADT mcc[11]) */ + int stop_ret; + int coproc_ret; + int apcie_ret; + unsigned int ps_late, gp_late; + /* Saved as the commit starts, put back by t6041_sleep_return(). */ + u64 cntkctl, siq_cfg; + u32 pre_perf[2], pre_pmc[5], pre_clk[4], pre_mask[2], pre_wake[9], pre_votes[5]; + /* Each AMCC plane's cache ways (+0x2800) and status (+0x2804). */ + u32 pre_ways[32], pre_cache[32]; + u32 full_pre[15]; /* t6041_full_ps as the commit starts */ + bool pre_a0; + /* GP PCIe ports 0/1: port block, PHY word, and what they held as the commit started. */ + void __iomem *gp_port[2], *gp_phy[2]; + u32 gp_phy_pre[2], gp_app_pre[2], gp_perst_pre[2]; + bool gp_saved; + /* ANS's storage PCIe block (ADT /arm-io/apcie0 reg 1-6). */ + void __iomem *ap_rc, *ap_phy, *ap_phy_ip, *ap_axi, *ap_cio3pll, *ap_clkgen; + bool ans_stopped; + /* The AOP island's 44 MiniPMGR power words, 0x508304000 + 8 n. */ + void __iomem *aop_ps; + u32 aop_pre[44]; + /* The AP GPIO block's 200 pin words (ADT gpio0, #gpio-pins 0xc8), as the commit starts. */ + void __iomem *gpio; + u32 gpio_pre[200]; +}; + +static struct t6041_sleep *t6041_sleep; + +#include "t6041-macos-regs.h" +#include "t6041-ps-order.h" +#include "t6041-apcie-tunables.h" +/* Each register of the macOS commit-state table, mapped. */ +static void __iomem *t6041_reg_map[ARRAY_SIZE(t6041_macos_regs)]; + +/* + * The full T6041 has eight AMCCs and 32 memory channels, the binned one six and 24, and the + * macOS commit table above is from a binned J616c. These are the PS words of the other eight + * channels (named from a full J614c's ADT; on the binned part they read target and actual 0), + * each with the value macOS has for the same kind of word: an AMCC slice 0x1000030f (AUTO, + * auto state 0), a DCS channel 0x1400034f (auto state 4). Left forced on, they keep fab3_soc + * (East +0x118) on at the commit. dcs_27 isn't among the words the J614c has on there. + */ +static const struct { u8 east; u16 off; u32 val; } t6041_full_ps[15] = { + { 0, 0x248, 0x1000030f }, /* amcc24 */ + { 0, 0x250, 0x1400034f }, /* dcs_24 */ + { 0, 0x258, 0x1000030f }, /* amcc25 */ + { 0, 0x260, 0x1400034f }, /* dcs_25 */ + { 0, 0x268, 0x1000030f }, /* amcc26 */ + { 0, 0x270, 0x1400034f }, /* dcs_26 */ + { 0, 0x280, 0x1000030f }, /* amcc27 */ + { 1, 0x2c0, 0x1000030f }, /* amcc28 */ + { 1, 0x2c8, 0x1000030f }, /* amcc29 */ + { 1, 0x2d0, 0x1000030f }, /* amcc30 */ + { 1, 0x2d8, 0x1000030f }, /* amcc31 */ + { 1, 0x340, 0x1400034f }, /* dcs_28 */ + { 1, 0x348, 0x1400034f }, /* dcs_29 */ + { 1, 0x350, 0x1400034f }, /* dcs_30 */ + { 1, 0x358, 0x1400034f }, /* dcs_31 */ +}; + +static void __iomem *t6041_full_reg(struct t6041_sleep *s, unsigned int i) +{ + return (t6041_full_ps[i].east ? s->east : s->ps) + t6041_full_ps[i].off; +} + +/* On the full part only: the extra channels' words to their macOS kind, as t6041_ps_match_all(). */ +static void t6041_full_ps_match(struct t6041_sleep *s) +{ + const u32 mask = PS_TARGET | PS_AUTO_ENABLE | GENMASK(27, 24); + unsigned int i; + + if (s->ngroups != 8) + return; + for (i = 0; i < ARRAY_SIZE(t6041_full_ps); i++) { + void __iomem *reg = t6041_full_reg(s, i); + u32 now = readl(reg), mac = t6041_full_ps[i].val; + + if ((now & mask) != (mac & mask)) + writel((now & ~mask) | (mac & mask), reg); + } +} + +/* ADT cpu number: E 0-3, P0 4-8, P1 10-14. */ +static unsigned int t6041_cpu_id(u64 mpidr) +{ + unsigned int cl = MPIDR_AFFINITY_LEVEL(mpidr, 1), core = MPIDR_AFFINITY_LEVEL(mpidr, 0); + + return cl == 0 ? core : cl == 1 ? 4 + core : 10 + core; +} + +/* + * A core's last WFI, as the Neo's and the Air's t8140_deep_wfi() and macOS's last-WFI routine + * with x0 = 1: timers masked, the deep-sleep bits (C6_2 bit 0 and 63) set, then WFI for good. + * (Not the runtime offline branch, which leaves C6_2 clear.) + */ +static void __noreturn t6041_offline_wfi(void) +{ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_BITS); + isb(); + while (true) { + dsb(sy); + isb(); + wfi(); + aic_sleep_ack(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void t6041_cpu_stop(u64 mpidr) +{ + writel(BIT(t6041_cpu_id(mpidr)), t6041_sleep->misc + 0x0); + dsb(sy); +} + +static int t6041_cpu_on(u64 mpidr) +{ + void __iomem *misc = t6041_sleep->misc; + unsigned int cl = MPIDR_AFFINITY_LEVEL(mpidr, 1); + + writel(BIT(t6041_cpu_id(mpidr)), misc + 0x4); + writel(BIT(MPIDR_AFFINITY_LEVEL(mpidr, 0)), misc + 0x8 + 4 * cl); + return 0; +} + + +static void t6041_ps_lower(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); +} + +/* One byte to a PMU register over nub-spmi0, as XNU's writes: command, then the data. */ +static void t6041_pmu_write(struct t6041_sleep *s, u16 reg, u8 val) +{ + unsigned int us; + + /* A reply someone else left (aura's own writes at the wake), so the next one is ours. */ + while (!(readl(s->spmi + SPMI_STATUS) & SPMI_RX_EMPTY)) + readl(s->spmi + 0x8); + writel((u32)reg << 16 | BIT(15) | PMU_SID << 8 | 0x30, s->spmi + SPMI_CMD); + writel(val, s->spmi + SPMI_CMD); + /* + * Take the reply once it is there. The controller stays powered through the sleep, so a + * reply that comes in after we stop looking becomes the reply to the SPMI driver's next + * command after the wake, which then fails as an unexpected response. + */ + for (us = 0; us < 1000 && (readl(s->spmi + SPMI_STATUS) & SPMI_RX_EMPTY); us++) + udelay(1); + while (!(readl(s->spmi + SPMI_STATUS) & SPMI_RX_EMPTY)) + readl(s->spmi + 0x8); + /* macOS acknowledges every command here (its interrupt handler writes 1 to +0x80). */ + writel(1, s->spmi + SPMI_IRQ_ACK); +} + +static void east(struct t6041_sleep *s, u32 off, u32 v) +{ + writel(v, s->east + off); +} + +static void cpm_gates(struct t6041_sleep *s) +{ + writel(0x80000000, s->pwrgate + 0x100); + writel(0x80000000, s->pwrgate + 0x108); + writel(0x80000000, s->pwrgate + 0x110); +} + +/* + * The core and CPM words macOS has on at its commit (0x1f0: on, target 0, so they drop when + * the core idles). Linux runs the cores with them reading off, so nothing ever drops. + */ +static const u16 t6041_cpu_ps[] = { + 0x00, 0x08, 0x10, 0x18, /* ecpu 0-3 */ + 0x20, 0x28, 0x30, 0x38, 0x40, /* pcpu0 0-4 */ + 0x50, 0x58, 0x60, 0x68, 0x70, /* pcpu1 0-4 */ + 0x80, 0x88, /* ecpm, pcpm0 */ +}; + +/* + * Every named PS word to macOS's target, AUTO_ENABLE and auto state at its commit, as the Neo's match + * step does: macOS leaves half the DCS channels and their AMCC slices to the hardware. + * Returns how many words changed. + */ +static int t6041_ps_match_all(void) +{ + unsigned int i; + int n = 0; + + for (i = 0; i < ARRAY_SIZE(t6041_macos_regs); i++) { + /* + * With the auto state [27:24], the state AUTO_ENABLE drops an idle domain to: + * macOS has 0 (off) for half the AMCC slices and dcs_23, 4 for fab3_soc, where + * Linux kept iBoot's 4 and f (AVE test, 9 Nov 19:24). + */ + u32 mask = PS_TARGET | PS_AUTO_ENABLE | GENMASK(27, 24), mac = t6041_macos_regs[i].val, now; + + if (!t6041_macos_regs[i].ps) + continue; + now = readl(t6041_reg_map[i]); + if ((now & mask) == (mac & mask)) + continue; + writel((now & ~mask) | (mac & mask), t6041_reg_map[i]); + n++; + } + return n; +} + +static void t6041_spin_s(u64 ms) +{ + u64 t0 = __arch_counter_get_cntpct(), f = arch_timer_get_cntfrq(); + + while (__arch_counter_get_cntpct() - t0 < ms * f / 1000) + cpu_relax(); +} + +static void t6041_cpu_off(u64 mpidr) +{ + t6041_offline_wfi(); +} + +static const struct spin_table_power_ops t6041_cpu_power_ops = { + .cpu_stop = t6041_cpu_stop, + .cpu_off = t6041_cpu_off, + .cpu_on = t6041_cpu_on, +}; + +static void __noreturn t6041_boot_wfi(struct t6041_sleep *s) +{ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_BITS); + while (aic_sleep_ack()) + ; + while (true) { + dsb(sy); + isb(); + wfi(); + aic_sleep_ack(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + + +/* Raise each to on, then drop its target as macOS has it. Returns how many read on. */ +static int t6041_cpu_ps_match(struct t6041_sleep *s) +{ + unsigned int i; + int up = 0; + u32 v; + + for (i = 0; i < ARRAY_SIZE(t6041_cpu_ps); i++) { + void __iomem *reg = s->ps + t6041_cpu_ps[i]; + + v = readl(reg); + writel((v & ~(PS_TARGET | PS_AUTO_ENABLE)) | 0xf, reg); + if (!readl_poll_timeout_atomic(reg, v, FIELD_GET(PS_ACTUAL, v) == 0xf, 1, 2000)) + up++; + } + for (i = 0; i < ARRAY_SIZE(t6041_cpu_ps); i++) { + void __iomem *reg = s->ps + t6041_cpu_ps[i]; + + writel(readl(reg) & ~PS_TARGET, reg); + } + return up; +} + +/* + * Every USB-C port's power domains, deepest first, then the fabric above them. The macOS + * trace these states were matched to had aura holding ATC0's chain on for its debug link + * (dj: on the Neo, macOS only slept once aura dropped that link). PS offsets: pmgr2 + * (0x502800000) for the commons and CIO/PCIe, pmgr3 AON (0x508300000) for the rest. + */ +static const u16 t6041_atc_aon[] = { + 0x150, 0x158, 0x160, 0x168, /* atcN_phymxwrap */ + 0x118, 0x120, 0x140, 0x148, /* atcN_usb */ + 0x0c8, 0x0d0, 0x0d8, 0x0e0, /* atcN_usb_aon */ + 0x0a0, 0x0a8, 0x0b0, 0x0b8, /* atcN_common_dp */ +}; +static const u16 t6041_atc_soc[] = { + 0x3b0, 0x3a8, 0x3a0, 0x398, /* atc0 cio_usb, cio_pcie, pcie, cio */ + 0x3d8, 0x3d0, 0x3c8, 0x3c0, /* atc1 */ + 0x400, 0x3f8, 0x3f0, 0x3e8, /* atc2 */ + 0x438, 0x430, 0x428, 0x420, /* atc3 */ + 0x390, 0x3b8, 0x3e0, 0x408, /* atcN_common */ +}; +/* fab2_soc and afi: forcing them off hung the Max (8 Nov). Left to the hardware instead. */ +static const u16 t6041_fab_auto[] = { 0x338, 0x110 }; + +static void t6041_atc_down(struct t6041_sleep *s) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(t6041_atc_aon); i++) + t6041_ps_lower(s->ps_aon + t6041_atc_aon[i]); + for (i = 0; i < ARRAY_SIZE(t6041_atc_soc); i++) + t6041_ps_lower(s->ps + t6041_atc_soc[i]); + udelay(2000); + for (i = 0; i < ARRAY_SIZE(t6041_fab_auto); i++) { + void __iomem *reg = s->ps + t6041_fab_auto[i]; + + writel(readl(reg) | PS_AUTO_ENABLE, reg); + } +} + +/* + * The M4 Pro's fix for the same stall (linux#1477, t6040-sleep fab_on bit 2): the engine's ATC + * step raises each atcN_common and polls ACTUAL f; with every common off, their parent fab2_soc + * is auto-gated and the common never comes up. So right before the commit: fab2_soc on by hand + * (AUTO off, target f), the four commons on (target f, no AUTO), then fab2_soc's AUTO back; the + * powered commons hold it up. + */ +static int t6041_ps_raise_start(void __iomem *reg) +{ + u32 v; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + return readl_poll_timeout_atomic(reg, v, (v & PS_ACTUAL) == PS_ACTUAL, 1, 5000); +} + +static void t6041_atc_commons_on(struct t6041_sleep *s) +{ + static const u16 common[4] = { 0x390, 0x3b8, 0x3e0, 0x408 }; + void __iomem *fab2 = s->ps + 0x338; + int i; + + t6041_ps_raise_start(fab2); + for (i = 0; i < 4; i++) + t6041_ps_raise_start(s->ps + common[i]); + writel((readl(fab2) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, fab2); +} + +/* + * The Neo's step just before its commit (dj: "the last stall"): on every AMCC plane, the + * DSID age-out (word 0 bit 25) and unexpected RT hit (word 1 bit 8) error enables off. aura's + * boot leaves an unexpected RT hit latched in INTSTS1 (0x100) on the Max's planes; those are + * written back to clear them too. + */ +static void t6041_amcc_quiesce(struct t6041_sleep *s) +{ + int p, w; + + for (p = 0; p < 4 * s->ngroups; p++) { + void __iomem *a = s->amcc[p]; + + writel(readl(a + 0x0) & ~BIT(25), a + 0x0); + writel(readl(a + 0x8) & ~BIT(8), a + 0x8); + for (w = 0; w < 7; w++) { + u32 v = readl(a + 4 + 8 * w); + + if (v) + writel(v, a + 4 + 8 * w); + } + } + dsb(sy); +} + +/* + * What macOS's memory controller driver does and the EL2 path never did (kernelcache 26.6.2, + * AppleT6041MemCacheController and AppleT6041SOCTuner; neither is in a trace, the first is + * done at boot and the second in the quiesce actions after the watchdog arm): + * - mccSelectDCSDynamicPowerGateMode(1), from the tuner at boot (ADT mcc-power-gating = 1): + * +0x20 bits 8 and 12 set on every enabled DCS channel block. + * - cacheFlush, the driver's quiesce action: per AMCC +0x82c = 1 (allocation off) and + * +0x2844 = 1 (flush), then +0x2844 bit 0 polled clear on each of its four planes. + * macOS has every amcc slice power-gated at its commit (0x1000030f); here they only + * clock-gated (0x1400024f), and the engine stalls in its memory cache program. + * Run while the slices are still forced on. + */ +static void t6041_mcc_macos(struct t6041_sleep *s) +{ + unsigned int i, p; + u32 v; + + for (i = 0; i < s->ngroups; i++) { + /* +0x1c0000 is the AMCC's all-planes window: macOS writes there, polls per plane. */ + writel(1, s->amccb[i] + 0x1c082c); + writel(1, s->amccb[i] + 0x1c2844); + for (p = 0; p < 4; p++) + readl_poll_timeout_atomic(s->amccb[i] + 0x40000 * p + 0x2844, v, + !(v & BIT(0)), 10, 100000); + } + dsb(sy); + for (i = 0; i < 4 * s->ngroups; i++) + writel(readl(s->dcs[i] + 0x20) | 0x1100, s->dcs[i] + 0x20); + dsb(sy); +} + +/* + * AppleT6041PMGR's AXI2AF workaround (macOS 26.6.2, the table at the kernelcache's _DATA and the + * routine that writes it): 15 fabric bridges, each with the PS word of its block. Before a sleep, + * every bridge whose block is on gets bit 0 set in its CPG node (+0x104) and link teardown + * (+0x108) words, so the fabric sees the link down; macOS asserts all 15 blocks are off by then. + * Linux powers these blocks off as plain PS words, never tearing the link down. + * pmgr: 0 = 0x502800000, 2 = 0x502280000. + */ +static const struct { u64 base; u8 pmgr; u16 ps; } t6041_axi2af[15] = { + { 0x380000000, 0, 0x390 }, /* atc0_common */ + { 0x388000000, 0, 0x3b8 }, /* atc1_common */ + { 0x390000000, 0, 0x3e0 }, /* atc2_common */ + { 0x398000000, 0, 0x408 }, /* atc3_common */ + { 0x403000000, 2, 0x258 }, /* disp_sys */ + { 0x3a3000000, 0, 0x340 }, /* dispext0_sys */ + { 0x603000000, 0, 0x490 }, /* dispext1_sys */ + { 0x607000000, 0, 0x4e0 }, /* dispext2_sys */ + { 0x307000000, 0, 0x2d8 }, /* dispext3_sys */ + { 0x40b000000, 2, 0x370 }, /* ans */ + { 0x280000000, 0, 0x170 }, /* avd_sys */ + { 0x282000000, 0, 0x278 }, /* venc0_sys */ + { 0x284000000, 0, 0x2a0 }, /* venc1_sys */ + { 0x486000000, 2, 0x3a0 }, /* jpg */ + { 0x416000000, 2, 0x390 }, /* apcie_sys_gp */ +}; + +/* + * iBoot's own S2R entry ends with the wake masks, a SEPROM command, then SEP PS |= bit 10 + * (DEV_DISABLE) and the commit. macOS at its commit (HV stop, 9 Nov) has SLP_DDR 0, exactly + * the 25 ADT ap-wake-sources unmasked in the 9 mask words at +0x0c, and SEP PS 0x1000070f. + */ +static const u32 t6041_wake_masks[9] = { + 0xe5ff5fff, 0xff42edff, 0xffffffff, 0x3ffffbdf, 0xf7ffffc0, + 0xffffff7f, 0xffffffff, 0xffffffff, 0x00003fff, +}; + +static void t6041_wake_sep(struct t6041_sleep *s) +{ + unsigned int i; + u32 v; + + if (readl(s->wake + 0x0)) + writel(0, s->wake + 0x0); + for (i = 0; i < ARRAY_SIZE(t6041_wake_masks); i++) + writel(t6041_wake_masks[i], s->wake + 0xc + 4 * i); + + v = readl(s->east + 0xc00); + if (!(v & BIT(10))) + writel(v | BIT(10), s->east + 0xc00); + dsb(sy); +} + +/* A perf word: the low nibble asks, [11:8] is the state reached, bit 31 busy. */ +static bool t6041_perf_set(void __iomem *reg, u32 state) +{ + u32 v; + + if (FIELD_GET(GENMASK(11, 8), readl(reg)) == state) + return true; + writel((readl(reg) & ~0xf) | state, reg); + return !readl_poll_timeout_atomic(reg, v, + !(v & BIT(31)) && FIELD_GET(GENMASK(11, 8), v) == state, 1, 5000); +} + +/* SoC to 3, voters 0, DCS to 6: macOS commits at DCS 6 (0x7666); the engine takes it to 4. */ +static void t6041_mem_down(struct t6041_sleep *s) +{ + int i; + + t6041_perf_set(s->perf + 0x0, 3); + for (i = 0; i < 4; i++) + writel(0, s->pmc + i * 0x4000); + t6041_perf_set(s->perf + 0x400, 6); +} + +/* + * The cluster words at 0x502820000: +0 E, +4 P0, +8 P1 (macOS's runtime writes and its offline + * order), +0xc and +0x10 the two AVE complexes. Bit 5 is the up-done flag (CLUSTER_UP_DONE in + * the Neo's driver), left set by iBoot's up and cleared by writing it. XNU clears it with its + * first write and turns the AVE complexes off at boot, so at macOS's commit +0xc/+0x10 read 0 + * and the engine has no sequence to run for them; Linux had them on (0x2f), then 0x10/0x30 at + * the commit. As the Neo's t8140_sleep_ave_off(): target 0 with the flag, then wait for 0. + */ +static void t6041_ave_off(struct t6041_sleep *s) +{ + static const u16 ave[] = { 0xc, 0x10 }; + unsigned int i; + u32 v; + + for (i = 0; i < ARRAY_SIZE(ave); i++) { + writel(BIT(5), s->cluster + ave[i]); + readl_poll_timeout_atomic(s->cluster + ave[i], v, !v, 1, 5000); + } + v = readl(s->cluster + 0x8); + if (v & BIT(5)) + writel((v & 0xf) | BIT(5), s->cluster + 0x8); +} + +static void t6041_save(struct t6041_sleep *s); + +/* + * macOS lowers an East domain with two writes: AUTO off with the target still f, then target 0. + * On a domain Linux already has off, the first one turns it on under a parent that is off, and + * bit 11 latches. So only when it is on. + */ +static void east_lower(struct t6041_sleep *s, u32 off, u32 first, u32 second) +{ + u32 v = readl(s->east + off); + + if (!FIELD_GET(PS_TARGET, v) && !FIELD_GET(PS_ACTUAL, v)) + return; + writel(first, s->east + off); + writel(second, s->east + off); +} + +static int t6041_sleep_commit(unsigned long unused) +{ + struct t6041_sleep *s = t6041_sleep; + unsigned int i; + + t6041_save(s); + s->stop_ret = spin_table_stop_parked(); + t6041_ave_off(s); + /* While every AMCC slice is still on: a clock-gated plane faults. */ + t6041_amcc_quiesce(s); + t6041_mcc_macos(s); + /* + * The wake arms bit 0 of both words of every bridge whose block is on (and of ANS's in + * the storage raise), and boot has them clear (bridge 4: 0x10000 / 0x101330). Lowered with + * its bridge still armed, a block latches bit 11: on the second sleep of a boot ans and + * venc1_sys did, the engine waited and the chip reset. So only that bit is cleared again + * here, on blocks that are on (a bridge behind an off block can't be read), as linux#1477 + * does before it lowers APCIE_SYS_GP. + */ + for (i = 0; i < 15; i++) { + u32 v = readl((t6041_axi2af[i].pmgr ? s->east : s->ps) + t6041_axi2af[i].ps); + + if (FIELD_GET(PS_ACTUAL, v) != 0xf) + continue; + v = readl(s->axi2af[i] + 0x4); + if (!(v & 1) && !(readl(s->axi2af[i] + 0x8) & 1)) + continue; + writel(v & ~1U, s->axi2af[i] + 0x4); + writel(readl(s->axi2af[i] + 0x8) & ~1U, s->axi2af[i] + 0x8); + } + t6041_cpu_ps_match(s); + t6041_mem_down(s); + + for (i = 0; i < ARRAY_SIZE(t6041_macos_off); i++) { + t6041_ps_lower(s->off[i]); + udelay(100); + } + udelay(5000); + t6041_full_ps_match(s); + t6041_ps_match_all(); + /* As macOS at its commit (dj): every PMC voter at 0, and the PMS word it has at 0. */ + for (i = 0; i < 4; i++) + writel(0, s->pmc + i * 0x4000); + writel(0, s->pmc + 0xa9c18); + udelay(5000); + /* macOS 82 ms before its commit (trace, 8 Nov): the four cluster PTD votes, no demand. */ + writeq(0x2000000000000000ULL, s->ptd + 0xf80); + writeq(0x2000000000000000ULL, s->ptd + 0xf88); + writeq(0x2000000000000000ULL, s->ptd + 0xf90); + writeq(0x2000000000000000ULL, s->ptd + 0xfa0); + t6041_atc_down(s); + t6041_wake_sep(s); + + writel(0xffef, s->misc + 0x0); + /* + * macOS's order: its other cores are in their last WFI, deep-sleep bits set, 37 ms + * before the commit, and everything below runs with only the boot core up. AIC masked + * first so nothing ends that WFI. + */ + aic_sleep_mask_all(); + spin_table_release_stopped(); + t6041_spin_s(30); + + writel(0x3f833ff1, s->spmi + SPMI_NOTIFY_MASK); + /* The PMU boot stage to 0x50 (asleep). */ + t6041_pmu_write(s, 0xf801, 0x50); + east(s, 0x180, 0x3f0); + east(s, 0x168, 0x3f0); + east(s, 0x160, 0x3f0); + east(s, 0x158, 0x3f0); + writel(0x2000001, s->spmi + SPMI_NOTIFY_MASK); + /* macOS writes nub-spmi-a0's mask too (-113 ms). Only with its domain on, which macOS keeps. */ + if (FIELD_GET(PS_ACTUAL, readl(s->ps_aon + 0x68))) + writel(0x2000001, s->spmi_a0 + SPMI_NOTIFY_MASK); + east(s, 0x1c8, 0x3f0); + cpm_gates(s); + writel(0, s->cluster + 0x0); + cpm_gates(s); + writel(0, s->cluster + 0x8); + east_lower(s, 0x398, 0x400004f, 0x4000340); + east_lower(s, 0x388, 0xf, 0x300); + east(s, 0x110, 0x3f0); + east_lower(s, 0x380, 0xf0000ff, 0xf0003f0); + east(s, 0x108, 0x3f0); + east_lower(s, 0x390, 0xf, 0x300); + east(s, 0x100, 0x3f0); + east_lower(s, 0x370, 0xf0000ff, 0xf0003f0); + /* macOS 10 and 4 ms before arming WD2: the SoC PTD word, then without bit 33. */ + writeq(0x200000800ULL, s->ptd + 0xf00); + udelay(5000); + writeq(0x800ULL, s->ptd + 0xf00); + writel(0, s->wdt + 0x0c); + writel(0, s->wdt + 0x2c); + writel(0, s->wdt + 0x20); + writel(0x7270e00, s->wdt + 0x28); + writel(1, s->wdt + 0x2c); + writel(0, s->aon + 0x8); + cpm_gates(s); + cpm_gates(s); + cpm_gates(s); + writel(0x10, s->cluster + 0x00); + writel(0, s->cluster + 0x04); + writel(0x10, s->cluster + 0x08); + writel(0x20, s->cluster + 0x0c); + writel(0, s->cluster + 0x10); + east(s, 0x120, 0x400024f); + east(s, 0x130, 0x400024f); + writel(0xa0100000, s->clkgen + 0x60); + writel(0xa0100000, s->clkgen + 0x64); + writel(0xa0100000, s->clkgen + 0x54); + writel(0xa0100000, s->clkgen + 0x50); + /* aura reads this with the MMU off on the way back. */ + WRITE_ONCE(s->mbox->resume_pa, __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + t6041_atc_commons_on(s); + /* + * No SPMI near the commit or after it, as on macOS (its last command is 113 ms before) + * and on the Air (nothing to the PMU after the commit): the engine drives this bus. + */ + t6041_spin_s(20); + /* + * macOS's last writes before its commit (ApplePMGR's keep-alive off, -0.6 ms): every + * keep-alive word to 0, the two tables in turn. + */ + for (i = 0; i < KA_WORDS; i++) { + writel(0, s->ka[0] + i * KA_STRIDE); + writel(0, s->ka[1] + i * KA_STRIDE); + } + dsb(sy); + writel(1, s->commit); + dsb(sy); + /* The other cores are in their deep WFI already. AIC masked again, 3 ms, then this core's. */ + aic_sleep_mask_all(); + t6041_spin_s(3); + t6041_boot_wfi(s); +} + +/* + * The display pipes' PMC votes (linux#1477 "G41": macOS sets them before a pipe's power words: + * boost 0x5048ac028, DVMR 0x5048a4000 with its ack at +0x20, BWR 0x5048ac020, the cluster rail + * 0x5048a9c18). iBoot leaves them set (boost 0x1f, rail 0xc) and Linux never touched them; after + * a wake they read 0, and the display then runs through every step and shows black. Saved as + * the commit starts and put back after the wake, offsets into the PMC block at 0x504800000. + */ +static const u32 t6041_pmc_vote_off[5] = { 0xac028, 0xa4000, 0xac020, 0xa9c10, 0xa9c20 }; + +/* + * macOS's memory-cache Active action (AppleT6041MemCacheController::mccEnableCacheMode(true)): + * the way count back through the all-planes window, then each plane's status polled. After a + * wake every plane reads +0x2800 = 0x80000000 and +0x2804 = 0 (cache off), where Linux had + * 0x8000000c / 0x0c000c00. The allocation-disable word +0x82c is left as it was. + */ +static void t6041_mcc_active(struct t6041_sleep *s) +{ + unsigned int i, p; + u32 v; + + for (i = 0; i < s->ngroups; i++) { + writel(s->pre_ways[4 * i], s->amccb[i] + 0x1c2800); + for (p = 0; p < 4; p++) + readl_poll_timeout_atomic(s->amccb[i] + 0x40000 * p + 0x2804, v, + v == s->pre_cache[4 * i + p], 10, 100000); + } +} + +/* Puts back the display's PMC votes; the DVMR one with its ack. */ +static void t6041_pmc_votes(struct t6041_sleep *s) +{ + unsigned int i; + u32 v; + + for (i = 0; i < ARRAY_SIZE(t6041_pmc_vote_off); i++) { + void __iomem *reg = s->pmc + t6041_pmc_vote_off[i]; + + if (readl(reg) != s->pre_votes[i]) + writel(s->pre_votes[i], reg); + if (t6041_pmc_vote_off[i] == 0xa4000 && s->pre_votes[i]) + readl_poll_timeout_atomic(reg + 0x20, v, v & BIT(0), 1, 8000); + } +} + +/* What the commit is about to change, as Linux left it. */ +static u32 t6041_pre[ARRAY_SIZE(t6041_macos_regs)]; +static const u16 t6041_clk_mux[4] = { 0x50, 0x54, 0x64, 0x60 }; + +static void t6041_save(struct t6041_sleep *s) +{ + unsigned int i; + + for (i = 0; i < 4 * s->ngroups; i++) { + void __iomem *plane = s->amccb[i / 4] + 0x40000 * (i % 4); + + s->pre_ways[i] = readl(plane + 0x2800); + s->pre_cache[i] = readl(plane + 0x2804); + } + for (i = 0; i < ARRAY_SIZE(t6041_pmc_vote_off); i++) + s->pre_votes[i] = readl(s->pmc + t6041_pmc_vote_off[i]); + s->cntkctl = read_sysreg(cntkctl_el1); + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + for (i = 0; i < ARRAY_SIZE(t6041_macos_regs); i++) + if (t6041_macos_regs[i].ps) + t6041_pre[i] = readl(t6041_reg_map[i]); + for (i = 0; i < ARRAY_SIZE(t6041_full_ps) && s->ngroups == 8; i++) + s->full_pre[i] = readl(t6041_full_reg(s, i)); + s->pre_perf[0] = readl(s->perf + 0x0); + s->pre_perf[1] = readl(s->perf + 0x400); + for (i = 0; i < 4; i++) + s->pre_pmc[i] = readl(s->pmc + i * 0x4000); + s->pre_pmc[4] = readl(s->pmc + 0xa9c18); + for (i = 0; i < 4; i++) + s->pre_clk[i] = readl(s->clkgen + t6041_clk_mux[i]); + s->pre_mask[0] = readl(s->spmi + SPMI_NOTIFY_MASK); + s->pre_a0 = FIELD_GET(PS_ACTUAL, readl(s->ps_aon + 0x68)) != 0; + if (s->pre_a0) + s->pre_mask[1] = readl(s->spmi_a0 + SPMI_NOTIFY_MASK); + for (i = 0; i < 9; i++) + s->pre_wake[i] = readl(s->wake + 0xc + 4 * i); + for (i = 0; i < 44; i++) + s->aop_pre[i] = readl(s->aop_ps + 8 * i); + for (i = 0; i < 200; i++) + s->gpio_pre[i] = readl(s->gpio + 4 * i); + /* Only with the GP PCIe block on: apcie_gp (East +0x100) and apcie_sys_gp (+0x390). */ + s->gp_saved = FIELD_GET(PS_ACTUAL, readl(s->east + 0x100)) == 0xf && + FIELD_GET(PS_ACTUAL, readl(s->east + 0x390)) == 0xf; + for (i = 0; i < 2 && s->gp_saved; i++) { + s->gp_phy_pre[i] = readl(s->gp_phy[i]); + s->gp_app_pre[i] = readl(s->gp_port[i] + 0x800); + s->gp_perst_pre[i] = readl(s->gp_port[i] + 0x82c); + } +} + +/* + * The AP GPIO block loses every pin's configuration in the sleep. Linux's pinctrl driver only + * writes back the pins it configured itself; the rest (pin functions iBoot set up for other + * users, the display's among them) stayed at their reset values, and after a wake the DCP's + * power-on ended in "No device added after powering on the rails. HPD=0". macOS restores all + * of them: its ADT lists only three exceptions (gpio0 no-restore-gpio-pins = 190, 191, 192). + */ +static void t6041_gpio_restore(struct t6041_sleep *s) +{ + unsigned int i; + + for (i = 0; i < 200; i++) { + if (i >= 190 && i <= 192) + continue; + if (readl(s->gpio + 4 * i) != s->gpio_pre[i]) + writel(s->gpio_pre[i], s->gpio + 4 * i); + } +} + +/* + * linux#1477 t6040_aop_wake (aop_wake bits 1 and 2): the AOP island's power words are not in + * our PS table. Parents first (their order table backwards), every word that was on before the + * sleep goes to manual target f, AUTO off, so it can't gate again before the AOP driver reads + * its ASC. + */ +static const u16 t6041_aop_ps_order[] = { + 0x0b0, 0x128, 0x120, 0x158, 0x150, 0x148, 0x140, 0x138, 0x130, 0x118, 0x0c0, 0x0c8, + 0x0e8, 0x0f0, 0x0f8, 0x100, 0x108, 0x110, 0x118, 0x0e0, 0x0d8, 0x0d0, 0x0b8, 0x090, + 0x020, 0x098, 0x0a0, 0x0a8, 0x028, 0x030, 0x068, 0x070, 0x078, 0x080, 0x038, 0x040, + 0x048, 0x058, 0x060, 0x088, 0x050, 0x010, 0x018, 0x008, 0x000, +}; + +static void t6041_aop_wake(struct t6041_sleep *s) +{ + int k; + u32 v; + + for (k = ARRAY_SIZE(t6041_aop_ps_order) - 1; k >= 0; k--) { + u16 off = t6041_aop_ps_order[k]; + void __iomem *r = s->aop_ps + off; + u32 now = readl(r); + + if (!FIELD_GET(PS_ACTUAL, s->aop_pre[off / 8])) + continue; + if (FIELD_GET(PS_ACTUAL, now) == 0xf && !(now & PS_AUTO_ENABLE)) + continue; + writel((now & ~(PS_AUTO_ENABLE | PS_TARGET)) | PS_TARGET, r); + readl_poll_timeout_atomic(r, v, FIELD_GET(PS_ACTUAL, v), 1, 2000); + } +} + +static void t6041_tun_apply(void __iomem *regs, const struct t6041_tun *t, unsigned int n, + bool only_changed) +{ + unsigned int i; + + for (i = 0; i < n; i++) { + u32 old = readl(regs + t[i].off); + u32 val = (old & ~t[i].mask) | (t[i].val & t[i].mask); + + if (!only_changed || val != old) + writel(val, regs + t[i].off); + } +} + +static int t6041_ps_raise(void __iomem *reg) +{ + int ret = t6041_ps_raise_start(reg); + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, reg); + return ret; +} + +/* + * ANS's storage link from power-off, a port of linux#1477 t6040_sleep_raise_apcie_st() with its + * defaults (apcie_wake = 7): the domains in macOS's order with the two bridges armed and SYS_GP + * left in manual on, the axi2af / cio3pll / clkgen tunables and the PLL start, APCIE_PHY_SW, + * the PHY tunables and its two clock requests, the PHY-IP PLL and lane tables, then the root + * complex's ready handshake. Returns 0, or the step (1-8) that timed out. Runs only when the + * NVMe driver stopped ANS for this sleep. + */ +#define TUN(reg, tbl, chg) t6041_tun_apply(reg, tbl, ARRAY_SIZE(tbl), chg) + +static int t6041_apcie_raise(struct t6041_sleep *s) +{ + void __iomem *ps = s->east, *ctrl = s->ap_phy + 0x8000, *run = s->ap_phy + 0x8004; + u32 v; + int ret; + + ret = t6041_ps_raise(ps + 0x370); + if (!ret) { + writel(readl(s->axi2af[9] + 0x4) | 1, s->axi2af[9] + 0x4); + writel(readl(s->axi2af[9] + 0x8) | 1, s->axi2af[9] + 0x8); + } + ret |= t6041_ps_raise(ps + 0x100); + if (!ret) + ret = t6041_ps_raise_start(ps + 0x390); + if (!ret) { + writel(readl(s->ap_axi + 0x104) | 1, s->ap_axi + 0x104); + writel(readl(s->ap_axi + 0x108) | 1, s->ap_axi + 0x108); + } + ret |= t6041_ps_raise(ps + 0x108); + ret |= t6041_ps_raise(ps + 0x380); + ret |= t6041_ps_raise(ps + 0x110); + ret |= t6041_ps_raise(ps + 0x388); + if (ret) + return 1; + + TUN(s->ap_axi, t6041_tun_axi2af, false); + TUN(s->ap_cio3pll, t6041_tun_cio3pll, false); + TUN(s->ap_clkgen, t6041_tun_clkgen, false); + writel(readl(s->ap_clkgen + 0x4) | BIT(31), s->ap_clkgen + 0x4); + writel(readl(s->ap_clkgen + 0x0) & ~(BIT(2) | BIT(1)), s->ap_clkgen + 0x0); + if (readl_poll_timeout_atomic(s->ap_clkgen + 0x0, v, !(v & BIT(31)), 1, 50000)) + return 2; + if (t6041_ps_raise(ps + 0x398)) + return 3; + TUN(s->ap_phy, t6041_tun_phy, false); + if (readl_poll_timeout_atomic(s->ap_phy + 0x4000, v, v & BIT(31), 1, 50000)) + return 4; + writel(readl(ctrl) | BIT(0), ctrl); + if (readl_poll_timeout_atomic(ctrl, v, v & BIT(2), 1, 50000)) + return 5; + writel(readl(ctrl) | BIT(1), ctrl); + if (readl_poll_timeout_atomic(ctrl, v, v & BIT(3), 1, 50000)) + return 6; + writel(readl(ctrl) & ~BIT(4), ctrl); + + writel(readl(run) | BIT(0), run); + TUN(s->ap_phy_ip, t6041_tun_phy_ip_pll, true); + TUN(s->ap_phy_ip, t6041_tun_phy_ip_lanes, true); + writel(readl(s->ap_phy_ip + 0x90) & ~BIT(16), s->ap_phy_ip + 0x90); + writel(readl(run) | BIT(4), run); + if (readl_poll_timeout_atomic(s->ap_phy + 0x8008, v, v & BIT(0), 1, 50000)) + return 7; + writel(readl(s->ap_phy + 0x4000) | BIT(0), s->ap_phy + 0x4000); + writel(readl(ctrl) | BIT(27), ctrl); + + writel(0, s->ap_rc + 0x04); + writel(0x140, s->ap_rc + 0x54); + writel(1, s->ap_rc + 0x50); + if (readl_poll_timeout_atomic(s->ap_rc + 0x58, v, v & BIT(0), 1, 50000)) + return 8; + return 0; +} + +/* + * linux#1477 t6040_gp_pcie_clk (its el2.123/124): the DARTs at 0x410000000 and 0x411000000 sit + * in the blocks of GP PCIe ports 0 and 1, whose clocks the bootloader set up and nothing does + * after a real sleep; reading them then raises an SError. Per port: both reference clock + * requests with their acks, reset off, the enables and link bit as saved, the app clock, the + * controller's PERST word under its guard bit, then PORT_STATUS ready. Returns a mask of the + * ports that didn't come back. + */ +static unsigned int t6041_gp_pcie_clk(struct t6041_sleep *s) +{ + unsigned int i, late = 0; + u32 v; + + for (i = 0; i < 2; i++) { + void __iomem *phy = s->gp_phy[i], *port = s->gp_port[i]; + u32 cfg = s->gp_phy_pre[i], app = s->gp_app_pre[i]; + + if (!s->gp_saved || FIELD_GET(PS_ACTUAL, readl(s->east + 0x100)) != 0xf || + FIELD_GET(PS_ACTUAL, readl(s->east + 0x390)) != 0xf) + continue; + writel(readl(phy) | BIT(0), phy); + if (readl_poll_timeout_atomic(phy, v, v & BIT(2), 1, 50000)) + late |= BIT(i); + writel(readl(phy) | BIT(1), phy); + if (readl_poll_timeout_atomic(phy, v, v & BIT(3), 1, 50000)) + late |= BIT(i); + if (!(cfg & BIT(4))) { + writel(readl(phy) & ~BIT(4), phy); + udelay(1); + } + writel(readl(phy) | (cfg & (BIT(9) | BIT(10))), phy); + writel(readl(phy) | (cfg & BIT(31)), phy); + writel((readl(port + 0x800) & ~(BIT(0) | BIT(8))) | (app & (BIT(0) | BIT(8))), + port + 0x800); + writel(readl(port + 0x13c) | BIT(8), port + 0x13c); + writel(s->gp_perst_pre[i], port + 0x82c); + writel(readl(port + 0x13c) & ~BIT(8), port + 0x13c); + if (readl_poll_timeout_atomic(port + 0x804, v, v & BIT(0), 1, 50000)) + late |= BIT(i); + } + return late; +} + +/* + * Every PS word back to the target and AUTO it had as the commit started: raises parents + * first, then lowers children first, so no word is switched under a parent that is off (that + * latches bit 11). Returns how many raises didn't reach their target in 3 ms. + */ +static unsigned int t6041_ps_restore(struct t6041_sleep *s) +{ + const u32 keep = ~(u32)(PS_AUTO_ENABLE | PS_FLAGS | PS_TARGET); + unsigned int late = 0; + int k; + u32 v; + + for (k = 0; k < ARRAY_SIZE(t6041_ps_order); k++) { + void __iomem *reg = t6041_reg_map[t6041_ps_order[k]]; + u32 pre = t6041_pre[t6041_ps_order[k]], cur = readl(reg); + u32 pt = pre & PS_TARGET; + + if (pt > (cur & PS_TARGET)) { + writel((cur & keep) | pt, reg); + if (readl_poll_timeout_atomic(reg, v, FIELD_GET(PS_ACTUAL, v) == pt, 1, 3000)) + late++; + cur = readl(reg); + } + if (pt == (cur & PS_TARGET) && ((pre ^ cur) & PS_AUTO_ENABLE)) + writel((cur & ~(PS_AUTO_ENABLE | PS_FLAGS)) | (pre & PS_AUTO_ENABLE), reg); + } + for (k = ARRAY_SIZE(t6041_ps_order) - 1; k >= 0; k--) { + void __iomem *reg = t6041_reg_map[t6041_ps_order[k]]; + u32 pre = t6041_pre[t6041_ps_order[k]], cur = readl(reg); + u32 pt = pre & PS_TARGET; + + if (pt >= (cur & PS_TARGET)) + continue; + if (cur & PS_AUTO_ENABLE) { + cur &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(cur, reg); + } + writel((cur & keep) | pt | PS_FLAGS, reg); + udelay(50); + if (pre & PS_AUTO_ENABLE) + writel((readl(reg) & ~PS_FLAGS) | PS_AUTO_ENABLE, reg); + } + return late; +} + +/* The full part's extra channel words back to their target and AUTO, as t6041_ps_restore(). */ +static unsigned int t6041_full_ps_restore(struct t6041_sleep *s) +{ + const u32 keep = ~(u32)(PS_AUTO_ENABLE | PS_FLAGS | PS_TARGET); + unsigned int i, late = 0; + u32 v; + + for (i = 0; i < ARRAY_SIZE(t6041_full_ps) && s->ngroups == 8; i++) { + void __iomem *reg = t6041_full_reg(s, i); + u32 pre = s->full_pre[i], cur = readl(reg), pt = pre & PS_TARGET; + + if (pt > (cur & PS_TARGET)) { + writel((cur & keep) | pt, reg); + if (readl_poll_timeout_atomic(reg, v, FIELD_GET(PS_ACTUAL, v) == pt, + 1, 3000)) + late++; + cur = readl(reg); + } + if (pt == (cur & PS_TARGET) && ((pre ^ cur) & PS_AUTO_ENABLE)) + writel((cur & ~(PS_AUTO_ENABLE | PS_FLAGS)) | (pre & PS_AUTO_ENABLE), reg); + } + return late; +} + +static u64 t6041_ret_t0, t6041_ret_t1; + +/* Back on the boot core through aura's mailbox: the clusters on, the masks back, WD2 off. */ +static void t6041_sleep_return(struct t6041_sleep *s) +{ + unsigned int i; + u32 v; + + t6041_ret_t0 = read_sysreg(cntpct_el0); + /* The reset left the timer control and the AIC's IRQ routing at their EL2 defaults. */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + writel(0, s->wdt + 0x2c); + /* Keep-alive back on, as macOS on its way back and as iBoot leaves it. */ + for (i = 0; i < KA_WORDS; i++) { + writel(1, s->ka[0] + i * KA_STRIDE); + writel(1, s->ka[1] + i * KA_STRIDE); + } + spin_table_stop_finished(); + aic_sleep_restore(); + writel(0xf, s->cluster + 0x0); + writel(0xf, s->cluster + 0x4); + writel(0xf, s->cluster + 0x8); + /* As restoreHW (linux#1477): the two wake gates, then the CPM voters. */ + writel(readl(s->pwrgate + 0x000) | BIT(31), s->pwrgate + 0x000); + writel(readl(s->pwrgate + 0x040) | BIT(31), s->pwrgate + 0x040); + cpm_gates(s); + /* SEP's DEV_DISABLE off again, bit 11 polled clear; the S2R request word; the wake masks. */ + v = readl(s->east + 0xc00); + if (v & BIT(10)) + writel(v & ~BIT(10), s->east + 0xc00); + readl_poll_timeout_atomic(s->east + 0xc00, v, !(v & BIT(11)), 1, 5000); + writel(0, s->wake + 0x0); + for (i = 0; i < 9; i++) + writel(s->pre_wake[i], s->wake + 0xc + 4 * i); + s->ps_late = t6041_ps_restore(s); + /* Before t6041_mcc_active() touches their planes. */ + s->ps_late += t6041_full_ps_restore(s); + t6041_mcc_active(s); + t6041_aop_wake(s); + t6041_gpio_restore(s); + /* A bridge whose block is back on is armed again, or it drops its traffic (linux#1477). */ + for (i = 0; i < 15; i++) { + v = readl((t6041_axi2af[i].pmgr ? s->east : s->ps) + t6041_axi2af[i].ps); + if ((v & PS_TARGET) != PS_TARGET || FIELD_GET(PS_ACTUAL, v) != 0xf) + continue; + writel(readl(s->axi2af[i] + 0x4) | 1, s->axi2af[i] + 0x4); + writel(readl(s->axi2af[i] + 0x8) | 1, s->axi2af[i] + 0x8); + } + s->apcie_ret = s->ans_stopped ? t6041_apcie_raise(s) : 0; + s->gp_late = t6041_gp_pcie_clk(s); + /* The clock muxes the commit parked, last parked first; perf states; PMC voters. */ + for (i = 4; i-- > 0;) + writel(s->pre_clk[i], s->clkgen + t6041_clk_mux[i]); + t6041_perf_set(s->perf + 0x400, s->pre_perf[1] & 0xf); + t6041_perf_set(s->perf + 0x0, s->pre_perf[0] & 0xf); + for (i = 0; i < 4; i++) + writel(s->pre_pmc[i], s->pmc + i * 0x4000); + writel(s->pre_pmc[4], s->pmc + 0xa9c18); + t6041_pmc_votes(s); + writel(s->pre_mask[0], s->spmi + SPMI_NOTIFY_MASK); + if (s->pre_a0 && FIELD_GET(PS_ACTUAL, readl(s->ps_aon + 0x68))) + writel(s->pre_mask[1], s->spmi_a0 + SPMI_NOTIFY_MASK); + writel(0, s->wdt + 0x2c); + /* The PMU boot stage back from 0x50 (asleep) to 0x30 (OS running), as macsmc-reboot's probe. */ + t6041_pmu_write(s, 0xf801, 0x30); + t6041_ret_t1 = read_sysreg(cntpct_el0); +} + +static int t6041_sleep_enter(suspend_state_t state) +{ + struct t6041_sleep *s = t6041_sleep; + int ret; + + /* With no way back, a sleep would end in aura's boot flow over this kernel. */ + if (!s->mbox) + return -EOPNOTSUPP; + /* + * Every parked CPU answers the hold while a refusal is still possible: at the commit + * only a held CPU stops, and one left awake there keeps the engine waiting until the + * watchdog resets the SoC. + */ + ret = spin_table_hold_parked(); + if (ret) { + pr_err("t6041-sleep: refused: a parked CPU didn't answer the hold: %d\n", ret); + return ret; + } + /* macOS stops the PMP before its last steps; past this a refusal can't go back. */ + ret = apple_sleep_coprocs_stop_all(); + s->coproc_ret = ret; + s->ans_stopped = apple_sleep_coprocs_ans_stopped(); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, t6041_sleep_commit); + if (!ret) + t6041_sleep_return(s); + return ret; +} + +static void t6041_sleep_end(void) +{ + struct t6041_sleep *s = t6041_sleep; + + if (!s || !t6041_ret_t1) + return; + dev_dbg(s->dev, "back from deep sleep, return path %llu ms\n", + div64_u64((t6041_ret_t1 - t6041_ret_t0) * 1000, arch_timer_get_cntfrq())); + t6041_ret_t1 = 0; + if (s->coproc_ret < 0) + dev_warn(s->dev, "a coprocessor didn't stop for the sleep: %d\n", s->coproc_ret); + if (s->stop_ret) + dev_warn(s->dev, "the parked CPUs didn't all stop: %d\n", s->stop_ret); + /* One of them takes longer than 3 ms on every wake here, and comes up all the same. */ + if (s->ps_late) + dev_dbg(s->dev, "%u power domains took more than 3 ms to come back\n", s->ps_late); + if (s->apcie_ret) + dev_warn(s->dev, "storage link raise timed out at step %d\n", s->apcie_ret); + if (s->gp_late) + dev_warn(s->dev, "GP PCIe ports %#x didn't come back\n", s->gp_late); +} + +static int t6041_sleep_begin(suspend_state_t state) +{ + pm_set_suspend_via_firmware(); + return 0; +} + +static const struct platform_suspend_ops t6041_sleep_ops = { + .valid = suspend_valid_only_mem, + .begin = t6041_sleep_begin, + .end = t6041_sleep_end, + .enter = t6041_sleep_enter, +}; + +static int t6041_sleep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct t6041_sleep *s; + unsigned int i; + int ret; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + s = devm_kzalloc(dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = dev; + s->ps = devm_ioremap(dev, PA_PS, 0x18000); + s->east = devm_ioremap(dev, PA_EAST, 0x30000); + s->cluster = devm_ioremap(dev, PA_CLUSTER, 0x4000); + s->pwrgate = devm_ioremap(dev, PA_PWRGATE, 0x4000); + s->misc = devm_ioremap(dev, PA_MISC_CORES, 0x4000); + s->wdt = devm_ioremap(dev, PA_WDT, 0x4000); + s->aon = devm_ioremap(dev, PA_AON_MISC, 0x4000); + s->commit = devm_ioremap(dev, PA_COMMIT, 0x4); + s->clkgen = devm_ioremap(dev, PA_CLKGEN, 0x4000); + s->spmi = devm_ioremap(dev, PA_SPMI0, 0x4000); + s->spmi_a0 = devm_ioremap(dev, 0x509188000UL, 0x4000); + if (!s->spmi_a0) + return -ENOMEM; + s->ka[0] = devm_ioremap(dev, PA_KA_PMGR, KA_WORDS * KA_STRIDE); + s->ka[1] = devm_ioremap(dev, PA_KA_EAST, KA_WORDS * KA_STRIDE); + if (!s->ka[0] || !s->ka[1]) + return -ENOMEM; + s->ps_aon = devm_ioremap(dev, 0x508300000UL, 0x4000); + s->pmc = devm_ioremap(dev, 0x504800000UL, 0xb0000); + for (i = 0; i < 15; i++) { + s->axi2af[i] = devm_ioremap(dev, t6041_axi2af[i].base + 0x100, 0x10); + if (!s->axi2af[i]) + return -ENOMEM; + } + s->ptd = devm_ioremap(dev, 0x5003d0000UL, 0x4000); + if (!s->ptd) + return -ENOMEM; + s->perf = devm_ioremap(dev, 0x50020c000UL, 0x800); + if (!s->perf) + return -ENOMEM; + s->wake = devm_ioremap(dev, 0x508004000UL, 0x58); + if (!s->wake) + return -ENOMEM; + /* + * The full part: amcc24's word (+0x248) and amcc28's (East +0x2c0) aren't both target and + * actual 0 as on the binned one, where those channels are fused off. + */ + if (!s->ps || !s->east) + return -ENOMEM; + s->ngroups = (readl(s->ps + 0x248) & (PS_TARGET | PS_ACTUAL)) && + (readl(s->east + 0x2c0) & (PS_TARGET | PS_ACTUAL)) ? 8 : 6; + dev_info(dev, "%u memory channels\n", 4 * s->ngroups); + /* AMCC error words: groups of 4 planes, 0x220025000 + 0x2000000 * g + 0x40000 * p. */ + for (i = 0; i < 4 * s->ngroups; i++) { + s->amcc[i] = devm_ioremap(dev, 0x220025000UL + 0x2000000UL * (i / 4) + + 0x40000UL * (i % 4), 0x40); + if (!s->amcc[i]) + return -ENOMEM; + } + for (i = 0; i < 4 * s->ngroups; i++) { + s->dcs[i] = devm_ioremap(dev, 0x220200000UL + 0x2000000UL * (i / 4) + + 0x200000UL * (i % 4), 0x40); + if (!s->dcs[i]) + return -ENOMEM; + } + for (i = 0; i < s->ngroups; i++) { + s->amccb[i] = devm_ioremap(dev, 0x220000000UL + 0x2000000UL * i, 0x200000); + if (!s->amccb[i]) + return -ENOMEM; + } + if (!s->ps || !s->east || !s->cluster || !s->pwrgate || !s->misc || !s->wdt || !s->aon || + !s->commit || !s->clkgen || !s->spmi || !s->ps_aon || !s->pmc) + return -ENOMEM; + for (i = 0; i < ARRAY_SIZE(t6041_macos_regs); i++) { + t6041_reg_map[i] = devm_ioremap(dev, t6041_macos_regs[i].pa, 4); + if (!t6041_reg_map[i]) + return -ENOMEM; + } + for (i = 0; i < ARRAY_SIZE(t6041_macos_off); i++) { + s->off[i] = devm_ioremap(dev, t6041_macos_off[i], 4); + if (!s->off[i]) + return -ENOMEM; + } + + { + u64 pa; + + if (!of_property_read_u64(of_chosen, "aurora,s2r-mailbox", &pa)) { + s->mbox = devm_memremap(dev, pa, sizeof(*s->mbox), MEMREMAP_WB); + if (IS_ERR(s->mbox) || READ_ONCE(s->mbox->magic) != T6041_S2R_MAGIC) + s->mbox = NULL; + } + /* With no way back, a sleep would end in aura's boot flow over this kernel. */ + if (!s->mbox) { + dev_info(dev, "no aura S2R mailbox: no deep sleep\n"); + return -ENODEV; + } + } + s->gpio = devm_ioremap(dev, 0x51c000000UL, 0x400); + s->aop_ps = devm_ioremap(dev, 0x508304000UL, 0x160); + if (!s->aop_ps || !s->gpio) + return -ENOMEM; + s->ap_rc = devm_ioremap(dev, 0x414000000UL, 0x4000); + s->ap_phy = devm_ioremap(dev, 0x417000000UL, 0x40000); + s->ap_phy_ip = devm_ioremap(dev, 0x417040000UL, 0x28000); + s->ap_axi = devm_ioremap(dev, 0x416000000UL, 0x21000); + s->ap_cio3pll = devm_ioremap(dev, 0x415046200UL, 0x4000); + s->ap_clkgen = devm_ioremap(dev, 0x415044000UL, 0x4000); + if (!s->ap_rc || !s->ap_phy || !s->ap_phy_ip || !s->ap_axi || !s->ap_cio3pll || !s->ap_clkgen) + return -ENOMEM; + s->gp_port[0] = devm_ioremap(dev, 0x410028000UL, 0x1000); + s->gp_port[1] = devm_ioremap(dev, 0x411028000UL, 0x1000); + s->gp_phy[0] = devm_ioremap(dev, 0x417020000UL, 0x10); + s->gp_phy[1] = devm_ioremap(dev, 0x417024000UL, 0x10); + if (!s->gp_port[0] || !s->gp_port[1] || !s->gp_phy[0] || !s->gp_phy[1]) + return -ENOMEM; + t6041_sleep = s; + ret = spin_table_set_power_ops(&t6041_cpu_power_ops); + if (ret) + return dev_err_probe(dev, ret, "cannot control CPU power\n"); + suspend_set_ops(&t6041_sleep_ops); + dev_info(dev, "T6041 deep sleep at EL2 through aura's S2R mailbox\n"); + return 0; +} + +static const struct of_device_id t6041_sleep_of_match[] = { + { .compatible = "apple,t6041-sleep" }, + { } +}; + +static struct platform_driver t6041_sleep_driver = { + .driver = { + .name = "t6041-sleep", + .of_match_table = t6041_sleep_of_match, + .suppress_bind_attrs = true, + }, + .probe = t6041_sleep_probe, +}; +builtin_platform_driver(t6041_sleep_driver); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index b855d2bc82c83..571c9cbde1d2a 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -571,6 +571,23 @@ static int apple_spmi_resume_noirq(struct device *dev) return 0; guard(mutex)(&spmi->fifo_lock); + /* + * Firmware and the platform's sleep code use this queue while Linux is + * away. Every command of ours was answered before the suspend, so any + * reply waiting now is someone else's, and would be read as the reply + * to our next command. + */ + if (pm_resume_via_firmware()) { + unsigned int stale = 0; + + while (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) && + stale < 64) { + readl(spmi->regs + spmi->hw->reply); + stale++; + } + if (stale) + dev_info(dev, "dropped %u stale reply words after the wake\n", stale); + } raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); /* Whether the enables survive a real sleep has not been measured. */ if (pm_resume_via_firmware()) { diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index f6f27b4e0e152..ea3dbffcb2bc4 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -184,6 +184,8 @@ int apple_dart_sleep_prepare(struct device *dev); void apple_dart_sleep_restore(struct device *dev); /* A display DART's streams around a display power cycle, see the DART driver. */ int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush); +/* A locked display DART's TCR and TTBR words kept across a real sleep (T6041). */ +int apple_dart_display_keep(struct device *dev, u32 streams, bool save); /* A GAPF slice released before its coprocessor's CPU domain comes up after a wake. */ int apple_dart_gapf_ready(struct device *dev); /* A display DART whose domain goes off at runtime while others still map into it. */ @@ -213,6 +215,11 @@ static inline int apple_dart_display_streams(struct device *dev, u32 streams, bo return -ENODEV; } +static inline int apple_dart_display_keep(struct device *dev, u32 streams, bool save) +{ + return -ENODEV; +} + static inline int apple_dart_gapf_ready(struct device *dev) { return 0; diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 07e03087cbc3d..c831208cb685b 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -27,6 +27,10 @@ struct device_node; int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, int (*last_stop)(void *data), void *data); +/* For a SoC sleep driver of its own (T6041): the coprocessor stop, with interrupts off. */ +int apple_sleep_coprocs_stop_all(void); +bool apple_sleep_coprocs_ans_stopped(void); + /* * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index e0c4be586801d..9d39fc8726294 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -102,6 +102,12 @@ impl Mailbox { to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) } + /// Whether the firmware has room for another message (its receive FIFO isn't full). + pub fn send_ready(&self) -> bool { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_send_ready(self.mbox) } + } + /// Sets bit 0 of the receive control word, for firmware that powers itself down. pub fn set_recv_enabled(&self, enabled: bool) { // SAFETY: Calling the c function, `mbox` is a valid pointer