From bf32e65cea926a486246d8187fdf4ef0b10b36a4 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:05 -0500 Subject: [PATCH 01/73] dt-bindings: soc: apple: add the T8140 sleep controller A kernel that runs directly at EL2 on T8140 has no monitor to suspend the system or power a CPU off for it, so it writes the hardware itself. The registers it needs belong to several blocks, the PMGR, the watchdog, the wake controller, the AMCC, the PMC, SPMI, the memory controller and the clock blocks among them, and other drivers use some of them too. Describe them as one node whose driver doesn't claim its ranges. The node also names what the sleep depends on elsewhere: the wake sources to leave enabled, the display coprocessors that must be off before the commit, the port controller that carries the debug link, the PCIe host whose shared block ANS's link runs on, and the AP pin controller whose pins a deep sleep resets, with the pins to leave alone. Signed-off-by: Diljit Singh --- .../bindings/soc/apple/apple,t8140-sleep.yaml | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml new file mode 100644 index 0000000000000..adcd550f3d39a --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-sleep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 system sleep controls + +maintainers: + - Diljit Singh + +description: | + The registers a kernel running directly at EL2 on T8140 writes to enter and + return from suspend-to-RAM and to power its secondary CPUs off. They belong to + several hardware blocks, some of which other drivers also use, so this node + overlaps those drivers' register ranges and the driver does not claim them. + +properties: + compatible: + const: apple,t8140-sleep + + reg: + maxItems: 28 + + reg-names: + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: amcc2 + - const: amcc3 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: mcc2 + - const: ptd + - const: clk-perf + - const: cluster-dcs + + apple,wake-enable: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The wake source IDs left enabled across suspend-to-RAM. Every other + source is masked. + + apple,apcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PCIe host whose shared registers and tunables the storage controller's + link depends on. They are needed even with that host disabled. + + apple,pinctrl: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The pin controller whose pins a deep sleep resets. Its pins are saved + before a sleep and put back after it, before the storage controller + starts. + + apple,pinctrl-skip-pins: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Pins of that controller to leave alone across a sleep, neither saved nor + put back. + + apple,debug-port: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The USB-C port controller that carries the debug link, stopped for a sleep. + + apple,display-coprocessors: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The DCP and the external DCP, which must be off before a commit, in that + order. + +required: + - compatible + - reg + - reg-names + - apple,wake-enable + - apple,display-coprocessors + +additionalProperties: false From ec1e0d8e48c22fcc6ee3c88d456ba0e998e2bf3f Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:05 -0500 Subject: [PATCH 02/73] dt-bindings: soc: apple: add the CPU cluster power gates Each CPU cluster has a PMGR word whose top bit lets its cores power down in WFI. A core that PMGR powers on comes up allowed to, but the boot CPU only follows the word, and iBoot leaves it clear for T8140's P cluster. Describe the words, so that a driver can set the bit before the secondaries start. Signed-off-by: Diljit Singh --- .../soc/apple/apple,cluster-pwrgate.yaml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml new file mode 100644 index 0000000000000..8d4e6ab0029db --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,cluster-pwrgate.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC CPU cluster power-gate enables + +maintainers: + - Diljit Singh + +description: | + One PMGR word per CPU cluster, whose top bit lets the cluster's cores power + down in WFI. A core that PMGR powers on comes up allowed to, but the boot CPU + was started by the boot ROM and only follows the word, which the boot loader + may leave clear. The words are set early in boot, in the order listed. + +properties: + compatible: + items: + - enum: + - apple,t8140-cluster-pwrgate + - const: apple,cluster-pwrgate + + reg: + minItems: 1 + maxItems: 8 + description: One word per cluster. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + }; From cf6efa86d9ebb003690f5e5a64d5a813590a2470 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:05 -0500 Subject: [PATCH 03/73] dt-bindings: power: apple,pmgr-pwrstate: add apple,sleep-driver-owned After a firmware-driven suspend the power controller puts each domain back where genpd's books have it. Some domains need their driver's order on the way back instead: the GPU's needs its target, then its bridge, and only then automatic power management. Let the tree mark such a domain, so that the power controller leaves it to its driver after a wake. Signed-off-by: Diljit Singh --- .../devicetree/bindings/power/apple,pmgr-pwrstate.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d9cf0b9129926..4adb6f66c0079 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -23,6 +23,7 @@ allOf: properties: apple,firmware-owned: false apple,sleep-retained: false + apple,sleep-driver-owned: false apple,always-on: false apple,min-state: false apple,force-disable: false @@ -122,6 +123,13 @@ properties: system sleep. Runtime power management is unaffected. type: boolean + apple,sleep-driver-owned: + description: + The driver behind this domain lowers it before a system sleep and brings + it back afterwards, in an order its hardware needs. The power controller + leaves it alone after a wake. + type: boolean + apple,firmware-owned: description: The boot loader hands over active hardware whose power domain remains From f6526808f57c0ad0444dd07923a816c7963b2fc3 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:06 -0500 Subject: [PATCH 04/73] dt-bindings: mailbox: apple: add the secure RTBuddy port A coprocessor can have a second mailbox for its secure side, the ADT's iop,secure-rtbuddy-proxy. It has the registers of the AOP setup port, and its interrupts aren't described yet, so its consumer polls it. Add its compatible, and require interrupts for every mailbox but this one. Signed-off-by: Diljit Singh --- .../bindings/mailbox/apple,mailbox.yaml | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index add726470307d..cb70ea20b8872 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -76,6 +76,14 @@ properties: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox + - description: + A coprocessor's secure RTBuddy port, the ADT's + iop,secure-rtbuddy-proxy. It is polled, with no interrupts described. + items: + - enum: + - apple,t8140-secure-rtbuddy-proxy + - const: apple,secure-rtbuddy-proxy + - description: An older ASC mailbox interface found on T2 and A11 that is also used for the NVMe coprocessor and the system management @@ -134,6 +142,16 @@ properties: maxItems: 2 allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,secure-rtbuddy-proxy + then: + required: + - interrupts + - interrupt-names - if: properties: compatible: @@ -212,8 +230,6 @@ allOf: required: - compatible - reg - - interrupts - - interrupt-names - "#mbox-cells" additionalProperties: false From cfb1eb41aa747a5f7f98bc7c61da989ae880ff44 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:06 -0500 Subject: [PATCH 05/73] dt-bindings: iommu: apple,dart: describe a DART's state across a deep sleep A sleep that powers the SoC down resets every DART, and some of their state comes from the boot loader, not from Linux: - apple,dapf-ranges and apple,dapf-r20: the address filter the loader programs, to be written back before the consumer's firmware starts. - A GAPF slice in place of the DAPF registers: the part of the fabric filter that guards a coprocessor CPU's power state, cleared to power that CPU down. - apple,sleep-prepare, apple,sleep-restore and apple,sleep-restore-late: register writes around a sleep for a DART whose consumer gates itself and saves and restores it in its own order. - apple,resume-tunable: the loader's tunables for a DART whose loader has already applied them, to be written back where a power-off lost them. Signed-off-by: Diljit Singh --- .../devicetree/bindings/iommu/apple,dart.yaml | 66 ++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 9ba41ab9806d2..fe8ee9234b718 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -78,13 +78,15 @@ properties: - description: DART registers - description: Address filter (DAPF) registers, which restrict the physical - addresses the DART lets its masters reach + addresses the DART lets its masters reach, or the slice of the + fabric filter (GAPF) that protects the power state of the + coprocessor CPU behind this DART, cleared to power that CPU down reg-names: minItems: 1 items: - const: dart - - enum: [dapf, auxiliary] + - enum: [dapf, gapf, auxiliary] interrupts: maxItems: 1 @@ -116,6 +118,54 @@ properties: may select a firmware-visible alias of the translation aperture. A consumer may further specify its own aperture in its binding. + apple,dapf-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-matrix + items: + items: + - description: First address of a range + - description: Last address of a range + description: + The address filter the loader programs into the dapf registers, one + entry per range. A sleep that reaches the hardware clears it, and the + DART writes it back before its consumer's firmware starts again. + + apple,dapf-r20: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The last word of each filter entry, one per range in apple,dapf-ranges. + 1 where absent. + + apple,sleep-prepare: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes before a consumer that gates itself is told about a + firmware-driven system suspend. Needs apple,sleep-restore. + + apple,sleep-restore: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after such a suspend, before the translations are put + back. With this, the consumer saves and restores the DART around the + suspend, as the order relative to its firmware's start matters. + + apple,sleep-restore-late: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after the translations are back and before the streams + are enabled. + apple,dma-offset: $ref: /schemas/types.yaml#/definitions/uint64 description: > @@ -144,6 +194,18 @@ properties: Tree by the bootloader and required to initialize dynamically powered DART instances. + apple,resume-tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + The same tuples for a DART whose bootloader has already applied them, + written back where a power-off lost them, before its consumer's + firmware starts again. + apple,bypass-sids: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 From 3279deb3c2836a967dd25d182c10f60f45c044d4 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:06 -0500 Subject: [PATCH 06/73] dt-bindings: pci: apple,pcie: describe T8140's shared PCIe block ANS's NAND link runs on the shared PCIe block whether or not the PCIe host is bound, and after a deep sleep that block has to be brought back from power-off with the boot loader's settings. On T8140 the host node has five more register blocks (the AXI bridge, the CIO3 PLL, the clock generator, the PHY and its IP block) and the loader publishes six tunables for them, the PHY's PLL and lane tables among them. Allow them, for T8140 only. Signed-off-by: Diljit Singh --- .../devicetree/bindings/pci/apple,pcie.yaml | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 245231ecfc3a9..ad4a2abaec936 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -98,11 +98,11 @@ properties: reg: minItems: 3 - maxItems: 11 + maxItems: 13 reg-names: minItems: 3 - maxItems: 11 + maxItems: 13 ranges: minItems: 2 @@ -169,6 +169,30 @@ properties: $ref: '#/$defs/tunable' description: Initialization tuples for the oe-fabric register region. + apple,tunable-axi2af: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's AXI bridge. + + apple,tunable-cio3pll: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's PLL. + + apple,tunable-clkgen: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's clock generator. + + apple,tunable-phy: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block. + + apple,tunable-phy-ip-pll: + $ref: '#/$defs/tunable' + description: PLL settings of the shared PHY, relative to its phy-ip registers. + + apple,tunable-phy-ip-auspma: + $ref: '#/$defs/tunable' + description: Lane settings of the shared PHY, relative to its phy-ip registers. + apple,piodma: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -367,8 +391,9 @@ allOf: properties: reg: minItems: 8 - maxItems: 8 + maxItems: 13 reg-names: + minItems: 8 items: - const: config - const: rc @@ -378,6 +403,11 @@ allOf: - const: phy0 - const: phy1 - const: phy2 + - const: axi2af + - const: cio3pll + - const: clkgen + - const: phy-block + - const: phy-ip interrupts: maxItems: 3 else: From a7a0eebc3b456e2ffa62fc6ae78667163cdbd3f0 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 13:22:02 -0500 Subject: [PATCH 07/73] dt-bindings: display: add the Apple T8140 display power words A T8140 display coprocessor's driver writes three words around a restart of its firmware, as macOS does: the display PMC word that counts the powered DCPs, and the power manager's power state and power gate words. They belong to the power manager and the display block, so no driver binds the node that holds them. A DCP names it in its apple,display-power property. Describe the node, so it has a compatible, and the PMC word's address as its own rather than the power manager's. Signed-off-by: Diljit Singh --- .../display/apple,t8140-display-power.yaml | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml diff --git a/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml new file mode 100644 index 0000000000000..41bd5586820f2 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,t8140-display-power.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 display power words + +maintainers: + - Diljit Singh + +description: + The words a display coprocessor's driver writes to power its domains off + and on around a restart of its firmware, as macOS does. A display + coprocessor names this node in its apple,display-power property. The words + belong to the power manager and the display block, so no driver binds it. + +properties: + compatible: + const: apple,t8140-display-power + + reg: + items: + - description: the display PMC word that counts the powered coprocessors + - description: the power manager's power state words + - description: the power manager's power gate words + + reg-names: + items: + - const: pmc-dcp + - const: ps + - const: pwrgate + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + }; From a3fc59fe05f3e1e357ebb434e78b484f1cb7ffe9 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:06 -0500 Subject: [PATCH 08/73] dt-bindings: display: apple,dcp: add the secure port and display power The DCP's firmware takes an IOP idle only with its secure RTBuddy port answered, so the driver needs that second mailbox. Allow it as "secure". To restart its firmware after a sleep, the driver writes the display's power words itself, in the order macOS does. apple,display-power points at the node whose ps, pwrgate and pmc-dcp registers hold them, such as an apple,t8140-display-power node. Signed-off-by: Diljit Singh --- .../devicetree/bindings/display/apple,dcp.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 3db18e058dce3..bca94e7453be9 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -78,11 +78,14 @@ properties: description: Optional host display-engine interrupt sources. mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 mbox-names: + minItems: 1 items: - const: mbox + - const: secure iommus: maxItems: 1 @@ -123,6 +126,13 @@ properties: items: pattern: '^(dp-phy|typec[0-9]+)$' + apple,display-power: + $ref: /schemas/types.yaml#/definitions/phandle + description: + A node whose ps, pwrgate and pmc-dcp registers hold the display power + words this coprocessor's driver writes around a firmware restart, such + as an apple,t8140-display-power node. + apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array items: From e8c95e7575d6161817c9f84d43c454ee40c85ed3 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:07 -0500 Subject: [PATCH 09/73] dt-bindings: dma: apple,sio: name T8140's register blocks T8140's SIO is stopped for a sleep and started again after, and the sleep controller finds its coprocessor registers by name. Name T8140's three blocks: the ASC registers before the mailbox window ("asc"), the auxiliary ASC window, and the ASC registers after the mailbox window. Signed-off-by: Diljit Singh --- Documentation/devicetree/bindings/dma/apple,sio.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index ad512ac8491df..b40cca489279f 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -82,6 +82,11 @@ allOf: reg: minItems: 3 maxItems: 3 + reg-names: + items: + - const: asc + - const: auxiliary + - const: asc-tail else: if: properties: @@ -137,6 +142,10 @@ properties: status: true + reg-names: + minItems: 1 + maxItems: 3 + '#dma-cells': const: 1 description: From d3e07c8e1a0cf8ed53400aca01a1bea87f2abcd6 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:07 -0500 Subject: [PATCH 10/73] dt-bindings: gpu: apple,agx: add the PMP report entry The PMP has to know when the GPU is in use. Around a sleep the GPU's driver withdraws that request before it lowers the GPU's power state, and makes it again after raising it, as macOS does. Let the tree name the PMP report entry that carries it. Signed-off-by: Diljit Singh --- Documentation/devicetree/bindings/gpu/apple,agx.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 202544c07b0bb..a33aaedfd90ef 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -95,6 +95,13 @@ properties: GFX supplier power cannot represent GPU work on firmware that keeps its coprocessor domain powered while the cores are idle. + apple,pmp-report-entry: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + PMP report entry that tells the PMP the GPU is in use. The driver withdraws + this request before it lowers the GPU's power state for a system sleep, and + makes it again after raising it. + apple,idle-off-delay-ms: minimum: 2 maximum: 1000 From b25de3eff1b8354cde2a6176df6bd63da2d48fc0 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:07 -0500 Subject: [PATCH 11/73] dt-bindings: usb: apple,sn201202x: describe sleep, debug link and USB wake Three things a port controller's driver does for a firmware-driven system sleep, where the board asks for them: - apple,sleep-ssps: tell the controller the system's power state, S3 before the sleep and S0 after, as macOS does. - apple,debug-lock-key: the board's key that unlocks the controller's debug mode, to rearm the debug link after a sleep ended it. - apple,usb-remote-wake: this controller serves the USB controller's root port, and watches its USB2 lines for a remote wakeup during the sleep. Signed-off-by: Diljit Singh --- .../bindings/usb/apple,sn201202x.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index c2d2eb1068ea8..34ffb6d38b9fc 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -41,6 +41,24 @@ properties: wakeup-source: true + apple,sleep-ssps: + type: boolean + description: + Tell the controller the system's power state (SSPS) around a + firmware-driven suspend, S3 before it and S0 after. + + apple,debug-lock-key: + $ref: /schemas/types.yaml#/definitions/string + description: + The four-character key that unlocks the controller's debug mode, for + rearming the debug link after a sleep ended it. + + apple,usb-remote-wake: + type: boolean + description: + This controller serves the USB controller's root port, and watches its + USB2 lines for a remote wakeup during a firmware-driven system sleep. + connector: $ref: /schemas/connector/usb-connector.yaml# From 934d745e8c889c9ff346d2307bbe1e362d8d9417 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:07 -0500 Subject: [PATCH 12/73] dt-bindings: phy: apple,atcphy: add T8140's USB power control block The block next to the PIPE handler takes the USB controller's core in and out of hibernation: a power state request, a status with SLEEP, and event, enable and clear words for VVDD_USB_OFF, D0_ON and two wake events. macOS's controller driver uses it at every system sleep. In Linux the PHY drives that controller's reset and suspends after it, so give the block to the PHY, as an optional seventh window. Signed-off-by: Diljit Singh --- Documentation/devicetree/bindings/phy/apple,atcphy.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 5364f6dd8871b..8c84a8f3f369b 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -125,6 +125,10 @@ allOf: - description: USB2 PHY specific controls - description: USB3 PIPE interface controls - description: Secondary eUSB2 PHY register bank (T8140) + - description: + Power and hibernation control of the USB controller the PHY + serves (T8140). The PHY drives that controller's reset and + takes it through hibernation for a system sleep. reg-names: minItems: 5 items: @@ -134,6 +138,7 @@ allOf: - const: usb2phy - const: pipehandler - const: usb2phy-reg + - const: pwrctl required: - power-domains - if: From 806fb9758c81ca44b02ef152a883b095f0158444 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:08 -0500 Subject: [PATCH 13/73] dt-bindings: sound: apple,t8140-aop-audio: add the mic array's supply The microphone array has a power rail of its own, which macOS switches on for each capture and off after it. Let the tree name it. Signed-off-by: Diljit Singh --- .../devicetree/bindings/sound/apple,t8140-aop-audio.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index b023080e48c7f..7329ac57a538f 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -82,6 +82,10 @@ properties: Chassis identifier used to name the sound card, matching the apple,aop-audio convention. + mic-supply: + description: + The microphone array's power rail, switched on around each capture. + apple,machine-kind: $ref: /schemas/types.yaml#/definitions/string description: From b0e4c03da6186a13d6939aef77fc631c585f8579 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:09:37 -0500 Subject: [PATCH 14/73] PM: suspend: Go back to sleep from a dark wake A platform can find that what woke the system needs nothing from user space. On Apple laptops the SMC wakes the SoC for the lid closing as well as for it opening. macOS takes the closing as a dark wake: every driver wakes as usual, the display stays off, and the power management root domain puts the system back to sleep once it sees the lid closed. In macOS's 25G83 kernel, read statically, AppleSMC sends kIOPMClamshellClosed, and the dark wake policy sleeps with reason Maintenance when the clamshell is closed and the machine isn't in desktop mode on AC. suspend_again() can't do the same. It goes back to sleep after the early resume only, so drivers that do their sleep work in the normal suspend and resume phases are left half way. Add a dark wake flag for the platform to set during the noirq resume. Devices still resume fully but may leave out what the user would see. Once dpm_resume_end() is done, enter_state() runs suspend_devices_and_enter() again with user space still frozen, so user space sees one sleep. Only a wake from the sleep state can be dark, and each pass starts with the flags cleared. Signed-off-by: Diljit Singh --- include/linux/suspend.h | 29 +++++++++++++++++++++++++++++ kernel/power/suspend.c | 12 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index b02876f1ae38a..365437ce2cc02 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -159,6 +159,7 @@ extern unsigned int pm_suspend_global_flags; #define PM_SUSPEND_FLAG_FW_SUSPEND BIT(0) #define PM_SUSPEND_FLAG_FW_RESUME BIT(1) #define PM_SUSPEND_FLAG_NO_PLATFORM BIT(2) +#define PM_SUSPEND_FLAG_DARK_WAKE BIT(3) static inline void pm_suspend_clear_flags(void) { @@ -180,6 +181,11 @@ static inline void pm_set_suspend_no_platform(void) pm_suspend_global_flags |= PM_SUSPEND_FLAG_NO_PLATFORM; } +static inline void pm_set_dark_wake(void) +{ + pm_suspend_global_flags |= PM_SUSPEND_FLAG_DARK_WAKE; +} + /** * pm_suspend_via_firmware - Check if platform firmware will suspend the system. * @@ -237,6 +243,27 @@ static inline bool pm_suspend_no_platform(void) return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_NO_PLATFORM); } +/** + * pm_dark_wake - Check if the system woke up for nothing user space has to see. + * + * To be called during system-wide power management transitions from sleep + * states. + * + * Return 'true' if the platform found that the event that woke up the system + * needs nothing from user space (for example, a laptop lid closing while the + * system slept). Every device then resumes as usual, and the system goes back + * to the same sleep state with user space still frozen. A device should not do + * what the user would notice, such as lighting a display, during such a wake. + * + * Only a wake from the sleep state can be dark. A device that had to leave out + * part of its resume finishes it in a later resume that is not dark, or in + * ->complete() if the next suspend fails before suspending it. + */ +static inline bool pm_dark_wake(void) +{ + return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_DARK_WAKE); +} + /* Suspend-to-idle state machnine. */ enum s2idle_states { S2IDLE_STATE_NONE, /* Not suspended/suspending. */ @@ -287,6 +314,8 @@ static inline void pm_set_resume_via_firmware(void) {} static inline bool pm_suspend_via_firmware(void) { return false; } static inline bool pm_resume_via_firmware(void) { return false; } static inline bool pm_suspend_no_platform(void) { return false; } +static inline void pm_set_dark_wake(void) {} +static inline bool pm_dark_wake(void) { return false; } static inline bool pm_suspend_default_s2idle(void) { return false; } static inline void suspend_set_ops(const struct platform_suspend_ops *ops) {} diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 57c44268698f7..34253036bbbd9 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -532,6 +532,9 @@ int suspend_devices_and_enter(suspend_state_t state) do { error = suspend_enter(state, &wakeup); } while (!error && !wakeup && platform_suspend_again(state)); + /* Only a wake from the sleep state can be dark. */ + if (error) + pm_suspend_global_flags &= ~PM_SUSPEND_FLAG_DARK_WAKE; Resume_devices: suspend_test_start(); @@ -616,6 +619,15 @@ static int enter_state(suspend_state_t state) trace_suspend_resume(TPS("suspend_enter"), state, false); pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); error = suspend_devices_and_enter(state); + /* + * Every device has resumed, but nothing woke the system that user space + * needs to see: go back to sleep with user space still frozen. + */ + while (!error && pm_dark_wake()) { + pr_info("dark wake: suspending again (%s)\n", mem_sleep_labels[state]); + pm_suspend_clear_flags(); + error = suspend_devices_and_enter(state); + } Finish: events_check_enabled = false; From b90ca804f476c56de3917e94e996bca5335fe8a1 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:08 -0500 Subject: [PATCH 15/73] arm64: restore Apple CPU state after a firmware-driven suspend The enable callbacks of these features run through hotplug, and a context switch only writes ACTLR when the thread's value changes. A firmware-driven system suspend really resets the boot CPU, which comes back through cpu_resume and not through hotplug, so it keeps reset values with nothing to correct them. With AMX, that can leave EL0 access in the wrong state for the current thread. Restore them in __cpu_suspend_exit when pm_resume_via_firmware() says the CPU was reset. That's only set during such a system suspend, and the restore does nothing on a CPU without these features, so other machines don't change. Signed-off-by: Diljit Singh --- arch/arm64/include/asm/cpufeature.h | 1 + arch/arm64/kernel/cpufeature_impdef.c | 18 ++++++++++++++++++ arch/arm64/kernel/suspend.c | 8 ++++++++ 3 files changed, 27 insertions(+) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 0a421a88b3012..65982bd36a11f 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -1089,6 +1089,7 @@ static inline bool cpu_has_lpa2(void) } void __init init_cpucap_indirect_list_impdef(void); +void arm64_impdef_cpu_reset_restore(void); void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c index 4078f95146c32..f08e210bcb813 100644 --- a/arch/arm64/kernel/cpufeature_impdef.c +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -363,6 +364,23 @@ static const struct arm64_cpu_capabilities arm64_impdef_features[] = { {}, }; +/* + * Restore what these features keep per CPU, on a CPU that was reset without + * going through CPU hotplug: its enable callbacks do not run again, and a + * context switch only writes ACTLR when the thread's value changes. + */ +void arm64_impdef_cpu_reset_restore(void) +{ +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + write_sysreg(current->thread.actlr, actlr_el1); +#endif +#ifdef CONFIG_ARM64_APPLE_AMX + if (cpus_have_cap(ARM64_HAS_AMX_APPLE)) + cpu_disable_el0_amx(NULL); +#endif +} + void __init init_cpucap_indirect_list_impdef(void) { init_cpucap_indirect_list_from_array(arm64_impdef_features); diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index c41724a40b756..2a69c79b4ed91 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,13 @@ void notrace __cpu_suspend_exit(void) /* Restore additional feature-specific configuration */ ptrauth_suspend_exit(); + + /* + * A firmware-driven system suspend really reset this CPU. Nothing else on + * this path restores implementation-defined state. + */ + if (pm_resume_via_firmware()) + arm64_impdef_cpu_reset_restore(); } /* From fa4e8a93d6396942de76c99f141d309ef8adae32 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:09 -0500 Subject: [PATCH 16/73] arm64: smp: add a spin table whose CPUs can park and power off Plain spin-table has no way out of the kernel, so disabling the non-boot CPUs for system sleep fails with -EOPNOTSUPP. That rules out suspend-to-RAM on a kernel that a boot monitor enters directly at EL2 with spin-table, with no PSCI below it. Where iBoot locks a core's reset vector to the boot object, a core that powers off and is started again goes back through the boot object. If that keeps the core's release address, the core returns to secondary_holding_pen on its own, and hotplug needs no more from it. Such a boot object says so once, with /chosen/aurora,s2r-mailbox, the mailbox the boot core resumes through. With that property present, a CPU whose enable method is "spin-table" gets the parking ops added here, so the device tree keeps the method every kernel knows, and a kernel without this change ignores the property and stays on plain spin-table. A boot object can also name the "aurora,rvbar-spin-table" method for a CPU. A secondary that takes its deep WFI at hotplug stops the whole machine within seconds on T8140, and macOS takes that WFI only after the sleep commit. So a CPU leaving the kernel parks here, still coherent, with interrupts masked. For a suspend it writes its stop bit at the start of the last mile and takes the deep WFI after the commit; otherwise it goes back through secondary_holding_pen, staying at EL2. Every core loses its power in a real sleep, so after the wake a core still parked, one that never saw its stop, is taken as off and started through the platform's on step, which only writes its start bits. Measured on a J700, though, a sleep committed with a core that missed its stop doesn't come back at all, so the sleep has to refuse for such a core while it still can. The sleep controller's hold does that. The SoC-specific stop, off and on steps come from a platform driver through spin_table_set_power_ops(). Until one registers, hotplug fails as on plain spin-table, and so does a kexec load: the parking ops have a cpu_die, so cpus_are_stuck_in_kernel() no longer catches them, and the kexec would otherwise fail at machine_kexec() with the CPUs still online. Without KEXEC_CORE (for cpu_soft_restart) hotplug is refused too. Other machines keep the ops they have, since only that property or the explicit method selects these. Signed-off-by: Diljit Singh --- arch/arm64/include/asm/cpu_ops.h | 44 ++++++ arch/arm64/kernel/cpu_ops.c | 19 +++ arch/arm64/kernel/machine_kexec.c | 3 +- arch/arm64/kernel/smp_spin_table.c | 206 +++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index a444c8915e886..1bebfa1c12efc 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -7,6 +7,7 @@ #include #include +#include /** * struct cpu_operations - Callback operations for hotplugging CPUs. @@ -46,6 +47,49 @@ struct cpu_operations { #endif }; +/** + * struct spin_table_power_ops - Platform control of parking spin-table CPUs + * (smp_spin_table.c). + * + * @cpu_stop: Tell the hardware the calling CPU is stopping. The CPU keeps + * running, and stays coherent, until @cpu_off. + * @cpu_off: Power off the calling CPU, which must not return. + * @cpu_on: Start a CPU that @cpu_off powered off, at its reset vector. + * + * Each takes the CPU's MPIDR hardware ID. + */ +struct spin_table_power_ops { + void (*cpu_stop)(u64 mpidr); + void (*cpu_off)(u64 mpidr); + int (*cpu_on)(u64 mpidr); +}; + +#ifdef CONFIG_HOTPLUG_CPU +int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +int spin_table_stop_parked(void); +void spin_table_release_stopped(void); +void spin_table_stop_finished(void); +bool spin_table_cpus_stuck(void); +#else +static inline int spin_table_stop_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_release_stopped(void) {} +static inline void spin_table_stop_finished(void) {} + +static inline int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return -EOPNOTSUPP; +} + +static inline bool spin_table_cpus_stuck(void) +{ + return false; +} +#endif + int __init init_cpu_ops(int cpu); extern const struct cpu_operations *get_cpu_ops(int cpu); diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index e133011f64b5b..7e88e4b6ca501 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -15,6 +15,9 @@ #include extern const struct cpu_operations smp_spin_table_ops; +#ifdef CONFIG_HOTPLUG_CPU +extern const struct cpu_operations smp_rvbar_spin_table_ops; +#endif #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL extern const struct cpu_operations acpi_parking_protocol_ops; #endif @@ -24,6 +27,9 @@ static const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; static const struct cpu_operations *const dt_supported_cpu_ops[] __initconst = { &smp_spin_table_ops, +#ifdef CONFIG_HOTPLUG_CPU + &smp_rvbar_spin_table_ops, +#endif &cpu_psci_ops, NULL, }; @@ -109,6 +115,19 @@ int __init init_cpu_ops(int cpu) return -EOPNOTSUPP; } +#ifdef CONFIG_HOTPLUG_CPU + /* + * A boot object that sends a reset CPU back through its release address + * says so with /chosen/aurora,s2r-mailbox. Its spin-table CPUs can then + * leave the kernel, through the parking spin table. + */ + if (cpu_ops[cpu] == &smp_spin_table_ops && + of_property_present(of_chosen, "aurora,s2r-mailbox")) { + cpu_ops[cpu] = &smp_rvbar_spin_table_ops; + pr_info_once("spin-table CPUs can park: the boot object brings them back\n"); + } +#endif + return 0; } diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 9c4a407222ea1..4fb4e7bb1e39e 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -64,7 +64,8 @@ int machine_kexec_prepare(struct kimage *kimage) return -EOPNOTSUPP; } - if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { + if (kimage->type != KEXEC_TYPE_CRASH && + (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck())) { pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); return -EBUSY; } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 49029eace3ad9..f6590af9d79df 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include extern void secondary_holding_pen(void); @@ -125,3 +127,207 @@ const struct cpu_operations smp_spin_table_ops = { .cpu_prepare = smp_spin_table_cpu_prepare, .cpu_boot = smp_spin_table_cpu_boot, }; + +/* + * A spin table whose CPUs can leave the kernel. + * + * Plain spin-table has no way out of the kernel, so disabling the non-boot + * CPUs for system sleep fails with -EOPNOTSUPP. Some boot monitors leave a + * CPU's reset vector locked to their own image and, after a reset, send the + * CPU back through the same release address. Such a monitor publishes + * /chosen/aurora,s2r-mailbox, and its "spin-table" CPUs then get these ops + * (cpu_ops.c), so the device tree keeps the method every kernel knows. One can + * also name "aurora,rvbar-spin-table" as a CPU's method. + * + * A CPU that really powers off can stop the rest of the system on such a chip, + * so a CPU leaving the kernel parks here instead: still coherent, interrupts + * masked. From there it goes back through secondary_holding_pen, or, when the + * platform suspends the system, it stops and takes its last WFI after the commit. + * It is then started again at its reset vector. Stopping and starting a CPU is + * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug + * fails cleanly, as on plain spin-table. + */ +#ifdef CONFIG_HOTPLUG_CPU +static const struct spin_table_power_ops *spin_table_power __read_mostly; +static struct cpumask spin_table_cpus_parked; +static struct cpumask spin_table_cpus_unpark; +static struct cpumask spin_table_cpus_off; +static bool spin_table_stop_requested; +static bool spin_table_stop_released; + +#define SPIN_TABLE_WAIT_US 100000 + +int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; +} + +/* + * For a caller about to suspend the system, interrupts off: parked CPUs stop and + * wait for spin_table_release_stopped(). Returns -ETIMEDOUT if any didn't stop. + */ +int spin_table_stop_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_stop_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (cpumask_empty(&spin_table_cpus_parked)) + break; + udelay(1); + } + WRITE_ONCE(spin_table_stop_requested, false); + + return cpumask_empty(&spin_table_cpus_parked) ? 0 : -ETIMEDOUT; +} + +/* After the commit: the stopped CPUs take their last WFI. */ +void spin_table_release_stopped(void) +{ + WRITE_ONCE(spin_table_stop_released, true); + dsb(ish); + sev(); +} + +/* + * Back from a suspend, before starting any CPU. Every CPU lost its power with the + * SoC, so one that never saw the stop is started at its reset vector too. + */ +void spin_table_stop_finished(void) +{ + WRITE_ONCE(spin_table_stop_released, false); + cpumask_or(&spin_table_cpus_off, &spin_table_cpus_off, &spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_unpark); +} + +static int smp_rvbar_spin_table_cpu_boot(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + int ret; + + /* The CPU may run straight into the pen: release it before it starts. */ + write_pen_release(mpidr); + + if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { + ret = spin_table_power->cpu_on(mpidr); + if (ret) { + cpumask_set_cpu(cpu, &spin_table_cpus_off); + return ret; + } + } else if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) { + cpumask_set_cpu(cpu, &spin_table_cpus_unpark); + } + + dsb(ish); + sev(); + + return 0; +} + +static bool smp_rvbar_spin_table_cpu_can_disable(unsigned int cpu) +{ + return cpu != 0; +} + +static int smp_rvbar_spin_table_cpu_disable(unsigned int cpu) +{ + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + return -EOPNOTSUPP; + + return READ_ONCE(spin_table_power) ? 0 : -EOPNOTSUPP; +} + +#ifdef CONFIG_KEXEC_CORE +static void smp_rvbar_spin_table_restart(void) +{ + typeof(cpu_soft_restart) *restart; + + cpu_install_idmap(); + restart = (void *)__pa_symbol(cpu_soft_restart); + restart(0, __pa_symbol(secondary_holding_pen), 0, 0, 0); +} +#else +/* Never reached: cpu_die() doesn't leave its loop without KEXEC_CORE. */ +static void smp_rvbar_spin_table_restart(void) +{ +} +#endif + +static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + + /* The early-boot and crash paths get here without hotplug support. */ + if (!READ_ONCE(spin_table_power) || !IS_ENABLED(CONFIG_KEXEC_CORE)) { + while (1) + wfe(); + } + + cpumask_set_cpu(cpu, &spin_table_cpus_parked); + + while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { + if (READ_ONCE(spin_table_stop_requested)) { + spin_table_power->cpu_stop(mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_off); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + while (!READ_ONCE(spin_table_stop_released)) + wfe(); + spin_table_power->cpu_off(mpidr); + } + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_unpark); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + smp_rvbar_spin_table_restart(); +} + +static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) +{ + int us; + + if (!READ_ONCE(spin_table_power)) + return 0; + + for (us = 0; us < SPIN_TABLE_WAIT_US; us += 10) { + if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) + return 0; + udelay(10); + } + + return -ETIMEDOUT; +} + +const struct cpu_operations smp_rvbar_spin_table_ops = { + .name = "aurora,rvbar-spin-table", + .cpu_init = smp_spin_table_cpu_init, + .cpu_prepare = smp_spin_table_cpu_prepare, + .cpu_boot = smp_rvbar_spin_table_cpu_boot, + .cpu_can_disable = smp_rvbar_spin_table_cpu_can_disable, + .cpu_disable = smp_rvbar_spin_table_cpu_disable, + .cpu_die = smp_rvbar_spin_table_cpu_die, + .cpu_kill = smp_rvbar_spin_table_cpu_kill, +}; + +/* + * Parking CPUs leave the kernel only through a platform's power ops. Until one + * registers, kexec has to take them as stuck, as on plain spin-table. + */ +bool spin_table_cpus_stuck(void) +{ + unsigned int cpu; + + if (READ_ONCE(spin_table_power)) + return false; + for_each_possible_cpu(cpu) + if (get_cpu_ops(cpu) == &smp_rvbar_spin_table_ops) + return true; + return false; +} +#endif /* CONFIG_HOTPLUG_CPU */ From 67e70ac4765e17f228c92ce2ed4aba3705742f15 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:09 -0500 Subject: [PATCH 17/73] irqchip/apple-aic: save and restore T8140 state across a firmware suspend T8140's firmware-driven suspend powers the controller down, and the AIC driver has no suspend code at all. Mask every line before the commit so that nothing can end the final WFI, and put the controller's state back after a real power-off. Unmask from the IRQ core's books, since reading the mask registers has not been shown to work. The boot CPU doesn't go through hotplug on the way back, so run its per-CPU setup from resume too, keeping the FIQ state of its existing users. The sleep engine also waits for every CPU to stop, and a raised line ends WFI even with interrupts off at the CPU. macOS masks every line right after the commit. Export that step, its undo and an event read to the sleep controller, which runs them with one CPU up and interrupts off. The syscore ops are only registered on T8140 and only act on a firmware-driven suspend, so nothing changes on other machines or in s2idle. Signed-off-by: Diljit Singh --- drivers/irqchip/irq-apple-aic.c | 181 ++++++++++++++++++++++++++ include/linux/irqchip/irq-apple-aic.h | 17 +++ 2 files changed, 198 insertions(+) create mode 100644 include/linux/irqchip/irq-apple-aic.h diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 1125008130ef4..1afc73dd62617 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -388,6 +391,8 @@ struct aic_irq_chip { int nr_die; int max_die; + u32 *sleep_masks; + struct aic_info info; }; @@ -410,6 +415,47 @@ static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) writel_relaxed(val, ic->base + reg); } +/* + * A sleep that a firmware engine finishes waits for the CPUs to stop, and a raised + * line ends WFI even with interrupts off. Every line is masked for that last step. + */ +void aic_sleep_mask_all(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, reg, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++) { + reg = ic->info.mask_set + off + i * 4; + *saved++ = aic_ic_read(ic, reg); + aic_ic_write(ic, reg, U32_MAX); + } + } +} + +void aic_sleep_restore(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++, saved++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, *saved); + aic_ic_write(ic, ic->info.mask_clr + off + i * 4, ~*saved); + } + } +} + +/* Takes the next event, which also masks its line. */ +u32 aic_sleep_ack(void) +{ + return readl(aic_irqc->event + aic_irqc->info.event); +} + /* * IRQ irqchip */ @@ -1083,6 +1129,134 @@ static void build_fiq_affinity(struct aic_irq_chip *ic, struct device_node *aff) } } +#ifdef CONFIG_PM_SLEEP +/* + * T8140's firmware-driven suspend powers the controller down and back up. How + * much it keeps isn't measured, and macOS saves and restores it in software. So + * mask everything before the commit, so that no line can end the final WFI, and + * put it all back after. Linux never writes IRQ_CFG or these global registers + * outside of config's enable bit, so they hold what the bootloader set. + */ +static struct { + u32 config; + u32 timeout; + u32 cluster_prio; + u32 *irq_cfg; + bool saved; +} aic_sleep; + +static int aic_syscore_suspend(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!pm_suspend_via_firmware()) + return 0; + + aic_sleep.config = aic_ic_read(ic, AIC2_CONFIG); + aic_sleep.timeout = aic_ic_read(ic, AIC2_TIMEOUT); + aic_sleep.cluster_prio = aic_ic_read(ic, AIC2_CLUSTER_PRIO); + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < ic->nr_irq; i++) + aic_sleep.irq_cfg[die * ic->nr_irq + i] = + aic_ic_read(ic, ic->info.irq_cfg + off + i * 4); + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + off += ic->info.die_stride; + } + aic_sleep.saved = true; + + return 0; +} + +/* The boot CPU doesn't pass through hotplug, so aic_init_cpu() doesn't rerun. */ +static void aic_resume_boot_cpu(void) +{ + u32 fiq = __this_cpu_read(aic_fiq_unmasked); + + aic_init_cpu(smp_processor_id()); + /* + * aic_init_cpu() is for a CPU with no FIQ users yet. This one's kept theirs: + * a timer left marked masked here would never be unmasked at its next EOI. + */ + __this_cpu_write(aic_fiq_unmasked, fiq); + if (is_kernel_in_hyp_mode()) { + if (fiq & BIT(AIC_TMR_EL02_PHYS)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_P, false); + if (fiq & BIT(AIC_TMR_EL02_VIRT)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V, false); + isb(); + } +} + +static void aic_syscore_resume(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!aic_sleep.saved) + return; + aic_sleep.saved = false; + + /* Otherwise the suspend was refused, and the controller stayed powered. */ + if (pm_resume_via_firmware()) { + aic_ic_write(ic, AIC2_TIMEOUT, aic_sleep.timeout); + aic_ic_write(ic, AIC2_CLUSTER_PRIO, aic_sleep.cluster_prio); + aic_ic_write(ic, AIC2_CONFIG, aic_sleep.config); + aic_resume_boot_cpu(); + } + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + aic_ic_write(ic, ic->info.sw_clr + off + i * 4, U32_MAX); + } + for (i = 0; i < ic->nr_irq; i++) { + u32 saved = aic_sleep.irq_cfg[die * ic->nr_irq + i]; + u32 reg = ic->info.irq_cfg + off + i * 4; + unsigned int virq; + + if (aic_ic_read(ic, reg) != saved) + aic_ic_write(ic, reg, saved); + + /* Restore from the IRQ core's books, not from the hardware's. */ + virq = irq_find_mapping(ic->hw_domain, AIC_IRQ_HWIRQ(die, i)); + if (virq && !irqd_irq_masked(irq_get_irq_data(virq))) + aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(i), + MASK_BIT(i)); + } + off += ic->info.die_stride; + } +} + +static const struct syscore_ops aic_syscore_ops = { + .suspend = aic_syscore_suspend, + .resume = aic_syscore_resume, +}; + +static struct syscore aic_syscore = { + .ops = &aic_syscore_ops, +}; + +static void __init aic_sleep_init(struct aic_irq_chip *irqc, struct device_node *node) +{ + /* The only Apple SoC with a suspend-to-RAM path. */ + if (!of_device_is_compatible(node, "apple,t8140-aic3")) + return; + + aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); + if (WARN_ON(!aic_sleep.irq_cfg)) + return; + + register_syscore(&aic_syscore); +} +#else +static void __init aic_sleep_init(struct aic_irq_chip *irqc, struct device_node *node) +{ +} +#endif + static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { int i, die; @@ -1168,6 +1342,11 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->info.die_stride = off - start_off; + irqc->sleep_masks = kcalloc(irqc->nr_die * (irqc->max_irq >> 5), + sizeof(*irqc->sleep_masks), GFP_KERNEL); + if (WARN_ON(!irqc->sleep_masks)) + goto err_unmap; + irqc->hw_domain = irq_domain_create_tree(of_fwnode_handle(node), &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) @@ -1219,6 +1398,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); + aic_sleep_init(irqc, node); + if (is_kernel_in_hyp_mode()) { struct irq_fwspec mi = { .fwnode = of_fwnode_handle(node), diff --git a/include/linux/irqchip/irq-apple-aic.h b/include/linux/irqchip/irq-apple-aic.h new file mode 100644 index 0000000000000..3c3781322a52b --- /dev/null +++ b/include/linux/irqchip/irq-apple-aic.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_IRQCHIP_IRQ_APPLE_AIC_H +#define _LINUX_IRQCHIP_IRQ_APPLE_AIC_H + +#include + +#ifdef CONFIG_APPLE_AIC +void aic_sleep_mask_all(void); +void aic_sleep_restore(void); +u32 aic_sleep_ack(void); +#else +static inline void aic_sleep_mask_all(void) { } +static inline void aic_sleep_restore(void) { } +static inline u32 aic_sleep_ack(void) { return 0; } +#endif + +#endif From c8e03faac4ad8d426f28ef56115bce1796184a3d Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:09 -0500 Subject: [PATCH 18/73] spmi: apple: quiet the response vector for a firmware suspend Every command asks for an interrupt on its response, and the driver leaves that vector enabled. On T8140 the wake logic watches this queue's line for the PMU's RTC alarm, so our own traffic could end up read as a wake at the commit. macOS clears the same enable bit shortly before it, measured under aura. Mask and clear the vector in the noirq phase of a firmware-driven suspend and poll for responses until resume. After a real power-off also write the vector enables back, since whether they survive hasn't been measured. Only that kind of suspend is affected, so s2idle and other machines see no change. Signed-off-by: Diljit Singh --- drivers/spmi/spmi-apple-controller.c | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index fbdb9ba0e7f2a..95678be5a9984 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -12,7 +12,9 @@ */ #include +#include #include +#include #include #include #include @@ -23,8 +25,10 @@ #include #include #include +#include #include #include +#include /* SPMI Controller Registers */ #define SPMI_STATUS_REG 0 @@ -84,6 +88,7 @@ struct apple_spmi { DECLARE_BITMAP(irq_mask_cache, SPMI_NUM_PERIPHERAL_IRQS); int irq; bool notify_irq; + bool notify_parked; bool prev_fail; }; @@ -468,6 +473,63 @@ static int apple_spmi_init_irq(struct platform_device *pdev, return 0; } +/* + * Every command asks for an interrupt on its response, on the vector this + * driver waits on. The system's wake logic watches this queue's interrupt line + * for the PMU's RTC alarm, so that vector has to be quiet and clear before a + * firmware-driven suspend, or the commit may read our own traffic as a wake. + * macOS clears the same enable bit at about the same point (measured under aura). + * Any command sent from here on polls for its response. + */ +static int apple_spmi_suspend_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + unsigned long flags; + + if (!spmi->notify_irq || !pm_suspend_via_firmware()) + return 0; + + guard(mutex)(&spmi->fifo_lock); + spmi->notify_irq = false; + spmi->notify_parked = true; + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + apple_spmi_irq_mask_raw(spmi, SPMI_IRQ_NOTIFY); + apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + + return 0; +} + +static int apple_spmi_resume_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + u32 words[SPMI_NUM_PERIPHERAL_IRQS / 32]; + unsigned long flags; + size_t i; + + if (!spmi->notify_parked) + return 0; + + guard(mutex)(&spmi->fifo_lock); + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + /* Whether the enables survive a real sleep has not been measured. */ + if (pm_resume_via_firmware()) { + bitmap_to_arr32(words, spmi->irq_mask_cache, SPMI_NUM_PERIPHERAL_IRQS); + for (i = 0; i < ARRAY_SIZE(words); i++) + writel(words[i], spmi->regs + SPMI_IRQ_MASK_BASE + i * 4); + } + apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); + apple_spmi_irq_unmask_raw(spmi, SPMI_IRQ_NOTIFY); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + spmi->notify_parked = false; + spmi->notify_irq = true; + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(apple_spmi_pm_ops, apple_spmi_suspend_noirq, + apple_spmi_resume_noirq); + static int apple_spmi_probe(struct platform_device *pdev) { struct apple_spmi *spmi; @@ -529,6 +591,7 @@ static struct platform_driver apple_spmi_driver = { .driver = { .name = "apple-spmi", .of_match_table = apple_spmi_match_table, + .pm = pm_sleep_ptr(&apple_spmi_pm_ops), }, }; module_platform_driver(apple_spmi_driver); From 0f9f95f9b8ebc29fb17f4993c36762e0e9e11b6c Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:10 -0500 Subject: [PATCH 19/73] rtc: apple-pmu: allow a firmware-driven suspend The driver refuses every sleep state but s2idle, because nothing routed the alarm as a wake source in any other. A firmware-driven suspend does that itself: on T8140 the alarm is wake source NUB.SPMI0.SW3, and macOS's RTC wakes come back through it with the same alarm registers (measured under aura). The suspend path clears the driver's refusal only when the platform has said it is driving the suspend, so other machines keep refusing. Signed-off-by: Diljit Singh --- drivers/rtc/rtc-apple-pmu.c | 9 ++++++++- tools/asahi/pmu-rtc-tests.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index d324234cf807b..5a5bc4e576722 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -382,7 +382,14 @@ static int apple_pmu_rtc_prepare(struct device *dev) { struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || rtc->io_failed) + if (rtc->io_failed) + return -EBUSY; + /* + * A firmware-driven suspend programs the wake sources itself, and the + * alarm stays armed in the PMU, which stays powered. Only a sleep state + * without either has no way to bring the alarm back. + */ + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !pm_suspend_via_firmware()) return -EBUSY; return 0; } diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 3c85238cc6343..6632caeb25fb3 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -64,6 +64,8 @@ struct nvmem_cell { u8 data[8]; size_t bytes; int error; }; struct regmap { u8 bytes[65536]; unsigned ctrl, counter, compare, mask, event, bit; int ops, fail; }; static int disabled, pm_suspend_target_state = PM_SUSPEND_TO_IDLE; +static bool firmware_suspend; +static bool pm_suspend_via_firmware(void) { return firmware_suspend; } static inline __attribute__((unused)) s64 sign_extend64(u64 v, int bit) { return (s64)(v << (63 - bit)) >> (63 - bit); } static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } static void mutex_lock(struct mutex *m) { (void)m; } @@ -244,6 +246,10 @@ reset(v); assert(!apple_pmu_rtc_prepare(&dev)); pm_suspend_target_state = 2; assert(apple_pmu_rtc_prepare(&dev) == -EBUSY); + /* A firmware-driven suspend programs the wake itself. */ + firmware_suspend = true; + assert(!apple_pmu_rtc_prepare(&dev)); + firmware_suspend = false; pm_suspend_target_state = PM_SUSPEND_TO_IDLE; } int main(void) { From 73600d2dd05e5ce162a4ca6346cbddaa99122169 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:10 -0500 Subject: [PATCH 20/73] pmdomain: apple: resync domains and PMC floors after a firmware suspend A firmware-driven suspend changes domain states behind genpd's back. macOS's wake record on T8140 shows AUTO_ENABLE gone from 22 domains and many others left clock gated: it raises the first group and lowers 30 of the second within 5 ms of a wake. genpd only powers a domain on if it turned it off, so a domain it left on stays wrong. On a resume from such a suspend, put each domain where genpd's books say it is. Follow the books rather than macOS's two lists, since the rule behind those lists hasn't been worked out. Firmware-owned domains stay untouched. Some domains need their driver's order on the way back. For the GPU's that is the target, the bridge, and only then automatic power management, and a resync would raise it with AUTO_ENABLE well before the GPU driver runs. Add apple,sleep-driver-owned for a domain whose driver lowers it before a sleep and brings it back afterwards; the resync leaves it alone. The PMC driver keeps which agents it enabled on each rail and the floors its domains hold, and writes neither again on its own. Restore them from the books in resume_early: a register that kept its value isn't written, and CLPC's sampling starts again only in resume, after this. Only pm_resume_via_firmware() runs any of this, so other paths are unchanged. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmc.c | 36 +++++++++++++++++++ drivers/pmdomain/apple/pmgr-pwrstate.c | 49 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 3765e9a775ff7..54c66b3539f0e 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -1385,6 +1386,40 @@ static const struct apple_pmc_hw apple_pmc_hw_t8152 = { .ap_agent = 3, }; +/* + * A system sleep that reached the hardware may have taken the enables and floors + * with it. Put back what the driver's books say: these are no-ops if they stayed. + */ +static int apple_pmc_resume_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + unsigned long flags; + unsigned int rail; + int ret = 0, err; + + if (!pm_resume_via_firmware() || !pmc) + return 0; + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails; rail++) { + err = apple_pmc_enable_write(pmc, rail); + ret = ret ?: err; + if (pmc->floor_rails & BIT(rail)) { + err = apple_pmc_floor_apply(pmc, rail); + ret = ret ?: err; + } + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + if (ret) + dev_err(dev, "state not restored after sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_pmc_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmc_resume_early) +}; + static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, @@ -1400,6 +1435,7 @@ static struct platform_driver apple_pmc_driver = { .driver = { .name = "apple-pmc", .of_match_table = apple_pmc_of_match, + .pm = pm_sleep_ptr(&apple_pmc_pm_ops), /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index b84bfc4432111..aeeb8434709d7 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,8 @@ struct apple_pmgr_ps { bool firmware_owned; /* apple,sleep-retained: kept on while the system sleeps */ bool sleep_retained; + /* apple,sleep-driver-owned: the driver behind it brings it back after a sleep */ + bool sleep_driver_owned; /* Never enable automatic power management (apple,no-auto-pm). */ bool no_auto_pm; bool inherited_on; @@ -360,6 +363,50 @@ static int apple_pmgr_ps_resume_early(struct device *dev) return 0; } +/* + * A firmware-driven suspend powers the SoC down and its sleep engine leaves the + * domains in states of its own: AUTO_ENABLE cleared, some at clock gating. genpd + * only powers a domain back on if it turned it off itself, so its idea of a + * domain it left on is now wrong. Put each domain back where genpd thinks it is. + * + * On T8140 macOS re-raises 22 domains in the first 5 ms after a wake and lowers + * 30 others (measured under aura). Which of them Linux needs isn't established, + * so this follows genpd's books rather than a list. + */ +static int apple_pmgr_ps_resume_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + u32 reg; + int ret; + + if (!pm_resume_via_firmware() || ps->firmware_owned || ps->inherited_on || + ps->sleep_driver_owned) + return 0; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ret = regmap_read(ps->regmap, ps->offset, ®); + if (!ret) { + if (ps->genpd.status == GENPD_STATE_OFF) { + if (FIELD_GET(APPLE_PMGR_PS_TARGET, reg) != APPLE_PMGR_PS_PWRGATE) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_PWRGATE, + false); + } else { + bool auto_enable = !ps->no_auto_enable && !ps->no_auto_pm; + + if (FIELD_GET(APPLE_PMGR_PS_TARGET, reg) != APPLE_PMGR_PS_ACTIVE || + (auto_enable && !(reg & APPLE_PMGR_AUTO_ENABLE))) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + auto_enable); + } + } + spin_unlock_irqrestore(&ps->genpd.slock, flags); + if (ret) + dev_err(dev, "PS %s: resync after sleep failed: %d\n", ps->genpd.name, ret); + + return 0; +} + static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); @@ -588,6 +635,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) */ ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); ps->sleep_retained = of_property_read_bool(node, "apple,sleep-retained"); + ps->sleep_driver_owned = of_property_read_bool(node, "apple,sleep-driver-owned"); /* * Only T8132 hands over inherited-on domains with this contract. Older * M3 device trees carry the property with another meaning; keep them @@ -742,6 +790,7 @@ MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmgr_ps_resume_noirq) }; static struct platform_driver apple_pmgr_ps_driver = { From 93fb08e3d563d263ed17dcbfa5fd62cb3524b964 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:10 -0500 Subject: [PATCH 21/73] pinctrl: apple: put configured pins back after a firmware-driven suspend A firmware-driven suspend powers the pin block off, and nothing sets a pin up again: the pin core thinks its state hasn't changed, so reselecting it does nothing, and a pin in a consumer's hands isn't reconfigured by anything. The PCIe host's CLKREQ mux and PERST# are two that need this. Mark each pin the driver writes, save those in suspend_noirq and write back the ones that differ in resume_noirq. Only those: the pins it hasn't configured belong to the firmware or another processor, and on T8140 a write to one raises an SError. Only on controllers whose regmap reaches the hardware, T8140 and T8132, since a cached one would hide the loss. A plain suspend-to-idle does nothing here, and other SoCs are unchanged. Signed-off-by: Diljit Singh --- drivers/pinctrl/pinctrl-apple-gpio.c | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index d3c482fa4234c..82b901b6a8374 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -26,6 +27,7 @@ #include #include +#include #include "pinctrl-utils.h" #include "core.h" @@ -49,6 +51,11 @@ struct apple_gpio_pinctrl { struct pinctrl_desc pinctrl_desc; struct gpio_chip gpio_chip; u8 irq_group_base; + /* The pins' words across a firmware-driven suspend */ + u32 *saved; + unsigned long *saved_valid; + /* Pins this driver has written. Others belong to someone else. */ + unsigned long *touched; struct apple_gpio_irq_group irqgrps[]; }; @@ -91,6 +98,8 @@ static const struct regmap_config regmap_config = { static void apple_gpio_set_reg(struct apple_gpio_pinctrl *pctl, unsigned int pin, u32 mask, u32 value) { + if (pctl->touched) + set_bit(pin, pctl->touched); regmap_update_bits(pctl->map, REG_GPIO(pin), mask, value); } @@ -589,6 +598,15 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); + /* Only with the regmap reaching the hardware: a cache would hide the loss. */ + if (hw && hw->no_regcache) { + pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); + pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->saved || !pctl->saved_valid || !pctl->touched) + return -ENOMEM; + } + for (i = 0; i < npins; i++) { pins[i].number = i; pins[i].name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "PIN%u", i); @@ -640,10 +658,63 @@ static const struct of_device_id apple_gpio_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_gpio_pinctrl_of_match); +/* + * A firmware-driven suspend powers this block off, and neither the pin core nor + * a consumer sets a pin up again. Slots that fault aren't readable in the regmap, + * so they stay out of this. + */ +static int apple_gpio_pinctrl_suspend_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i; + + if (!pctl->saved || pm_suspend_target_state != PM_SUSPEND_MEM || + !pm_suspend_via_firmware()) + return 0; + + /* + * Only the pins this driver configured. The others belong to the firmware or another + * processor, and writing one back after the wake raises an SError. + */ + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) + if (!regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) + set_bit(i, pctl->saved_valid); + + return 0; +} + +static int apple_gpio_pinctrl_resume_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i, val, restored = 0; + + if (!pctl->saved || !pm_resume_via_firmware()) + return 0; + + for_each_set_bit(i, pctl->saved_valid, pctl->pinctrl_desc.npins) { + if (!regmap_read(pctl->map, REG_GPIO(i), &val) && val == pctl->saved[i]) + continue; + regmap_write(pctl->map, REG_GPIO(i), pctl->saved[i]); + restored++; + } + dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + + return 0; +} + +static const struct dev_pm_ops apple_gpio_pinctrl_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_gpio_pinctrl_suspend_noirq, + apple_gpio_pinctrl_resume_noirq) +}; + static struct platform_driver apple_gpio_pinctrl_driver = { .driver = { .name = "apple-gpio-pinctrl", .of_match_table = apple_gpio_pinctrl_of_match, + .pm = pm_sleep_ptr(&apple_gpio_pinctrl_pm_ops), .suppress_bind_attrs = true, }, .probe = apple_gpio_pinctrl_probe, From 355801ab35cadbd5d944c250816601099a2af0a8 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:11 -0500 Subject: [PATCH 22/73] cpufreq: apple: take the cluster's state after a sleep as the expected one A firmware-driven sleep powers the cluster down, and it came back in pstate 2 with the driver expecting 1, the suspend state. The first request after the wake then stopped all requests as if another owner held the cluster, and the next suspend failed. Read the state again on resume, before the governor starts. Signed-off-by: Diljit Singh --- drivers/cpufreq/apple-soc-cpufreq.c | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 85e592faa0538..9a16751974059 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -841,6 +842,37 @@ static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) return cpufreq_generic_suspend(policy); } +/* + * A firmware-driven sleep powers the cluster down, and it comes back in a performance state + * of the hardware's choosing, not the one asked for at suspend. Take what it reads as the + * expected state again, before the governor starts, rather than stopping every request as if + * someone else owned it. + */ +static int apple_soc_cpufreq_resume(struct cpufreq_policy *policy) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned long flags; + u64 reg; + + if (!priv->info->verify_transition || !pm_resume_via_firmware()) + return 0; + + raw_spin_lock_irqsave(&priv->lock, flags); + reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + if (!(reg & APPLE_DVFS_CMD_BUSY)) { + unsigned int now = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); + + if (now != priv->expected_pstate) + dev_info(priv->cpu_dev, "DVFS state %u after the sleep, %u before\n", + now, priv->expected_pstate); + priv->expected_pstate = now; + priv->transition_failed = false; + } + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { @@ -1548,6 +1580,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, .suspend = apple_soc_cpufreq_suspend, + .resume = apple_soc_cpufreq_resume, }; static int __init apple_soc_cpufreq_module_init(void) From 9bacbcadf516b7929b9b654647f7cabaa43486e3 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:11 -0500 Subject: [PATCH 23/73] soc: apple: mailbox: add the secure RTBuddy port and sleep helpers A coprocessor can have a second mailbox, the ADT's iop,secure-rtbuddy-proxy, for its secure side. It has the registers of the AOP setup port. Its interrupts aren't described yet, so this variant has none and its consumer polls. Match the generic compatible, since the node isn't specific to one SoC. Firmware that powers itself down for a system sleep needs more from its mailbox: - A way to clear and set bit 0 of the receive control word around the stop, as macOS does. - A check for a free send slot that works with interrupts off, where apple_mbox_send()'s own deadline can't pass. - For an ASCWrap v6 CPU whose stop moves to the sleep's last step, with interrupts off and one CPU up, where neither the lifecycle lock nor runtime PM can be taken: its control word for polling its status, the stop's register writes alone, and a call that settles the books after the sleep without touching the registers, so that the next start goes through. The wait for room to send now ends only on a fatal signal. The freezer signals every user task as a system sleep begins, and a message half sent can break a coprocessor's handshake. Rust bindings cover the receive control and holding and releasing receive. Signed-off-by: Diljit Singh --- drivers/soc/apple/mailbox.c | 182 ++++++++++++++++++++++++------ include/linux/soc/apple/mailbox.h | 12 ++ rust/kernel/soc/apple/mailbox.rs | 18 +++ 3 files changed, 175 insertions(+), 37 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index c13c03f9d92ab..256bb0a593657 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -46,6 +46,7 @@ #define APPLE_ASC_MBOX_A2I_RECV1 0x818 #define APPLE_ASC_MBOX_I2A_CONTROL 0x114 +#define APPLE_ASC_MBOX_CONTROL_RECV_ENABLE BIT(0) #define APPLE_ASC_MBOX_I2A_SEND0 0x820 #define APPLE_ASC_MBOX_I2A_SEND1 0x828 #define APPLE_ASC_MBOX_I2A_RECV0 0x830 @@ -223,6 +224,8 @@ struct apple_mbox_hw { /* Status word counts RX occupancy and free TX slots; paired transfers. */ bool counted_fifo; bool ap_initializes_mailboxes; + /* No interrupts: the consumer calls apple_mbox_poll() while it expects traffic. */ + bool polled; unsigned int control_full; unsigned int control_empty; @@ -335,14 +338,11 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); -static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +static void apple_mbox_ascwrap_v6_run_off(struct apple_mbox *mbox) { void __iomem *cpu_control; u32 val; - if (!mbox->ascwrap_v6_cpu_running) - return; - if (mbox->hw->is_ascwrap_v8) { u32 state, ack; @@ -372,12 +372,24 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) val = readl(cpu_control); writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); } +} +static void apple_mbox_ascwrap_v6_stopped_locked(struct apple_mbox *mbox) +{ mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + if (!mbox->ascwrap_v6_cpu_running) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); + apple_mbox_ascwrap_v6_stopped_locked(mbox); +} + /* * A failed resume leaves the runtime PM error set, and runtime PM then fails * every later resume with -EINVAL without retrying the power domain. The @@ -481,6 +493,44 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return NULL; + + return (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_control); + +/* No lock and no runtime PM: the registers alone, as apple_mbox_ascwrap_v6_stop_cpu() has them. */ +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs || + !READ_ONCE(mbox->ascwrap_v6_cpu_running)) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu_atomic); + +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) + ret = -ENODEV; + else if (mbox->ascwrap_v6_cpu_running) + apple_mbox_ascwrap_v6_stopped_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_stopped); + static bool apple_mbox_is_g17(const struct apple_mbox *mbox) { return mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX || @@ -699,7 +749,8 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, * it unmasked for them. The handler masks it when it fires, * and removal masks it otherwise. */ - t = wait_for_completion_interruptible_timeout( + /* Not for the freezer's signal: a message half sent can break a handshake. */ + t = wait_for_completion_killable_timeout( &mbox->tx_empty, min(remaining, msecs_to_jiffies(APPLE_MBOX_TX_RECHECK_MS))); if (t < 0) @@ -886,6 +937,12 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) return IRQ_HANDLED; } +bool apple_mbox_send_ready(struct apple_mbox *mbox) +{ + return !(readl_relaxed(mbox->regs + mbox->hw->a2i_control) & mbox->hw->control_full); +} +EXPORT_SYMBOL(apple_mbox_send_ready); + int apple_mbox_poll(struct apple_mbox *mbox) { unsigned long flags; @@ -1003,7 +1060,8 @@ int apple_mbox_start(struct apple_mbox *mbox) goto out_unlock; } - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; out_unlock: @@ -1022,7 +1080,8 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) apple_mbox_v8_rx_stop(mbox); } else { mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1036,15 +1095,36 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +/* + * Bit 0 of the receive control word, read and written back. A client whose + * firmware powers itself down clears it when the firmware has answered, and sets + * it before starting the firmware again. That it gates receiving is inferred. + */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&mbox->rx_lock, flags); + val = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + val = enabled ? val | APPLE_ASC_MBOX_CONTROL_RECV_ENABLE : + val & ~APPLE_ASC_MBOX_CONTROL_RECV_ENABLE; + writel_relaxed(val, mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} +EXPORT_SYMBOL_GPL(apple_mbox_set_recv_enabled); + void apple_mbox_hold_rx(struct apple_mbox *mbox) { - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_hold_rx); void apple_mbox_release_rx(struct apple_mbox *mbox) { - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_release_rx); @@ -1085,7 +1165,7 @@ static void apple_mbox_quiesce(void *data) if (was_active) { if (apple_mbox_has_rx_poller(mbox)) apple_mbox_v8_rx_stop(mbox); - else + else if (mbox->irq_recv_not_empty >= 0) disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1105,7 +1185,8 @@ static void apple_mbox_quiesce(void *data) if (mbox->irq_send_empty >= 0) synchronize_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); - synchronize_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + synchronize_irq(mbox->irq_recv_not_empty); spin_lock_irqsave(&mbox->rx_lock, flags); had_rx = mbox->rx; @@ -1380,19 +1461,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; } - mbox->irq_recv_not_empty = - platform_get_irq_byname(pdev, "recv-not-empty"); - if (mbox->irq_recv_not_empty < 0) - return mbox->irq_recv_not_empty; + if (mbox->hw->polled) { + mbox->irq_recv_not_empty = -ENXIO; + mbox->irq_send_empty = -ENXIO; + } else { + mbox->irq_recv_not_empty = + platform_get_irq_byname(pdev, "recv-not-empty"); + if (mbox->irq_recv_not_empty < 0) + return mbox->irq_recv_not_empty; - /* - * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. - * apple_mbox_send then polls the A2I control register instead. - */ - mbox->irq_send_empty = - platform_get_irq_byname_optional(pdev, "send-empty"); - if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return mbox->irq_send_empty; + /* + * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. + * apple_mbox_send then polls the A2I control register instead. + */ + mbox->irq_send_empty = + platform_get_irq_byname_optional(pdev, "send-empty"); + if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) + return mbox->irq_send_empty; + } mbox->rx_util_min = -1; mbox->rx_util_min_applied = -1; @@ -1401,21 +1487,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); - if (!irqname) - return -ENOMEM; + if (mbox->irq_recv_not_empty >= 0) { + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); + if (!irqname) + return -ENOMEM; - if (apple_mbox_is_g17(mbox)) - ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, - apple_mbox_recv_irq, - IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, - irqname, mbox); - else - ret = devm_request_irq(dev, mbox->irq_recv_not_empty, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); - if (ret) - return ret; + if (apple_mbox_is_g17(mbox)) + ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, + apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_NO_SUSPEND, + irqname, mbox); + else + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (ret) + return ret; + } if (mbox->irq_send_empty >= 0) { irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", @@ -1596,6 +1685,24 @@ static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { .has_irq_controls = false, }; +/* The kind of port the secure RTBuddy proxy has, with the AOP setup port's registers. */ +static const struct apple_mbox_hw apple_mbox_secure_rtbuddy_hw = { + .polled = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_T8140_AOP_SETUP_A2I_CONTROL, + .a2i_send0 = APPLE_T8140_AOP_SETUP_A2I_SEND0, + .a2i_send1 = APPLE_T8140_AOP_SETUP_A2I_SEND1, + + .i2a_control = APPLE_T8140_AOP_SETUP_I2A_CONTROL, + .i2a_recv0 = APPLE_T8140_AOP_SETUP_I2A_RECV0, + .i2a_recv1 = APPLE_T8140_AOP_SETUP_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -1718,6 +1825,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, + { .compatible = "apple,secure-rtbuddy-proxy", .data = &apple_mbox_secure_rtbuddy_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* * t6030 gfx-asc mailbox in the boot.bin DTB: it carries no asc-mailbox-v4 fallback so that diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 7a07772cc5967..bf091d138968d 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -100,7 +100,11 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); +/* For a caller with interrupts off, where apple_mbox_send()'s own deadline can't pass. */ +bool apple_mbox_send_ready(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* Bit 0 of the receive control word, for firmware that powers itself down. */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled); /* * Leave received messages in the hardware FIFO, without an interrupt, until * apple_mbox_release_rx(). For consumers that must not wake the system. @@ -122,6 +126,14 @@ int apple_mbox_ascwrap_v6_get_lifecycle( int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); +/* + * For a CPU that a system sleep's last step stops with interrupts off: its control word, for + * polling its status, and the stop's register writes alone. Once the sleep is over, + * apple_mbox_ascwrap_v6_cpu_stopped() settles the books without touching the registers. + */ +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox); +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, u32 *control, u32 *status); diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index c4f86930e3c65..03b17af8851c9 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -96,6 +96,24 @@ impl Mailbox { // SAFETY: Calling the c function, `mbox` is a valid pointer to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) } + + /// Sets bit 0 of the receive control word, for firmware that powers itself down. + pub fn set_recv_enabled(&self, enabled: bool) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_set_recv_enabled(self.mbox, enabled) }; + } + + /// Stops taking receive interrupts, so that no handler reads the mailbox. + pub fn hold_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_hold_rx(self.mbox) }; + } + + /// Undoes [`Self::hold_rx`]. + pub fn release_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_release_rx(self.mbox) }; + } } impl Drop for Mailbox { From f7edd90a82d1b22d1c247738f533fc6054cd7c95 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:12 -0500 Subject: [PATCH 24/73] soc: apple: rtkit: take a co-processor through a system sleep A firmware-driven sleep powers the co-processors off, and RTKit had no way to stop one for that or to bring it back without a full reboot of its firmware. This adds what the drivers need, each step measured against macOS: - AP-only power notices. macOS tells the AOP, MTP and SMC about a sleep with the AP power state alone, 0x201 before it and 0x20 after, and leaves them running. A noirq variant polls. - A sleep stop. After the IOP's idle ack macOS waits for the CPU status to read stopped and idle, clears bit 0 of the mailbox's I2A control word, and clears the run bit twice, each time from what it reads back. apple_rtkit_sleep_stop() does that, with a 5 ms poll for the status; on an error the run bit stays set and the firmware still runs. - A polled idle for a stop that can't be taken back. ANS doesn't come back from idle with its power kept, so its idle follows every refusal, with interrupts off and one CPU up. The messages are sent and acked by polling, the syslog and IOReport notifications that want an ack are answered as they come, and OSLOG counters are echoed, below. A counter the receive work still owed, which can't run then, is echoed before the first state request, and whichever side takes it first echoes it, so none is echoed twice. - A warm restart. After a real sleep the firmware restarts with its buffers and rings in place and sends HELLO unasked. Nothing is freed and no endpoint is started again; the endpoint map is echoed as macOS does, with bit 1 added to the first part, and a driver can set what the last part gets back (ANS gets endpoint 0x20 only). apple_rtkit_sleep_return() does that after a sleep that reached the hardware. After one that didn't, a firmware that idled gets -EAGAIN: an idled firmware doesn't wake with its power kept, so it stays down until the next real sleep restarts it. One whose idle failed still runs, and gets 0. apple_rtkit_sleep_return_start() is the first half, for a driver that starts other firmware before waiting, and apple_rtkit_power_cycled_return() starts one its driver power-cycled without a sleep. - Hooks around the AP state change in boot and idle, for a driver that has to answer another port in between. - A polled port that only answers the co-processor: the DCP's secure RTBuddy port, whose HELLO and endpoint map macOS's secure side echoes back unchanged. - OSLOG counter echo. A firmware counts its log writes, holds the next counter until the last is echoed, and with one outstanding takes no AP power notice: a DCP restarted warm refuses its next AP idle, and the MTP and the SMC stopped answering after about eight sleeps. macOS echoes every one, so on T8140 every RTKit device's counters are echoed, and a driver can turn that off. - A crash on a crash log endpoint the firmware never announced. Such a firmware asks for that buffer only when it crashes, at an address it chose. The request was taken for a boot-time one and dropped, so the driver never learned of the crash and kept sending to it: the AOP after a wake, refusing every later sleep. It's reported at once, and the buffer is mapped through the device's IOMMU when that's one contiguous range and acknowledged, so the log is dumped when it comes. Management message 0xc, which follows the IOP's idle ack, is no longer logged as unknown. An adopted session completes its endpoint map at adoption, as its previous owner did that. A handshake given up halfway can leave a firmware started that nothing stops, so RTKit's waits end only on a fatal signal, and during a system sleep not even that. The freezer signals every user task as a sleep begins: on a J700 that cut a display power-on short as a sleep began, left the DCP running while its driver took it as stopped, and every later sleep was refused. Rust bindings cover idle, mailbox off, warm restart and the OSLOG echo. Nothing changes for a driver that doesn't call the new functions. Signed-off-by: Diljit Singh --- drivers/soc/apple/rtkit-internal.h | 13 + drivers/soc/apple/rtkit.c | 596 +++++++++++++++++- include/linux/soc/apple/rtkit.h | 104 +++ rust/kernel/soc/apple/rtkit.rs | 26 + tools/asahi/rtkit-oslog-tests.py | 5 + .../selftests/drivers/gpu/apple-dcp/boot.py | 3 +- 6 files changed, 727 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 2ad4d96bec768..5c3b8848366e6 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -61,6 +61,17 @@ struct apple_rtkit { * hand out kdebug buffers, and poll the mailbox while waiting. */ bool sync_syslog_ack; + /* Echo each OSLOG channel-0 counter, as macOS does. */ + bool oslog_echo; + /* A counter queued for the receive work and not yet echoed. */ + u64 oslog_owed; + /* Device interrupts are off: poll the mailbox while waiting. */ + bool poll_waits; + /* The firmware is restarting with its buffers and rings still in place. */ + bool warm_restart; + u32 warm_last_mask; + /* A port that only answers the co-processor, see apple_rtkit_port_handshake(). */ + bool port_only; DECLARE_KFIFO(sync_ack_fifo, struct apple_mbox_msg, 64); struct work_struct sync_ack_work; @@ -82,6 +93,8 @@ struct apple_rtkit { /* Session inherited from the bootloader: INIT geometry never observed. */ bool syslog_inherited; bool crashlog_inherited; + /* The crash log buffer is the firmware's own, mapped here when it crashed. */ + bool crashlog_placed; bool oslog_inherited; struct workqueue_struct *wq; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3b821415d54d8..864826487a49f 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -7,10 +7,13 @@ #include "rtkit-internal.h" #include +#include +#include #include #include #include #include +#include enum { APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */ @@ -45,6 +48,7 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, + APPLE_RTKIT_MGMT_IDLE_DONE = 0xc, }; #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0) @@ -61,6 +65,9 @@ enum { #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0) +#define APPLE_RTKIT_POLLED_STEP_US 10 +#define APPLE_RTKIT_POLLED_LOOPS 20000 + #define APPLE_RTKIT_CRASHLOG_CRASH 1 #define APPLE_RTKIT_BUFFER_REQUEST 1 @@ -115,6 +122,14 @@ enum { #define APPLE_RTKIT_CA_EVENT 2 #define APPLE_RTKIT_CA_EVENT_ACK 3 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +/* From the CPU control register. Both bits wait for the firmware's last message to be read. */ +#define APPLE_ASC_CPU_STATUS 4 +#define APPLE_ASC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_ASC_CPU_STATUS_IDLE BIT(5) +#define APPLE_ASC_CPU_STATUS_DOWN (APPLE_ASC_CPU_STATUS_STOPPED | APPLE_ASC_CPU_STATUS_IDLE) + #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 @@ -251,7 +266,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) * request early crashlog startup, need the endpoint before the map is * acknowledged: G17P can request its crash DVA right after. */ - if (apple_rtkit_early_crashlog(rtk) && base == 0 && + if (!rtk->warm_restart && apple_rtkit_early_crashlog(rtk) && base == 0 && test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) { ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); if (ret) { @@ -261,17 +276,46 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + /* macOS's secure side sends each part back unchanged. */ + if (rtk->port_only) { + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, msg); + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) { + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + } + return; + } + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; else reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; + /* + * macOS's reply to a restarted DCP echoes the map, and adds bit 1 to the + * first part: 0x51d becomes 0x51f. It starts no endpoints. Measured under + * aura, 3 of 3. Why bit 1 isn't known, but ep 1 is the crash log's. + */ + if (rtk->warm_restart) { + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) + reply |= bitmap & rtk->warm_last_mask; + else + reply |= bitmap | BIT_ULL(1); + } + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (rtk->warm_restart) { + rtk->warm_restart = false; + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + return; + } + for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) { switch (ep) { /* the management endpoint is started by default */ @@ -362,6 +406,10 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK: apple_rtkit_management_rx_ap_pwr_ack(rtk, msg); break; + case APPLE_RTKIT_MGMT_IDLE_DONE: + /* Follows the IOP's idle ack, with a zero argument. The name is a guess. */ + dev_dbg(rtk->dev, "RTKit: idle done: 0x%llx\n", msg); + break; default: dev_warn( rtk->dev, @@ -485,10 +533,14 @@ static void apple_rtkit_free_buffer(struct apple_rtkit *rtk, if (bfr->size == 0) return; - if (rtk->ops->shmem_destroy) + if (bfr == &rtk->crashlog_buffer && rtk->crashlog_placed) { + memunmap(bfr->buffer); + rtk->crashlog_placed = false; + } else if (rtk->ops->shmem_destroy) { rtk->ops->shmem_destroy(rtk->cookie, bfr); - else if (bfr->buffer) + } else if (bfr->buffer) { dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova); + } bfr->buffer = NULL; bfr->iomem = NULL; @@ -720,6 +772,46 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, len, DMA_FROM_DEVICE); } +/* + * The buffer a crashing firmware placed at its own address, mapped if its IOMMU + * translates every page of it to one contiguous range, and acknowledged as a placed + * buffer is. The firmware doesn't run again until a reboot. + */ +static int apple_rtkit_crashlog_map(struct apple_rtkit *rtk, u64 msg) +{ + struct apple_rtkit_shmem *bfr = &rtk->crashlog_buffer; + struct iommu_domain *domain = iommu_get_domain_for_dev(rtk->dev); + dma_addr_t iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + size_t size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; + phys_addr_t phys, page; + size_t off; + void *va; + + if (!domain || !iova || !size) + return -ENODEV; + phys = iommu_iova_to_phys(domain, iova); + for (off = 0; phys && off < size; off += PAGE_SIZE) { + page = iommu_iova_to_phys(domain, iova + off); + if (page != phys + off) { + dev_err(rtk->dev, "RTKit: crash log at %pad isn't contiguous at +%#zx\n", + &iova, off); + return -EINVAL; + } + } + if (!phys) + return -ENOENT; + va = memremap(phys, size, MEMREMAP_WB); + if (!va) + return -ENOMEM; + bfr->buffer = va; + bfr->size = size; + bfr->iova = iova; + bfr->is_mapped = true; + rtk->crashlog_placed = true; + + return apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_CRASHLOG, msg, NULL, false); +} + static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); @@ -732,6 +824,25 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } + /* + * A firmware that never announced this endpoint asks for its buffer only when + * it crashes, so there's no handshake to take the request for. Report the + * crash now, as the log may never come. + */ + if (!rtk->crashlog_buffer.size && !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) { + int ret = apple_rtkit_crashlog_map(rtk, msg); + + dev_err(rtk->dev, "RTKit: co-processor has crashed: crash log buffer at %#llx %s (%d)\n", + FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), + ret ? "not mapped" : "mapped, waiting for its log", ret); + if (!rtk->crashed) { + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + } + return; + } + if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) @@ -786,6 +897,9 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) "RTKit: Couldn't allocate crashlog shadow buffer\n"); } + /* Its driver heard of the crash when the buffer was asked for. */ + if (rtk->crashed) + return; rtk->crashed = true; if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); @@ -944,6 +1058,22 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); int err; + /* + * A firmware counts its log writes and holds the next counter until the last is + * echoed, and with one outstanding takes no AP power notice: a DCP after a warm + * restart, the MTP and the SMC after about eight sleeps. macOS echoes every one. + */ + if (type == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo) { + /* A polled idle that came first echoed it already. */ + if (cmpxchg(&rtk->oslog_owed, msg, 0) != msg) + return; + dev_dbg(rtk->dev, "RTKit: oslog counter %016llx, echoed\n", msg); + err = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, msg, NULL, false); + if (err) + dev_err(rtk->dev, "RTKit: oslog echo failed: %d\n", err); + return; + } + /* Preserve the tested Neo/M5 and synchronous-log no-ACK policy. */ if (type == APPLE_RTKIT_OSLOG_LOG && (rtk->sync_syslog_ack || of_machine_is_compatible("apple,t8140") || @@ -1064,12 +1194,17 @@ static void apple_rtkit_rx_work(struct work_struct *work) kfree(rtk_work); } -static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +/* An OSLOG counter this firmware waits to see echoed. */ +static bool apple_rtkit_oslog_owed_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return ep == APPLE_RTKIT_EP_OSLOG && + FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg) == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo; +} + +static bool apple_rtkit_notification_msg(u8 ep, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - if (!rtk->sync_syslog_ack) - return false; if (ep == APPLE_RTKIT_EP_SYSLOG) return type == APPLE_RTKIT_SYSLOG_LOG; if (ep == APPLE_RTKIT_EP_IOREPORT) @@ -1078,6 +1213,11 @@ static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) return false; } +static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return rtk->sync_syslog_ack && apple_rtkit_notification_msg(ep, msg); +} + static void apple_rtkit_sync_ack_work(struct work_struct *work) { struct apple_rtkit *rtk = @@ -1148,6 +1288,8 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, if (!work) return; + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) + WRITE_ONCE(rtk->oslog_owed, msg.msg0); work->rtk = rtk; work->ep = ep; work->msg = msg.msg0; @@ -1193,6 +1335,82 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +static int apple_rtkit_poll_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->epmap_completion)) + return rtk->boot_result; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return try_wait_for_completion(&rtk->epmap_completion) ? rtk->boot_result : -ETIME; +} + +static int apple_rtkit_port_power(struct apple_rtkit *rtk, unsigned int state, + unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + reinit_completion(&rtk->ap_pwr_ack_completion); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state)); + if (ret) + return ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->ap_pwr_ack_completion)) + return 0; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return -ETIME; +} + +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + rtk->port_only = true; + + return apple_rtkit_poll_handshake(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_map); + +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + /* The co-processor starts its side of this port itself. */ + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_INIT, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_on); + +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + int ret; + + ret = apple_rtkit_port_map(rtk, timeout_ms); + if (ret) + return ret; + + return apple_rtkit_port_on(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_handshake); + +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_IDLE, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_idle); + void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { apple_mbox_hold_rx(rtk->mbox); @@ -1268,6 +1486,10 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) set_bit(system_endpoints[i], rtk->endpoints); + /* apple_rtkit_wake() must not wait for an EPMAP that already happened. */ + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->syslog_inherited = true; @@ -1407,6 +1629,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, rtk->handshake_timeout_ms = APPLE_RTKIT_HANDSHAKE_TIMEOUT_MS; rtk->sync_syslog_ack = of_property_read_bool(dev->of_node, "apple,sync-syslog-ack"); + /* As macOS does; a driver whose firmware shouldn't get them turns it off. */ + rtk->oslog_echo = of_machine_is_compatible("apple,t8140"); INIT_KFIFO(rtk->sync_ack_fifo); INIT_WORK(&rtk->sync_ack_work, apple_rtkit_sync_ack_work); @@ -1434,6 +1658,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + rtk->warm_last_mask = U32_MAX; + rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { @@ -1487,6 +1713,19 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c); +/* + * A handshake given up halfway can leave a firmware started that nothing stops. The + * freezer signals every user task as a sleep begins, so only a fatal signal ends the + * wait, and during a system sleep not even that. + */ +static long apple_rtkit_wait_timeout(struct completion *c, unsigned int ms) +{ + if (pm_suspend_target_state == PM_SUSPEND_MEM) + return wait_for_completion_timeout(c, msecs_to_jiffies(ms)); + + return wait_for_completion_killable_timeout(c, msecs_to_jiffies(ms)); +} + /* * apple,sync-syslog-ack (AsahiLinux/linux#607): the mailbox can leave * messages queued without another interrupt during bootstrap, so poll it @@ -1498,7 +1737,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) long t; int ret; - if (!rtk->sync_syslog_ack) + if (!rtk->sync_syslog_ack && !rtk->poll_waits) return apple_rtkit_wait_for_completion(rtk, c); timeout = jiffies + msecs_to_jiffies(rtk->handshake_timeout_ms); @@ -1507,8 +1746,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) if (ret < 0) return ret; - t = wait_for_completion_interruptible_timeout(c, - msecs_to_jiffies(10)); + t = apple_rtkit_wait_timeout(c, 10); if (t < 0) return t; if (t > 0) @@ -1532,10 +1770,9 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted_with_buffers); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c) { - unsigned long timeout = msecs_to_jiffies(rtk->handshake_timeout_ms); long t; - t = wait_for_completion_interruptible_timeout(c, timeout); + t = apple_rtkit_wait_timeout(c, rtk->handshake_timeout_ms); if (t < 0) return t; else if (t == 0) @@ -1551,6 +1788,63 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, } EXPORT_SYMBOL_GPL(apple_rtkit_set_handshake_timeout); +/* + * Expect HELLO from a firmware being restarted warm: it keeps the buffers it was + * given and its rings, so unlike apple_rtkit_reinit() nothing is freed, and + * endpoints aren't started again. The caller then sets the run bit and calls + * apple_rtkit_boot(). + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk) +{ + unsigned long flags; + int ret; + + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + + reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + + rtk->crashed = false; + rtk->warm_restart = true; + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; + kfifo_reset(&rtk->sync_ack_fifo); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + WRITE_ONCE(rtk->shutting_down, false); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + ret = apple_mbox_start(rtk->mbox); + if (ret) { + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + } else { + apple_mbox_set_recv_enabled(rtk->mbox, true); + } + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_warm_restart); + +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask) +{ + rtk->warm_last_mask = last_mask; +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_warm_epmap_mask); + +int apple_rtkit_port_restart(struct apple_rtkit *rtk) +{ + int ret = apple_rtkit_warm_restart(rtk); + + rtk->warm_restart = false; + rtk->port_only = true; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_restart); + int apple_rtkit_reinit(struct apple_rtkit *rtk) { unsigned long flags; @@ -1653,7 +1947,8 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power, + void (*before_ap)(void *data), void *data) { int ret; @@ -1698,16 +1993,27 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) return ret; } - return early_ap_power ? 0 : - apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (early_ap_power) + return 0; + + if (before_ap) + before_ap(data); + + return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); } int apple_rtkit_boot(struct apple_rtkit *rtk) { - return apple_rtkit_boot_common(rtk, false); + return apple_rtkit_boot_common(rtk, false, NULL, NULL); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data) +{ + return apple_rtkit_boot_common(rtk, false, before_ap, data); +} +EXPORT_SYMBOL_GPL(apple_rtkit_boot_with); + int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { if (rtk->crashed) @@ -1752,26 +2058,66 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poweroff); -int apple_rtkit_idle(struct apple_rtkit *rtk) +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data) { int ret; /* if OFF is used here the co-processor will not wake up again */ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: AP idle not acknowledged: %d\n", ret); return ret; + } + + if (after_ap) + after_ap(data); ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: IOP idle not acknowledged: %d\n", ret); return ret; + } rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE; return 0; } +EXPORT_SYMBOL_GPL(apple_rtkit_idle_with); + +int apple_rtkit_idle(struct apple_rtkit *rtk) +{ + return apple_rtkit_idle_with(rtk, NULL, NULL); +} EXPORT_SYMBOL_GPL(apple_rtkit_idle); +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_off); + +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on) +{ + return apple_rtkit_set_ap_power_state(rtk, on ? APPLE_RTKIT_PWR_STATE_ON : + APPLE_RTKIT_PWR_STATE_IDLE); +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power); + +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on) +{ + int ret; + + rtk->poll_waits = true; + ret = apple_rtkit_ap_power(rtk, on); + rtk->poll_waits = false; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power_noirq); + int apple_rtkit_quiesce(struct apple_rtkit *rtk) { int ret; @@ -1816,7 +2162,7 @@ static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) if (ret) return ret; - return apple_rtkit_boot_common(rtk, early_ap_power); + return apple_rtkit_boot_common(rtk, early_ap_power, NULL, NULL); } int apple_rtkit_wake(struct apple_rtkit *rtk) @@ -1831,6 +2177,218 @@ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); +static int apple_rtkit_sleep_idle(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + u32 status; + int ret; + + ret = apple_rtkit_idle(rtk); + if (ret) + return ret; + + ret = readl_poll_timeout(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == APPLE_ASC_CPU_STATUS_DOWN, + 100, 5000); + if (ret) + dev_err(rtk->dev, "RTKit: idle, but not stopped: status %#x\n", status); + + return ret; +} + +/* With interrupts off: room in the FIFO, then the message. */ +static bool apple_rtkit_polled_send(struct apple_rtkit *rtk, struct apple_mbox_msg msg) +{ + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return false; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + + return !apple_mbox_send(rtk->mbox, msg, true); +} + +/* + * Only what an idle needs: the state acks, notifications answered as they come, and OSLOG + * counters echoed, without which the firmware takes no AP power notice. + */ +static void apple_rtkit_polled_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct apple_rtkit *rtk = cookie; + u8 ep = msg.msg1; + + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) { + if (!apple_rtkit_polled_send(rtk, msg)) + dev_err(rtk->dev, "RTKit: oslog echo failed while idling\n"); + return; + } + if (ep == APPLE_RTKIT_EP_MGMT) { + u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg.msg0); + unsigned int state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg.msg0); + + if (type == APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK) + rtk->iop_power_state = state; + else if (type == APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK) + rtk->ap_power_state = state; + } else if (apple_rtkit_notification_msg(ep, msg.msg0) && apple_mbox_send_ready(mbox)) { + apple_mbox_send(mbox, msg, true); + } +} + +static int apple_rtkit_polled_state(struct apple_rtkit *rtk, u8 type, unsigned int *state, + unsigned int *us) +{ + struct apple_mbox_msg msg = { + .msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type) | + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_IDLE), + .msg1 = APPLE_RTKIT_EP_MGMT, + }; + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return -ETIMEDOUT; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + apple_mbox_send(rtk->mbox, msg, true); + + for (i = 0; i < APPLE_RTKIT_POLLED_LOOPS; i++) { + apple_mbox_poll(rtk->mbox); + if (READ_ONCE(*state) == APPLE_RTKIT_PWR_STATE_IDLE) + return 0; + udelay(APPLE_RTKIT_POLLED_STEP_US); + *us += APPLE_RTKIT_POLLED_STEP_US; + } + + return -ETIMEDOUT; +} + +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us) +{ + unsigned long flags; + u64 owed; + u32 status; + int ret; + + *us = 0; + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_polled_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + /* What's already waiting is answered first, or the firmware may not go idle. */ + apple_mbox_poll(rtk->mbox); + /* Then a counter left for the receive work, which can't run now. */ + owed = xchg(&rtk->oslog_owed, 0); + if (owed) { + struct apple_mbox_msg echo = { .msg0 = owed, .msg1 = APPLE_RTKIT_EP_OSLOG }; + + if (!apple_rtkit_polled_send(rtk, echo)) + dev_err(rtk->dev, "RTKit: oslog echo failed before the idle\n"); + } + + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + &rtk->ap_power_state, us); + if (!ret) + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + &rtk->iop_power_state, us); + if (!ret) + ret = readl_poll_timeout_atomic(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == + APPLE_ASC_CPU_STATUS_DOWN, + APPLE_RTKIT_POLLED_STEP_US, 5000); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_idle_polled); + +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_recv_off); + +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + ret = apple_rtkit_sleep_idle(rtk, cpu_control); + if (ret) + return ret; + + apple_rtkit_mailbox_off(rtk); + + /* Twice, each time from what it reads back. */ + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_stop); + +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + /* + * No real sleep: a firmware that idled doesn't come back from that with its power kept, + * and stays down until the next sleep restarts it. One whose idle failed still runs. + */ + if (!pm_resume_via_firmware()) + return (readl(cpu_control) & APPLE_ASC_CPU_CONTROL_RUN) && + (rtk->iop_power_state & 0xff) == APPLE_RTKIT_PWR_STATE_ON ? 0 : -EAGAIN; + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return); + +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret = apple_rtkit_warm_restart(rtk); + + if (ret) + return ret; + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_power_cycled_return); + +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo) +{ + rtk->oslog_echo = echo; + dev_dbg(rtk->dev, "RTKit: OSLOG counters %s\n", echo ? "echoed" : "dropped"); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_oslog_echo); + +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + if (!pm_resume_via_firmware()) + return -EOPNOTSUPP; + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return_start); + void apple_rtkit_free(struct apple_rtkit *rtk) { apple_rtkit_stop_rx(rtk); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 86cdb8426488d..79190e4093bd2 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -215,6 +215,9 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* apple_rtkit_boot(), calling @before_ap between the IOP's report and the AP going on. */ +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data); + /* * Mark an instance created with apple_rtkit_init() as running without a * handshake, see apple_rtkit_init_adopted(). @@ -251,6 +254,92 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk); */ int apple_rtkit_idle(struct apple_rtkit *rtk); +/* + * Turn the mailbox off for a co-processor that has acknowledged idle, before its + * run bit and power state go. Only apple_rtkit_warm_restart() follows this. + */ +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk); + +/* + * The same with interrupts off and one CPU up, for a co-processor that doesn't come + * back from idle without losing power, so that its idle has to follow every refusal. + * @us is about how long the two acknowledgements took. Only + * apple_rtkit_warm_restart() follows this, once the sleep has reached the hardware. + */ +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us); + +/* + * Only the mailbox's receive bit, for the moment before that co-processor's run bit + * goes. Safe with interrupts off, and apple_rtkit_warm_restart() sets it again. + */ +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk); + +/* + * Prepare for a firmware being restarted with its buffers kept. + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk); + +/* + * For a firmware whose last endpoint map is answered, on a warm restart, with + * only the endpoints in @last_mask rather than an echo of all of them. + */ +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask); + +/* + * Tell a co-processor that keeps running across a system sleep about the AP's + * power state, without changing its own: idle before the sleep and on after. + */ +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on); + +/* The same, for a noirq phase: device interrupts are off, so it polls. */ +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on); + +/* + * Stop a co-processor for a system sleep that may remove its power: idle, wait + * for the status that shows it has finished, then clear the run bit. On an error + * the run bit is still set and the firmware still runs. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Bring back a co-processor taken down for a system sleep with + * apple_rtkit_sleep_stop() or apple_rtkit_idle(), from a process context with its + * mailbox's supplier resumed. + * + * Where the sleep reached the hardware (pm_resume_via_firmware()) the firmware + * restarts and keeps its buffers and endpoints: this restarts the receive path, sets + * the run bit, echoes the endpoint map and takes the IOP state it reports unasked, + * then sets the AP state. Where it didn't, one that idled gets -EAGAIN: an idled + * firmware doesn't wake with its power kept, so it stays down, and comes back the + * same way after the next sleep that does reach the hardware. One whose idle + * failed still runs, and gets 0. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * A start as after a real sleep, for a co-processor its driver power-cycled without one: + * the warm restart, the run bit set, then RTKit's handshake. + */ +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * The first half of apple_rtkit_sleep_return() after a real sleep: the restart and the run + * bit, with no wait. apple_rtkit_boot() then waits for the firmware to report on, so other + * firmware can be started in between. -EOPNOTSUPP after a sleep that kept the power. + */ +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Echo each OSLOG channel-0 counter the firmware sends, as macOS does. On by default on + * T8140, where a firmware with one outstanding takes no AP power notice. + */ +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo); + /* * Checks if RTKit is running and ready to handle messages. */ @@ -293,6 +382,21 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * A port that only answers the co-processor, polled, with no interrupts. Map answers + * its HELLO and endpoint map, and on brings the AP side of the port up. Handshake is + * both. Idle takes the AP side down. Restart prepares for a co-processor that was + * stopped. + */ +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_restart(struct apple_rtkit *rtk); + +/* apple_rtkit_idle(), calling @after_ap between the AP's idle and the IOP's. */ +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data); + /* * Stop taking interrupts for messages from the co-processor, which then * stay queued in the mailbox until apple_rtkit_resume_rx(). diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index ac9d7ab5032b6..236948ab75b13 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -373,6 +373,26 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) } + /// Asks the coprocessor to go idle, AP then IOP state, and waits for both acknowledgments. + pub fn idle(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_idle(self.rtk) }) + } + + /// Turns the mailbox off for a coprocessor that has acknowledged idle, before its run bit and + /// power state go. Only [`Self::warm_restart`] follows this. + pub fn mailbox_off(self: Pin<&mut Self>) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_mailbox_off(self.rtk) }; + } + + /// Prepares for HELLO from firmware restarting warm, which keeps its buffers and rings. The + /// caller then sets the run bit and calls [`Self::boot`]. + pub fn warm_restart(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_warm_restart(self.rtk) }) + } + /// Starts a non-system endpoint. pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { // SAFETY: `rtk` is valid per the type invariant. @@ -387,6 +407,12 @@ impl RtKit { }) } + /// Echoes the OSLOG counters the firmware sends, or drops them. + pub fn set_oslog_echo(&self, echo: bool) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_set_oslog_echo(self.rtk, echo) }; + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 160c163758d54..37d45153ecce2 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -37,6 +37,7 @@ def function(source, name): #define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) #define APPLE_RTKIT_MAX_ENDPOINTS 256 #define WRITE_ONCE(x,v) ((x)=(v)) +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) #define spin_lock_irqsave(l,f) ((void)(l), (f)=0) #define spin_unlock_irqrestore(l,f) ((void)(l), (void)(f)) struct apple_rtkit_shmem { size_t size; }; @@ -48,6 +49,9 @@ def function(source, name): struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer, pmp_aux_buffer; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; + bool oslog_echo; + u64 oslog_owed; + int boot_result; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4]; char *syslog_msg_buffer; @@ -67,6 +71,7 @@ def function(source, name): static void set_bit(unsigned n, unsigned long *b) { b[n/64] |= 1UL << (n%64); } static void bitmap_zero(unsigned long *b, unsigned n) { memset(b,0,n/8); } static void reinit_completion(int *c) { *c=0; } +static void complete_all(int *c) { *c=1; } static void apple_rtkit_rx(void) {} static void apple_rtkit_detach_rx(struct apple_rtkit *r) { ++detachments; r->shutting_down=true; } static void flush_workqueue(void *w) { (void)w; ++flushes; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py index 14c837ceeb5f3..95c02e87ddc02 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -19,7 +19,8 @@ def oslog(root): #define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctzll(mask)) #define APPLE_RTKIT_EP_OSLOG 8 struct buffer { unsigned sentinel; unsigned size; }; -struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack; }; +struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack, oslog_echo; u64 oslog_owed; }; +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) static unsigned warnings, notifications, requests, acks, errors; static u64 observed; static bool neo, m5; From ee3b6f40a8292ec6e131a72f3cc7e060c7f9b987 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:13 -0500 Subject: [PATCH 25/73] iommu: apple-dart: take the coprocessor DARTs through a firmware suspend A firmware-driven sleep resets every DART. After the first real wake on T8140 every RTKit coprocessor but the DCP timed out: the PMP and SIO restart their firmware in their early resume, and their first DMA went through a DART that had not been put back yet. - For a firmware-driven sleep, suspend and resume in the noirq phase, which every device finishes before any early resume, and after a real sleep write the saved state back even for a DART runtime PM had suspended, which pm_runtime_force_resume() leaves alone. Every other sleep keeps the normal phase, on every SoC: the PCIe-C tunnel DARTs need it, as their host stops the tunnel ports in its suspend_noirq and sets the port clock up again in its resume_noirq. On Apple SoCs only the T8140 sleep driver, later in this series, sets pm_suspend_via_firmware(), from .begin, and it holds until the next suspend, so both halves take the same phase. - Replay the loader's address filter. The PMP's first write after the wake lands in a range the loader programs into its DART's DAPF (the ADT's dapf-instance-0), which a real sleep clears. Take the ranges from apple,dapf-ranges and an optional apple,dapf-r20, and write them back wherever the translations are restored, unless the filter still holds them or is locked. - Write the loader's tunables back where a power-off lost them, from apple,resume-tunable, only the words that differ. The ISP's DARTs need both, as its firmware starts again after the wake. - A consumer-driven save and restore for a DART whose firmware gates itself before the DARTs suspend (the SEP's): apple,sleep-prepare, apple,sleep-restore and apple,sleep-restore-late list the writes around it, and the consumer calls apple_dart_sleep_prepare() and apple_dart_sleep_restore() in its own order. The DART's own callbacks then leave it alone. Its config words go back as the DART had them before the sleep, since Linux's setup of those words differs from macOS's. - GAPF. On T8140 the fabric rejects a write to a display coprocessor's CPU power state while bit 4 of its DART's GAPF slice is set. A display stop that flushes the DART waits for the flush, then clears the bit; it's set again when the streams come back on. apple_dart_display_streams() does macOS's stream writes for a display power cycle. A deep sleep resets the GAPF bank too, so the slice's value is saved at probe and protected with that, and apple_dart_gapf_ready() releases a slice the wake left protected before the coprocessor's CPU domain comes up. - While a display DART's streams are off for a power-down its registers may be unpowered: take each stream's table level from a snapshot made as they went off, and defer TLB invalidations until the streams are back on. apple_dart_power_dark() does the same for a display DART whose domain goes off at runtime while others still map into it: the display subsystem maps every framebuffer into DCPEXT's DART, and a map during DCPEXT's power cycle read a TCR in the unpowered DART. - A DART that runs on its PCIe port's clock, as the T8140 radio port's does, gates itself in its noirq resume after a real sleep, until its host has the port back and calls apple_dart_host_resume(). The first real sleep with the radio host bound took an SError there. - apple_dart_state_save() and apple_dart_state_restore() keep a DART's state across its consumer's own power cycle, outside a system sleep. They return -ENODEV for a device apple-dart isn't bound to. DARTs with none of these properties, and other sleeps, behave as before. Signed-off-by: Diljit Singh --- drivers/iommu/apple-dart.c | 644 +++++++++++++++++++++++++- include/linux/soc/apple/dart.h | 57 +++ tools/asahi/apple-dart-fw/fixture.c | 2 +- tools/asahi/apple-dart-range-tests.py | 4 +- 4 files changed, 693 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d5b60c897a4c2..6596a6ff36990 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -262,6 +262,11 @@ struct apple_dart_fw_root { u64 *accepted; }; +struct apple_dart_sleep_list { + const u32 *pairs; + unsigned int count; +}; + /* * Private structure associated with each DART device. * @@ -278,11 +283,29 @@ struct apple_dart_fw_root { * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ +#define APPLE_DART_GAPF_PROTECT BIT(4) + struct apple_dart { struct device *dev; const struct apple_dart_hw *hw; void __iomem *regs; + void __iomem *gapf; + /* The GAPF slice as it read at probe; a deep sleep resets it to 0 */ + u32 gapf_boot; + bool sleep_flushed; + /* + * A display DART whose streams are off for a power-down: its registers may be + * unpowered, so maps use TCR as it was, and invalidations wait for the streams. + */ + bool display_dark; + bool display_tlb_stale; + DECLARE_BITMAP(display_tcr_four, DART_MAX_STREAMS); + /* The address filter the loader programs, from the device tree, replayed after a sleep. */ + void __iomem *dapf; + u64 *dapf_ranges; + u32 *dapf_r20; + unsigned int dapf_count; int irq; struct clk_bulk_data *clks; @@ -300,15 +323,21 @@ struct apple_dart { u32 tunneled : 1; u32 per_stream : 1; u32 power_retained : 1; + /* State a consumer saved for a firmware-driven suspend and will restore. */ + u32 sleep_saved : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; + /* Clocked by its host's port: the host resumes it after a real sleep. */ + u32 host_resumes : 1; bool commands_gated; /* Resume restored the translations, but the last steps of its reset failed. */ bool reset_incomplete; struct apple_tunable *tunables; + /* The bootloader sets these; written back where a power-off lost them. */ + struct apple_tunable *resume_tunables; u16 version; /* J713 internal display DARTs, borrowed from the running DCP. */ @@ -335,6 +364,14 @@ struct apple_dart { unsigned int dispext_users; wait_queue_head_t dispext_wq; u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ + /* The register writes around a firmware-driven suspend, from the device tree. */ + struct apple_dart_sleep_list sleep_prepare; + struct apple_dart_sleep_list sleep_restore; + struct apple_dart_sleep_list sleep_restore_late; + u32 sleep_enable[BITS_TO_U32(DART_MAX_STREAMS)]; + /* Its own config words at the sleep, which the restore keeps. */ + u32 sleep_cfg[20]; + /* Retained radio/display state left in place across suspend-to-idle. */ bool s2idle_kept; /* Resources reachable by a retained domain outlive device removal. */ @@ -566,21 +603,27 @@ static inline u32 apple_dart_readl(struct apple_dart *dart, u32 offset) return value; } -static void apple_dart_apply_tunables(struct apple_dart *dart) +/* Returns how many words it had to change. */ +static unsigned int apple_dart_apply_tunables(struct apple_dart *dart, + const struct apple_tunable *tunables) { + unsigned int changed = 0; size_t i; - for (i = 0; i < dart->tunables->sz; ++i) { + for (i = 0; i < tunables->sz; ++i) { u32 old_value, value; - old_value = apple_dart_readl(dart, - dart->tunables->values[i].offset); - value = old_value & ~dart->tunables->values[i].mask; - value |= dart->tunables->values[i].value; - if (value != old_value) + old_value = apple_dart_readl(dart, tunables->values[i].offset); + value = old_value & ~tunables->values[i].mask; + value |= tunables->values[i].value; + if (value != old_value) { apple_dart_writel(dart, value, - dart->tunables->values[i].offset); + tunables->values[i].offset); + changed++; + } } + + return changed; } /* @@ -2094,6 +2137,17 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) continue; + /* + * Its display is powered off: publish to memory only. With its streams off + * nothing walks the tables, and the TLB is invalidated before they're on. + */ + if (READ_ONCE(stream.dart->display_dark)) { + ret = stream.dart->locked ? + apple_dart_hw_sync_locked(cfg, &stream, true) : 0; + WRITE_ONCE(stream.dart->display_tlb_stale, true); + err = err ?: ret; + continue; + } ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { dev_err_ratelimited(stream.dart->dev, @@ -2224,7 +2278,10 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, } continue; } - four = apple_dart_readl(dart, DART_TCR(dart, sid)) & + if (READ_ONCE(dart->display_dark)) + four = test_bit(sid, dart->display_tcr_four); + else + four = apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level; idx = four ? 0 : addr >> (pgshift + 2 * bits); slot = (addr >> (pgshift + (four ? 2 : 1) * bits)) & @@ -4127,6 +4184,134 @@ static bool apple_dart_has_live_rtkit_client(struct device_node *np) return false; } +static void apple_dart_parse_sleep_list(struct apple_dart *dart, const struct resource *res, + const char *name, struct apple_dart_sleep_list *list) +{ + struct device_node *np = dart->dev->of_node; + unsigned int i; + u32 *pairs; + int count; + + count = of_property_count_u32_elems(np, name); + if (count <= 0) + return; + if (count % 2) { + dev_err(dart->dev, "%s: not a list of offset and value\n", name); + return; + } + pairs = devm_kcalloc(dart->dev, count, sizeof(*pairs), GFP_KERNEL); + if (!pairs || of_property_read_u32_array(np, name, pairs, count)) + return; + for (i = 0; i < count; i += 2) { + if (pairs[i] % 4 || pairs[i] + 4 > resource_size(res)) { + dev_err(dart->dev, "%s: bad offset %#x\n", name, pairs[i]); + return; + } + } + list->pairs = pairs; + list->count = count / 2; +} + +/* The config words a sleep list writes: 0x200 to 0x22c and 0x300 to 0x31c. */ +#define DART_SLEEP_CFG_WORDS 20 + +static unsigned int apple_dart_sleep_cfg_off(unsigned int i) +{ + return i < 12 ? 0x200 + 4 * i : 0x300 + 4 * (i - 12); +} + +static void apple_dart_sleep_cfg_read(struct apple_dart *dart, u32 *cfg) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + cfg[i] = readl(dart->regs + apple_dart_sleep_cfg_off(i)); +} + +/* + * The DART's address filter, which the loader programs and a real sleep clears: one entry + * per range, 0x40 apart, written in the order the loader writes them. + * apple,dapf-ranges gives start and end pairs from the ADT's dapf-instance-0, and the + * optional apple,dapf-r20 each entry's last word, 1 if absent. + */ +#define DART_DAPF_LOCK 0x208 +#define DART_DAPF_LOCKED BIT(2) +#define DART_DAPF_STRIDE 0x40 +#define DART_DAPF_CONFIG 0x31 + +static int apple_dart_dapf_parse(struct platform_device *pdev, struct apple_dart *dart) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf"); + int n = of_property_count_u64_elems(np, "apple,dapf-ranges"); + unsigned int i; + + if (n <= 0 || n % 2) + return dev_err_probe(dev, -EINVAL, "apple,dapf-ranges needs start and end pairs\n"); + dart->dapf_count = n / 2; + if (resource_size(res) < dart->dapf_count * DART_DAPF_STRIDE) + return dev_err_probe(dev, -EINVAL, "DAPF region too small for %u ranges\n", + dart->dapf_count); + + dart->dapf = devm_platform_ioremap_resource_byname(pdev, "dapf"); + if (IS_ERR(dart->dapf)) + return PTR_ERR(dart->dapf); + dart->dapf_ranges = devm_kcalloc(dev, n, sizeof(*dart->dapf_ranges), GFP_KERNEL); + dart->dapf_r20 = devm_kcalloc(dev, dart->dapf_count, sizeof(*dart->dapf_r20), GFP_KERNEL); + if (!dart->dapf_ranges || !dart->dapf_r20) + return -ENOMEM; + of_property_read_u64_array(np, "apple,dapf-ranges", dart->dapf_ranges, n); + for (i = 0; i < dart->dapf_count; i++) + dart->dapf_r20[i] = 1; + if (of_property_count_u32_elems(np, "apple,dapf-r20") == dart->dapf_count) + of_property_read_u32_array(np, "apple,dapf-r20", dart->dapf_r20, dart->dapf_count); + + return 0; +} + +/* Entry 0 as written: the end reads back with its low two bits set, the last word's bytes. */ +static bool apple_dart_dapf_holds(struct apple_dart *dart) +{ + return readq(dart->dapf + 0x08) == dart->dapf_ranges[0] && + (readq(dart->dapf + 0x10) | 3) == (dart->dapf_ranges[1] | 3); +} + +/* + * After a power-off, before any consumer's firmware starts. Not while it's kept, and not + * while it's locked. + */ +static void apple_dart_dapf_replay(struct apple_dart *dart) +{ + unsigned int i; + + if (!dart->dapf_count) + return; + if (apple_dart_dapf_holds(dart)) + return; + if (readl(dart->regs + DART_DAPF_LOCK) & DART_DAPF_LOCKED) { + dev_warn(dart->dev, "DAPF locked after the sleep: not replayed\n"); + return; + } + + for (i = 0; i < dart->dapf_count; i++) { + void __iomem *entry = dart->dapf + i * DART_DAPF_STRIDE; + + writel(0, entry + 0x04); + writeq(dart->dapf_ranges[2 * i], entry + 0x08); + writeq(dart->dapf_ranges[2 * i + 1], entry + 0x10); + writel(DART_DAPF_CONFIG, entry + 0x00); + writel(dart->dapf_r20[i], entry + 0x20); + } + /* Its consumer's firmware faults on its first access without it: say so plainly. */ + if (!apple_dart_dapf_holds(dart)) { + dev_err(dart->dev, "DAPF write-back didn't take: entry 0 reads %#llx..%#llx\n", + readq(dart->dapf + 0x08), readq(dart->dapf + 0x10)); + return; + } + dev_dbg(dart->dev, "DAPF: %u ranges written back after the sleep\n", dart->dapf_count); +} + static int apple_dart_probe(struct platform_device *pdev) { int ret, sid; @@ -4304,6 +4489,17 @@ static int apple_dart_probe(struct platform_device *pdev) dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "gapf")) { + dart->gapf = devm_platform_ioremap_resource_byname(pdev, "gapf"); + if (IS_ERR(dart->gapf)) + return PTR_ERR(dart->gapf); + dart->gapf_boot = readl(dart->gapf); + } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf")) { + ret = apple_dart_dapf_parse(pdev, dart); + if (ret) + return ret; + } if (tunneled) { dart->tunables = devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable", res); @@ -4324,6 +4520,14 @@ static int apple_dart_probe(struct platform_device *pdev) } } } + dart->resume_tunables = devm_apple_tunable_parse(dev, dev->of_node, + "apple,resume-tunable", res); + if (IS_ERR(dart->resume_tunables)) { + if (PTR_ERR(dart->resume_tunables) != -ENOENT) + return dev_err_probe(dev, PTR_ERR(dart->resume_tunables), + "failed to parse resume tunables\n"); + dart->resume_tunables = NULL; + } if (j713_dcpext || j713_dispext) { /* @@ -4370,7 +4574,7 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled && dart->tunables) { dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", dart->tunables->sz); - apple_dart_apply_tunables(dart); + apple_dart_apply_tunables(dart, dart->tunables); } if (tunneled) { @@ -4534,6 +4738,10 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-prepare", &dart->sleep_prepare); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore", &dart->sleep_restore); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore-late", + &dart->sleep_restore_late); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); /* Preserve our retained-root path when no reserved-table handoff exists. */ @@ -5038,11 +5246,135 @@ static void apple_dart_save_state(struct apple_dart *dart) } } +static void apple_dart_sleep_write(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, list->pairs[2 * i + 1], list->pairs[2 * i]); +} + +/* + * A sleep list's config words go back as this DART had them before the sleep, not as + * the list has them: Linux's own setup of those words can differ from macOS's. + */ +static u32 apple_dart_sleep_value(struct apple_dart *dart, u32 off, u32 val) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + if (apple_dart_sleep_cfg_off(i) == off) + return dart->sleep_cfg[i]; + return val; +} + +/* A sleep list, with its config words as the DART had them before the sleep. */ +static void apple_dart_sleep_write_kept(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, + apple_dart_sleep_value(dart, list->pairs[2 * i], + list->pairs[2 * i + 1]), + list->pairs[2 * i]); +} + +static void apple_dart_sleep_wait_tlb(struct apple_dart *dart) +{ + u32 status; + + if (readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT)) + dev_err(dart->dev, "TLB command still busy around a sleep: %#x\n", status); +} + +/** + * apple_dart_sleep_prepare() - Save a DART whose consumer is about to gate itself + * @dev: the DART + * + * For a DART with apple,sleep-restore in its node. A consumer calls this before + * telling its firmware to sleep, while the DART's state can be read, and + * apple_dart_sleep_restore() before starting that firmware again. The DART's own + * callbacks then leave it alone. Without the property nothing happens here, and + * they do what they always do. + * + * Return: 0, or -ENODEV if the DART driver isn't bound. + */ +int apple_dart_sleep_prepare(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + if (!dart) + return -ENODEV; + if (!dart->sleep_restore.count) + return 0; + + apple_dart_sleep_cfg_read(dart, dart->sleep_cfg); + spin_lock_irqsave(&dart->lock, flags); + apple_dart_sleep_write(dart, &dart->sleep_prepare); + apple_dart_sleep_wait_tlb(dart); + apple_dart_save_state(dart); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + dart->sleep_enable[i] = apple_dart_readl(dart, dart->hw->enable_streams + 4 * i); + dart->sleep_saved = true; + spin_unlock_irqrestore(&dart->lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_prepare); + +/** + * apple_dart_sleep_restore() - Put back a DART saved by apple_dart_sleep_prepare() + * @dev: the DART + * + * A real sleep loses the DART's registers. A suspend that was refused lost nothing, + * so nothing is written then. + */ +void apple_dart_sleep_restore(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx, i; + unsigned long flags; + + if (!dart || !dart->sleep_saved) + return; + + spin_lock_irqsave(&dart->lock, flags); + dart->sleep_saved = false; + if (pm_resume_via_firmware()) { + apple_dart_sleep_write_kept(dart, &dart->sleep_restore); + for (sid = 0; sid < dart->num_streams; sid++) { + if (!dart->save_tcr[sid] && !dart->save_ttbr[sid][0]) + continue; + apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + } + apple_dart_sleep_write_kept(dart, &dart->sleep_restore_late); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, dart->sleep_enable[i], + dart->hw->enable_streams + 4 * i); + apple_dart_sleep_wait_tlb(dart); + } + spin_unlock_irqrestore(&dart->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_restore); + static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); dart->tunnel_state_restored = false; + /* Its consumer saved this already and restores it: see apple_dart_sleep_prepare(). */ + if (dart->sleep_saved) + return 0; if (READ_ONCE(dart->commands_gated)) return 0; /* @@ -5178,7 +5510,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) int ret; /* No reset or restore: see apple_dart_suspend(). */ - if (dart->cold_ane) + if (dart->cold_ane || dart->sleep_saved) + return 0; + /* Its block can't answer yet: apple_dart_host_resume() restores it. */ + if (dart->host_resumes && READ_ONCE(dart->commands_gated)) return 0; /* * An adopted external display DART may be off now, its display with it, @@ -5252,12 +5587,146 @@ static __maybe_unused int apple_dart_resume(struct device *dev) ret = apple_dart_hw_finish_reset(dart); } + /* The address filter went with the same power-off, and so may these. */ + apple_dart_dapf_replay(dart); + if (dart->resume_tunables) { + unsigned int changed = apple_dart_apply_tunables(dart, dart->resume_tunables); + + if (changed) + dev_dbg(dev, "%u of %zu tunables written back\n", changed, + dart->resume_tunables->sz); + } + /* Should they fail again, the next resume repeats them. */ dart->reset_incomplete = ret; return ret; } -static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); +static __maybe_unused int apple_dart_resume_noirq(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + /* + * After a real sleep a DART clocked by its host's port reads nothing until the host + * sets the port up again. Gate it, so that nothing touches it, until then. + */ + if (dart->host_resumes && pm_resume_via_firmware()) { + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, true); + return pm_runtime_force_resume(dev); + } + + ret = pm_runtime_force_resume(dev); + + /* + * A real sleep took the registers even of a DART runtime PM had suspended, which + * force_resume leaves alone. Its consumer's firmware may DMA through it before + * anything resumes it at runtime, so write the saved state back now. Not one with its + * own sleep list, as the SEP's: its consumer restores that with macOS's writes before + * starting its firmware, and this would land on top. + */ + if (!ret && pm_resume_via_firmware() && pm_runtime_status_suspended(dev) && + !dart->sleep_restore.count) { + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after sleep failed: %d\n", ret); + else + dev_dbg(dev, "restored after sleep\n"); + } + if (pm_resume_via_firmware()) + apple_dart_dapf_replay(dart); + + return ret; +} + +/* The host will set this DART's port clock up again after a real sleep, and resume it then. */ +int apple_dart_set_host_resumes(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart) + return -EPROBE_DEFER; + if (dart->tunneled || dart->power_retained || dart->sleep_saved || dart->cold_ane) + return -EINVAL; + dart->host_resumes = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_set_host_resumes); + +/* The host's port is up again: ungate the DART and restore what it saved. */ +int apple_dart_host_resume(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + if (!dart || !dart->host_resumes) + return -EINVAL; + if (!READ_ONCE(dart->commands_gated)) + return 0; + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, false); + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after its host came back failed: %d\n", ret); + else + dev_info(dev, "restored after its host came back\n"); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_host_resume); + +/* + * Its consumer's own power cycle, outside a system sleep: the registers saved and put back. + * Only for a device apple-dart is bound to. + */ +int apple_dart_state_save(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_suspend(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_save); + +int apple_dart_state_restore(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_resume(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_restore); + +/* + * A firmware-driven sleep takes the DARTs in the noirq phase, which every device finishes + * before any early resume: coprocessors such as the PMP and SIO restart their firmware in + * their early resume, and its first DMA has to find the DART already restored. Any other + * sleep keeps the normal phase, which the PCIe-C tunnel DARTs need: their host stops and + * restarts the port clock in its own noirq callbacks. + */ +static __maybe_unused int apple_dart_sys_suspend(struct device *dev) +{ + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_suspend(dev); +} + +static __maybe_unused int apple_dart_sys_resume(struct device *dev) +{ + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_resume(dev); +} + +static __maybe_unused int apple_dart_sys_suspend_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? pm_runtime_force_suspend(dev) : 0; +} + +static __maybe_unused int apple_dart_sys_resume_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? apple_dart_resume_noirq(dev) : 0; +} + +static const struct dev_pm_ops apple_dart_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend, apple_dart_sys_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend_noirq, apple_dart_sys_resume_noirq) + RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) +}; static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -5376,6 +5845,157 @@ int apple_dart_dcp_dispext_state(struct device_node *consumer) EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_state); #endif +/* + * Bit 4 of this DART's GAPF slice makes the fabric reject writes to its coprocessor's + * CPU power state. It's clear from the flush until the streams come back on. + */ +static int apple_dart_gapf_release(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf) + return 0; + + val = readl(dart->gapf); + writel(val & ~APPLE_DART_GAPF_PROTECT, dart->gapf); + if (readl(dart->gapf) & APPLE_DART_GAPF_PROTECT) { + dev_err(dart->dev, "GAPF slice stayed protected: %#x\n", val); + return -EIO; + } + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); + + return 0; +} + +/* Back to the probe value: after a wake the slice reads 0, and protect alone would give 0x10. */ +static void apple_dart_gapf_protect(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf || !dart->sleep_flushed) + return; + + val = readl(dart->gapf); + writel(val | dart->gapf_boot | APPLE_DART_GAPF_PROTECT, dart->gapf); + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); +} + +/* Before its coprocessor's CPU domain is written: a protected slice would fault the write. */ +int apple_dart_gapf_ready(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart->gapf || !(readl(dart->gapf) & APPLE_DART_GAPF_PROTECT)) + return 0; + + return apple_dart_gapf_release(dart); +} +EXPORT_SYMBOL_GPL(apple_dart_gapf_ready); + +/* + * What macOS writes to T8140's display DARTs around a display power cycle: + * measured under aura, with the offsets the driver has no name for. It never + * writes their TCR or TTBR, and reads the same values back after each power-on, + * deep sleep included, so neither does this. + * + * Stopping again after a sleep that never reached the commit does nothing: the + * domains may be off by then. + */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long bits = streams; + unsigned int sid; + u32 status; + int ret; + + if (on) { + apple_dart_gapf_protect(dart); + dart->sleep_flushed = false; + /* Maps or unmaps while it was dark left their invalidations here. */ + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); + writel(0, dart->regs + 0x210); + writel(0, dart->regs + 0x4060); + writel(streams, dart->regs + DART_T8110_ENABLE_STREAMS); + return 0; + } + + if (flush && dart->sleep_flushed) + return 0; + + if (!READ_ONCE(dart->display_dark)) { + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + } + + writel(U32_MAX, dart->regs + 0x4060); + writel(0, dart->regs + 0x4000); + writel(U32_MAX, dart->regs + 0x704); + for_each_set_bit(sid, &bits, BITS_PER_TYPE(streams)) + writel(BIT(sid), dart->regs + DART_T8110_DISABLE_STREAMS); + if (!flush) + return 0; + + writel(0xa000, dart->regs + DART_T8110_TLB_CMD); + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT); + if (ret) { + dev_err(dev, "flush for a power-down didn't finish: %#x\n", status); + return ret; + } + ret = apple_dart_gapf_release(dart); + if (ret) + return ret; + dart->sleep_flushed = true; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_display_streams); + +/* + * A DART whose domain goes off with its consumer's, at runtime, while other users still map + * into it: dark, maps reach memory only and their invalidations wait; lit, those run. Its + * streams are left as they are. Call it dark while it's still powered. + */ +void apple_dart_power_dark(struct device *dev, bool dark) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid; + + if (dark) { + if (READ_ONCE(dart->display_dark)) + return; + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + return; + } + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); +} +EXPORT_SYMBOL_GPL(apple_dart_power_dark); + static struct platform_driver apple_dart_driver = { .driver = { .name = "apple-dart", diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index ce81d6ec5bcd3..dae08188ec158 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -141,12 +141,69 @@ int apple_dart_save_tunnel_state(struct device *dev); int apple_dart_restore_tunnel_state(struct device *dev); /* Registered J713 managed radio provider, with no owned valid root or fault. */ int apple_dart_radio_check_idle(struct device *dev); +/* + * A DART clocked by its host's port: after a real sleep it stays gated until the host has + * set the port up and calls apple_dart_host_resume(). + */ +int apple_dart_set_host_resumes(struct device *dev); +int apple_dart_host_resume(struct device *dev); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_radio_check_idle(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_set_host_resumes(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_host_resume(struct device *dev) { return -EOPNOTSUPP; } +#endif + +/* For a consumer whose firmware gates itself around a firmware-driven suspend. */ +#if IS_REACHABLE(CONFIG_APPLE_DART) +int apple_dart_sleep_prepare(struct device *dev); +void apple_dart_sleep_restore(struct device *dev); +/* A display DART's streams around a display power cycle, see the DART driver. */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush); +/* A GAPF slice released before its coprocessor's CPU domain comes up after a wake. */ +int apple_dart_gapf_ready(struct device *dev); +/* A display DART whose domain goes off at runtime while others still map into it. */ +void apple_dart_power_dark(struct device *dev, bool dark); +/* A DART's state kept across its consumer's own power cycle, outside a system sleep. */ +int apple_dart_state_save(struct device *dev); +int apple_dart_state_restore(struct device *dev); +#else +static inline int apple_dart_sleep_prepare(struct device *dev) +{ + return -ENODEV; +} + +static inline void apple_dart_sleep_restore(struct device *dev) +{ +} + +static inline int apple_dart_display_streams(struct device *dev, u32 streams, bool on, + bool flush) +{ + return -ENODEV; +} + +static inline int apple_dart_gapf_ready(struct device *dev) +{ + return 0; +} + +static inline void apple_dart_power_dark(struct device *dev, bool dark) +{ +} + +static inline int apple_dart_state_save(struct device *dev) +{ + return -ENODEV; +} + +static inline int apple_dart_state_restore(struct device *dev) +{ + return -ENODEV; +} #endif #endif diff --git a/tools/asahi/apple-dart-fw/fixture.c b/tools/asahi/apple-dart-fw/fixture.c index 13e39fed942d5..c684d3c210435 100644 --- a/tools/asahi/apple-dart-fw/fixture.c +++ b/tools/asahi/apple-dart-fw/fixture.c @@ -50,7 +50,7 @@ static unsigned lock_depth; static size_t find_next_bit(unsigned long*p,size_t n,size_t a){for(;a Date: Sat, 7 Nov 2026 07:40:13 -0500 Subject: [PATCH 26/73] soc: apple: sart: restore entries after a firmware-driven suspend The SART sits in its coprocessor's power domain, which a firmware-driven suspend powers off, and the driver has no sleep code. A coprocessor that restarts with an empty SART can't reach its firmware's buffers. Save the entries in suspend_late, after the consumers have gone quiet, and write back every one that was in use in resume_noirq, the same or not, as macOS rewrites entries 0 to 7 at every wake. This includes the bootloader's protected entries: Linux doesn't change those, it only puts back what power loss took. macOS leaves unused entries at their power-on flags, and so does this. macOS writes the entries back once more right before ANS's run bit, config first, then address, then size. apple_sart_replay() does that on SARTv3, for the coprocessor's driver to call there, and warns about an entry that doesn't read back. Only a firmware-driven resume restores anything, so suspend-to-idle and every SoC that doesn't advertise deep sleep are unchanged. The SART also has throttle registers, which this doesn't touch since nothing in Linux knows their layout. Signed-off-by: Diljit Singh --- drivers/soc/apple/sart.c | 104 +++++++++++++++++++++++++++++++++ include/linux/soc/apple/sart.h | 6 ++ 2 files changed, 110 insertions(+) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 6cedcb867c61b..c7e238e1d92bf 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "sart-backend.h" @@ -85,6 +86,16 @@ struct apple_sart { unsigned long used_entries; const struct apple_sart_backend *backend; void *backend_data; + + /* The registers as they stood before a firmware-driven suspend */ + struct { + u8 flags; + phys_addr_t paddr; + size_t size; + } saved[APPLE_SART_MAX_ENTRIES]; + bool saved_valid; + /* saved[] still holds the last sleep's entries, for apple_sart_replay() */ + bool replay_valid; }; struct apple_sart *apple_sart_create_backend(struct device *dev, @@ -408,6 +419,98 @@ static void apple_sart_shutdown(struct platform_device *pdev) } } +/* + * A SART in a domain that a firmware-driven suspend powers off comes back empty, + * and its coprocessor can't reach its firmware or buffers until the entries are + * back. This includes the bootloader's, which Linux otherwise never writes. + */ +static int apple_sart_suspend_late(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + int i; + + sart->saved_valid = false; + sart->replay_valid = false; + if (pm_suspend_target_state != PM_SUSPEND_MEM || !pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) + sart->ops->get_entry(sart, i, &sart->saved[i].flags, + &sart->saved[i].paddr, &sart->saved[i].size); + sart->saved_valid = true; + + return 0; +} + +static int apple_sart_resume_noirq(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + unsigned int restored = 0; + int i; + + if (!sart->saved_valid || !pm_resume_via_firmware()) + return 0; + sart->saved_valid = false; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + /* An unused entry stays as the power left it, whatever that reads. */ + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + /* Every one in use, the same or not: macOS rewrites 0 to 7 at every wake. */ + sart->ops->set_entry(sart, i, sart->saved[i].flags, + sart->saved[i].paddr >> sart->ops->paddr_shift, + sart->saved[i].size >> sart->ops->size_shift); + restored++; + } + dev_dbg(dev, "%u entries restored after sleep\n", restored); + sart->replay_valid = true; + + return 0; +} + +/* + * A coprocessor's driver calls this right before its run bit. macOS writes the entries back + * there, config first, then address, then size, so this does the same on SARTv3. Unused + * entries stay as they read. + */ +int apple_sart_replay(struct apple_sart *sart) +{ + int i, bad = 0; + + if (sart->backend || sart->ops != &sart_ops_v3 || !sart->replay_valid) + return -EOPNOTSUPP; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + phys_addr_t paddr; + size_t size; + u8 flags; + + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + writel(sart->saved[i].flags, sart->regs + APPLE_SART3_CONFIG(i)); + writel(sart->saved[i].paddr >> APPLE_SART3_PADDR_SHIFT, + sart->regs + APPLE_SART3_PADDR(i)); + writel(sart->saved[i].size >> APPLE_SART3_SIZE_SHIFT, + sart->regs + APPLE_SART3_SIZE(i)); + sart->ops->get_entry(sart, i, &flags, &paddr, &size); + if (flags == sart->saved[i].flags && paddr == sart->saved[i].paddr && + size == sart->saved[i].size) + continue; + bad++; + dev_warn(sart->dev, "entry %d reads %#x %pa+%#zx, wrote %#x %pa+%#zx\n", + i, flags, &paddr, size, sart->saved[i].flags, &sart->saved[i].paddr, + sart->saved[i].size); + } + + return bad; +} +EXPORT_SYMBOL_GPL(apple_sart_replay); + +static const struct dev_pm_ops apple_sart_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_sart_suspend_late, NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, apple_sart_resume_noirq) +}; + static const struct of_device_id apple_sart_of_match[] = { { .compatible = "apple,t6000-sart", @@ -429,6 +532,7 @@ static struct platform_driver apple_sart_driver = { .driver = { .name = "apple-sart", .of_match_table = apple_sart_of_match, + .pm = pm_sleep_ptr(&apple_sart_pm_ops), }, .probe = apple_sart_probe, .shutdown = apple_sart_shutdown, diff --git a/include/linux/soc/apple/sart.h b/include/linux/soc/apple/sart.h index f5d0db5022831..53186e759e55d 100644 --- a/include/linux/soc/apple/sart.h +++ b/include/linux/soc/apple/sart.h @@ -64,4 +64,10 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, size_t size); +/* + * After a firmware-driven sleep, writes the entries in use before it back again and reads + * each back. Returns how many didn't hold, or -EOPNOTSUPP. + */ +int apple_sart_replay(struct apple_sart *sart); + #endif /* _LINUX_SOC_APPLE_SART_H_ */ From 113be2304f36949ce39b1504997b04812a1debd9 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:13 -0500 Subject: [PATCH 27/73] soc: apple: mcc: write the system level cache back before a deep sleep apple-mcc enables system level cache allocation at probe and has no PM ops. A deep sleep takes the cache's power with the SoC, and nothing wrote it back first or enabled allocation again after the wake. In the noirq phase of a deep sleep allocation stops and every plane is written back and invalidated, as at shutdown. After the wake allocation comes back, or stays off with a warning if a plane isn't ready. Suspend-to-idle keeps the cache, so it's left alone. Signed-off-by: Diljit Singh --- drivers/soc/apple/mcc.c | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 3b9b47a6d6213..12d40d05b305f 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) #include @@ -39,6 +41,8 @@ struct apple_mcc { unsigned int nr_planes; bool enabled; bool retained; + /* Turned off for a deep sleep, to be turned back on after it. */ + bool sleep_disabled; }; static void apple_mcc_writel(struct apple_mcc *mcc, unsigned int plane, @@ -236,6 +240,43 @@ static void apple_mcc_shutdown(struct platform_device *pdev) apple_mcc_quiesce(mcc); } +/* + * A deep sleep takes the cache's power with the rest of the SoC: stop allocating and write it + * back first, as at shutdown, and allocate again after. Suspend-to-idle keeps it. + */ +static int apple_mcc_suspend_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + + if (!pm_suspend_via_firmware() || !mcc->enabled || mcc->retained) + return 0; + apple_mcc_quiesce(mcc); + mcc->sleep_disabled = true; + + return 0; +} + +static int apple_mcc_resume_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + int ret; + + if (!mcc->sleep_disabled) + return 0; + mcc->sleep_disabled = false; + /* Only a performance setting: a cache not ready after the wake stays off. */ + ret = apple_mcc_enable(mcc); + mcc->enabled = !ret; + if (ret) + dev_warn(dev, "system level cache allocation stays off after the sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_mcc_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_mcc_suspend_noirq, apple_mcc_resume_noirq) +}; + static const struct of_device_id apple_mcc_of_match[] = { { .compatible = "apple,t8132-mcc" }, { .compatible = "apple,t8140-mcc" }, @@ -247,6 +288,7 @@ static struct platform_driver apple_mcc_driver = { .driver = { .name = "apple-mcc", .of_match_table = apple_mcc_of_match, + .pm = pm_sleep_ptr(&apple_mcc_pm_ops), }, .probe = apple_mcc_probe, .remove = apple_mcc_remove, From cc68b50b266d649f7e8ca3a98e9355fa21c4d552 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:14 -0500 Subject: [PATCH 28/73] rust: platform: let a driver give its PM callbacks A Rust platform driver has no way to set driver.pm, so one that has to run in a particular suspend phase, or to refuse a suspend, can't. Add a pm_ops() method that defaults to none, and helpers for pm_suspend_via_firmware() and pm_resume_via_firmware(), which a driver needs in order to tell a firmware-driven suspend from any other. No driver changes behaviour here. Signed-off-by: Diljit Singh --- rust/helpers/helpers.c | 1 + rust/helpers/suspend.c | 13 +++++++++++++ rust/kernel/platform.rs | 6 ++++++ 3 files changed, 20 insertions(+) create mode 100644 rust/helpers/suspend.c diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f763d5541472..389c67d5740ed 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -102,6 +102,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "suspend.c" #include "srcu.c" #include "sync.c" #include "task.c" diff --git a/rust/helpers/suspend.c b/rust/helpers/suspend.c new file mode 100644 index 0000000000000..a616907ecc0b5 --- /dev/null +++ b/rust/helpers/suspend.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_pm_suspend_via_firmware(void) +{ + return pm_suspend_via_firmware(); +} + +__rust_helper bool rust_helper_pm_resume_via_firmware(void) +{ + return pm_resume_via_firmware(); +} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 1f845937899db..7bce2fe29e841 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -80,6 +80,7 @@ unsafe impl driver::RegistrationOps for Adapter { (*pdrv.get()).driver.of_match_table = of_table; (*pdrv.get()).driver.acpi_match_table = acpi_table; (*pdrv.get()).driver.suppress_bind_attrs = T::suppress_bind_attrs(); + (*pdrv.get()).driver.pm = T::pm_ops().map_or(core::ptr::null(), core::ptr::from_ref); } // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. @@ -230,6 +231,11 @@ pub trait Driver { false } + /// The driver's power management callbacks, if it has any. + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + None + } + /// Platform driver probe. /// /// Called when a new platform device is added or discovered. From 3bd4bd1bf83573d9f4c42d988c84ccff56717743 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:14 -0500 Subject: [PATCH 29/73] soc: apple: set the CPU clusters' power-gate enables at boot Each CPU cluster has a PMGR word whose top bit lets its cores power down in WFI. A core that PMGR powers on comes up allowed to, but the boot CPU was started by the boot ROM and only follows the word, which iBoot leaves clear for T8140's P cluster. Without it the boot CPU only clock-gates in WFI, and a system sleep's engine, which waits for every core to power down, never starts: measured on a J700, the watchdog reset the machine 5 s into every sleep until this bit was set, and with it set the boot CPU power-gates in idle too. An early initcall sets the bit in every word an apple,cluster-pwrgate node lists, before the secondaries start. Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 10 +++++ drivers/soc/apple/Makefile | 2 + drivers/soc/apple/apple-cluster-pwrgate.c | 50 +++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 drivers/soc/apple/apple-cluster-pwrgate.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 4639f5b8ab9bf..a689f1da04e11 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -69,6 +69,16 @@ config APPLE_PMGR_DCS_KUNIT_TEST Also verify that boot constraints and concurrent endpoints retain their votes, and that suspend restores the last confirmed request. +config APPLE_CLUSTER_PWRGATE + bool "Apple SoC CPU cluster power gates" + depends on OF + default ARCH_APPLE + help + Sets the power-gate enable of each CPU cluster the device tree + lists, early in boot. Without it, a core that the boot loader + started and PMGR never powered on, such as the boot CPU, only + clock-gates in WFI. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 57d06930401c4..72a256f8d292c 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -8,6 +8,8 @@ apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o +obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/apple-cluster-pwrgate.c b/drivers/soc/apple/apple-cluster-pwrgate.c new file mode 100644 index 0000000000000..67d493cff8af6 --- /dev/null +++ b/drivers/soc/apple/apple-cluster-pwrgate.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC CPU cluster power gates + * + * Each CPU cluster has a PMGR word whose top bit lets its cores power down + * in WFI. A core PMGR powers on comes up allowed to, but the boot CPU was + * started by the boot ROM, so it only follows the word, which the boot + * loader may leave clear. Set it for every cluster the device tree lists. + */ + +#include +#include +#include +#include +#include +#include + +#define CLUSTER_PWRGATE_EN BIT(31) + +static const struct of_device_id apple_cluster_pwrgate_of_match[] __initconst = { + { .compatible = "apple,cluster-pwrgate" }, + { } +}; + +/* Before the secondaries start. A core picks the word up at its next WFI. */ +static int __init apple_cluster_pwrgate_init(void) +{ + struct device_node *np; + + for_each_matching_node(np, apple_cluster_pwrgate_of_match) { + void __iomem *gate; + int i; + + if (!of_device_is_available(np)) + continue; + + /* One word per cluster. Shared with the sleep code, so not claimed. */ + for (i = 0; (gate = of_iomap(np, i)); i++) { + u32 was = readl(gate); + + writel(was | CLUSTER_PWRGATE_EN, gate); + pr_debug("%pOF: cluster %d power gate %#x, now %#x\n", np, i, was, + readl(gate)); + iounmap(gate); + } + } + + return 0; +} +early_initcall(apple_cluster_pwrgate_init); From d43118bcb473bec083536a3da7ce46d480fdb196 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:15 -0500 Subject: [PATCH 30/73] soc: apple: add T8140 suspend-to-RAM at EL2 On T8140 a kernel can run directly at EL2 behind the CTXR execute window, with no monitor below it. No PSCI answers a request to power a CPU off or to suspend the system, so the kernel does both itself. iBoot locks every core's reset vector to the boot object, which opts in by publishing a resume mailbox in /chosen/aurora,s2r-mailbox. That also gives its spin-table CPUs the parking spin table. Without it, nothing here binds. The sleep controller does what macOS does in its last 14 ms, in its order, with the register blocks taken from the device tree: - CPU power control for the parking spin table: a core's stop bit, its deep WFI with the timers masked, and its start words. - Refusal checks before anything is stopped: one CPU up, the display coprocessor's driver bound and its firmware stopped, every registered coprocessor quiesced by its driver. T8140's coprocessors are named in the driver: both DCPs, the PMP, SIO, ANS, the MTP, AOP, SMC, SEP and the GPU have to be bound, and both DCPs, the PMP and SIO stopped by the last step, as a commit with the PMP and SIO still running reset the SoC 5.5 s later. An enclave that runs with no driver bound, or that didn't gate itself, refuses the sleep too: a commit with it running never returned. A display CPU whose domain is off isn't read, as DPMS off leaves it. Everything wrong is named, so one boot shows the whole list. An enclave whose request is left to the last refusal, below, isn't expected to have gated yet. - A registry for a coprocessor that can't be stopped until the sleep is certain and can't restart without one. Its last stop runs after every refusal, with interrupts off, and can't refuse: from the first last stop only a real sleep restarts the GPU, so a failure there goes on to the commit. What may still refuse goes in a last check, apple_sleep_coproc_set_last_check(), run with interrupts off after the display is down and before any last stop. One registered with apple_sleep_coproc_register_first() clears its own run bits and lowers its own domain, and stops in a first pass, before macOS's off words drop under it: the GPU, which withdraws its PMP request there, before the PMP's own stop. apple_sleep_coproc_stopped() tells a driver after the sleep whether its last stop ran. ANS's driver will register too; until its PCIe block can come back (next commit), ANS on refuses the sleep. - The enclave's sleep request, the very last refusal. A gated enclave comes back only through a real sleep, so a sleep refused anywhere before has to find it never told. Its driver registers the request with apple_sleep_enclave_register(), and it runs with interrupts off after the last checks and before any coprocessor stops; its own failure still refuses with everything else running. - The order in .enter: the refusal checks; the display off, before the GPU as macOS has it (the display is down about a second before the commit there, GFX's power word at 174 ms); the last checks; the enclave's request; the first pass of last stops; the wake masks and the match; the second pass. - What XNU polls off just before its complex words and panics on: AVD, VENC0/1, GFX and JPG still on at the last step are lowered there, and a sleep with one that won't go off is refused. They come back on after the sleep or a refusal, for genpd's books. - The wake-source mask and the display domains. USB's ATC0 domains are lowered at the commit, deepest first, as macOS has them off then; a USB host that sleeps in place keeps ATC0_USB_AON, as macOS's does, which its PHY says with apple_sleep_usb_set_asleep(). That host's glue also says which of the fixed hub's ports have a device that can wake, with apple_sleep_usb_set_wake_ports(), for the port controllers that arm USB wake. - The debug link: the port controller that carries it is cold-reset and told S3, which takes the debug domains to macOS's state. To the SMC that's a cable pulled, and it reports the port's events as wake reasons, so the SMC's driver cuts it with apple_sleep_debug_link_cut() before its AP idle notice; the cut in .enter covers a sleep with no SMC driver bound. Only the debug port's controller takes the cold reset, so the driver keeps the node apple,debug-port names, and apple_sleep_debug_link_was_cut() and apple_sleep_debug_link_cut_peek() answer only for the device on it. The cut is cleared at the next .begin, after the last sleep's .complete has had it. - The last mile: WD2, the PMU's boot stage as macOS leaves it, the SoC's perf state after the ten domains XNU polls have been waited for, DCS to its sleep state with the voters at 0, the memory controller's channel step-down, macOS's power words and last writes, the CLLT and DVFM words, the AMCC error enables that would turn a sleep around, the complex words, the commit, and every AIC line masked before the boot core's deep WFI. - The return: timer control, IRQ routing, the clusters, every power word put back as it was at the commit before any driver resumes, the SEP's disable bit, the AVE complex off, DVFM, and a 120 s WD2 over device resume until .end. - After a refused sleep, apple_sleep_engine_reset() restarts a firmware whose CPU domain shares a power switch and can't be power-cycled alone, as SIO's: the sleep engine's own PMGR reset of the device's domains, through targets 4 and 0, released with the domain off and raised again, its DARTs' state kept, each found in iommus by #iommu-cells. Measured on a J700 without a sleep, SIO's mailbox came back at its reset value and its firmware started. The named register blocks are mapped with devm_ioremap(), still unclaimed, and the driver publishes itself only once its probe can't fail. Once the controller binds, deep is the default sleep state, through the usual suspend_set_ops() path, so mem_sleep_default=s2idle still keeps s2idle. Without the boot object's mailbox it logs that once and doesn't bind, and s2idle stays the only sleep. Measured on a J700 with the boot object's mailbox: sleep holds, the RTC wakes it, and resume completes. Signed-off-by: Diljit Singh --- arch/arm64/configs/asahi.config | 1 + drivers/soc/apple/Kconfig | 10 + drivers/soc/apple/Makefile | 2 + drivers/soc/apple/t8140-sleep.c | 2458 +++++++++++++++++++++++++++++++ include/linux/soc/apple/sleep.h | 168 +++ 5 files changed, 2639 insertions(+) create mode 100644 drivers/soc/apple/t8140-sleep.c create mode 100644 include/linux/soc/apple/sleep.h diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 17f191d0e35a1..d10b2e1ab7b77 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -155,3 +155,4 @@ CONFIG_EDAC_APPLE_SOC=m # unless the bootloader supplies a window (/chosen/aurora,ctxr-exec-window) or # the ctxr_exec_window= command-line option is given. CONFIG_ARM64_CTXR_EXECMEM=y +CONFIG_APPLE_T8140_SLEEP=y diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index a689f1da04e11..a3c46e7ba313e 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -79,6 +79,16 @@ config APPLE_CLUSTER_PWRGATE started and PMGR never powered on, such as the boot CPU, only clock-gates in WFI. +config APPLE_T8140_SLEEP + bool "Apple T8140 system sleep at EL2" + depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) + depends on HOTPLUG_CPU && SUSPEND + help + Powers the secondary CPUs off and suspends the system to RAM on an + Apple T8140 whose kernel runs directly at EL2, with no monitor to ask. + It does nothing unless the boot object asks for it through + /chosen/aurora,s2r-mailbox. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 72a256f8d292c..73d102ddd8771 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -10,6 +10,8 @@ obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o +obj-$(CONFIG_APPLE_T8140_SLEEP) += t8140-sleep.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c new file mode 100644 index 0000000000000..e787b3bede8aa --- /dev/null +++ b/drivers/soc/apple/t8140-sleep.c @@ -0,0 +1,2458 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T8140 system sleep + * + * On T8140 a kernel can run directly at EL2 behind the CTXR execute window + * (Documentation/arch/arm64/ctxr-execmem.rst), with no monitor below it. So no + * PSCI answers a request to power a CPU off or to suspend the system, and the + * kernel does both itself. + * + * iBoot locks every core's reset vector to the boot object. A core that comes + * back from a power-off therefore restarts inside the boot object (aura), and + * that has to send it back here. The boot object opts in by publishing a mailbox + * in /chosen/aurora,s2r-mailbox, which also gives its spin-table CPUs the parking + * spin table (arch/arm64/kernel/cpu_ops.c). Without it this driver does nothing: + * + * - A secondary core goes back through its spin-table release address, which + * the boot object keeps, into secondary_holding_pen. + * - The boot core takes cpu_resume from the mailbox: EL2, MMU off. + * + * The register addresses are in the device tree. The ADT has every one of them, + * as /arm-io/pmgr's or /arm-io/wdt's reg entries. Register offsets and order not + * otherwise explained here were measured from a macOS sleep under aura. + * + * What the kernel does in suspend-to-RAM is what macOS does in its last 14 ms, + * with the order kept. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Must match aura's struct aura_s2r_mailbox. */ +#define T8140_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ + +struct t8140_s2r_mailbox { + u64 magic; /* aura, before the kernel starts */ + u64 boot_mpidr; /* aura: the only core that may take resume_pa */ + u64 resume_pa; /* kernel, armed before a commit; aura clears it */ + u64 resumes; /* aura: resume_pa taken this many times */ +}; + +/* PS word */ +#define PS_AUTO_ENABLE BIT(28) +#define PS_DEV_DISABLE BIT(10) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_OFF_TIMEOUT_US 5000 + +#define PS_DISP_CPU 0x10000 +#define PS_DISP_FE 0x318 +#define PS_DISP_SYS 0x280 +#define PS_DISPEXT0_CPU 0x328 +#define PS_DISPEXT0_FE 0x320 +#define PS_DISPEXT0_SYS 0x288 +#define PS_SIO_CPU 0x2a8 +#define PS_SEP 0xc00 +#define PS_ATC0_USB_AON 0x3e0 +#define PS_ATC0_COMMON 0x3e8 +#define PS_ATC0_USB 0x3f0 + +/* CPU start block, as aura's smp_start_cpu() uses it */ +#define CPU_START_STOP 0x0 +#define CPU_START_SYS 0x4 +#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) + +/* Power-gate page: reads back this value already. Purpose unknown. */ +#define PWRGATE_ON 0x80000000 +#define PWRGATE_WAKE_0 0x000 +#define PWRGATE_WAKE_1 0x040 +#define PWRGATE_CPM_0 0x060 +#define PWRGATE_CPM_1 0x068 +#define PWRGATE_DISP_0 0xc8c +#define PWRGATE_DISP_1 0xcd0 + +/* Three clock muxes macOS writes before the commit. */ +#define CLK_MUX_PARKED 0x80100000 +#define CLK_MUX_34 0x0 +#define CLK_MUX_38 0x4 +#define CLK_MUX_3C 0x8 + +/* Cluster control: only these three words, the next one is locked. */ +#define CLUSTER_E 0x0 +#define CLUSTER_P 0x4 +#define CLUSTER_AVE 0x8 +#define CLUSTER_ON 0xf +/* Set when a complex's up sequence ends, and cleared by writing 1. */ +#define CLUSTER_UP_DONE BIT(5) +#define CLUSTER_E_READY BIT(5) +#define CLUSTER_TIMEOUT_US 5000 + +/* CLLT: names from nothing. Bits measured cleared by macOS before the commit. */ +#define CLLT_0 0x30 +#define CLLT_1 0x34 +#define CLLT_2 0x3c +#define CLLT_0_CLEAR GENMASK(7, 0) +#define CLLT_1_CLEAR GENMASK(7, 0) +#define CLLT_2_CLEAR GENMASK(17, 16) + +/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ +#define WDT_WD0_COUNT 0x00 +#define WDT_WD0_CTL 0x0c +#define WDT_WD1_COUNT 0x10 +#define WDT_WD2_COUNT 0x20 +#define WDT_WD2_ALARM 0x28 +#define WDT_WD2_CTL 0x2c +#define WDT_TICKS_PER_S 24000000 +/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ +#define WDT_SLEEP_SECS 5 +/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ +#define WDT_SUSPEND_SECS 120 + +#define AON_AP_DEADLINE 0x08 + +/* Wake sources: a mask, so 0 enables. Status words are sticky. */ +#define WAKE_MASK(word) (0x08 + (word) * 4) +#define WAKE_STATUS(word) (0x28 + (word) * 4) +#define WAKE_WORDS 8 +/* Word 7 has 21 sources. Measured: macOS reads 0x001fef77 there. */ +#define WAKE_WORD7_ALL GENMASK(20, 0) +#define WAKE_SOURCES_MAX 32 + +/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ +#define SPMI_STATUS 0x00 +#define SPMI_STATUS_RX_EMPTY BIT(24) +#define SPMI_CMD 0x04 +#define SPMI_REPLY 0x08 +#define SPMI_NOTIFY_ENABLE 0x40 +#define SPMI_NOTIFY_ENABLE_SLEEP 0x2000001 +#define SPMI_NOTIFY_REPLY BIT(0) +#define SPMI_NOTIFY_ACK 0x80 +#define SPMI_CMD_ACTIVE BIT(15) +#define SPMI_OP_EXT_WRITE(n) ((n) - 1) +#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) +#define SPMI_OP_ZERO_WRITE 0x80 +#define SPMI_OP_SLEEP 0x11 +#define SPMI_OP_WAKEUP 0x13 +#define SPMI_POLL_US 20000 + +/* A port controller on SPMI: registers go through a page select, then a data window. */ +#define HPM_REG_SELECT 0x00 +#define HPM_REG_DATA 0x20 +#define HPM_REG_DATA_CMD 0xa0 +#define HPM_PAGE_CMD1 0x08 +#define HPM_PAGE_DATA1 0x09 +#define HPM_CMD_SSPS 0x53505353 +#define HPM_CMD_COLD_RESET 0x64696147 +#define HPM_SSPS_S3 3 +#define HPM_RESET_MS 1000 + +/* A PMU register write on the sleep's SPMI bus */ +#define SPMI_CMD_WRITE (BIT(15) | (0xe << 8) | 0x30) +#define PMU_BOOT_STAGE 0xf701 +#define PMU_BOOT_STAGE_SLEEP 0x50 + +/* Three of the memory controller's channels, and what macOS steps down on each. */ +#define MCC_CHANNELS 3 +#define MCC_ENABLES 0x4 +#define MCC_LANES 0x8 +#define MCC_LANES_MASK 0xc +#define MCC_LANE_STROBE(n) BIT(16 + 4 * (n)) +#define MCC_LANE(n) BIT(4 * (n)) + +/* Two words macOS zeroes just before the power gates. */ +#define PTD_A 0x0 +#define PTD_B 0x8 + +/* Perf-state words, each like DCS's. The SoC's is first. */ +#define CLK_PERF_SOC 0x0 +#define SOC_SLEEP_STATE 3 + +/* An always-on word that a live debug link holds up. macOS has it off. */ +#define PS_AON_DEBUG_SWITCH 0xb0 + +/* The PMC's voter agents, each with a word holding its floor for every rail. */ +#define PMC_VOTER_AGENTS 4 +#define PMC_VOTER_STRIDE 0x4000 + +/* DCS: the low nibble asks for a state, and [11:8] is the state it has reached. */ +#define DCS_STATE GENMASK(11, 8) +#define DCS_BUSY BIT(31) +#define DCS_SLEEP_STATE 4 +#define DCS_STEP_STATE 6 +#define DCS_TIMEOUT_US 5000 + +/* Word 0 of the wake block. macOS clears it at boot, and sets it only for a device's DDR use. */ +#define WAKE_SLP_DDR 0x00 + +/* + * AMCC error enables. Any error enabled and pending at the commit turns the + * sleep around within 20 ms with no error handler to service it. Linux has none. + * Measured: with macOS's handler hidden, as Linux's position is, 6 of 6 sleeps + * held with both of these off, and bit 25 alone still bounced. Their status + * stays set; only the enables matter. A deep sleep sets them again, so this has + * to be done before every commit. + */ +#define AMCC_WORD0_ENABLE 0x0 +#define AMCC_WORD1_ENABLE 0x8 +#define AMCC_WORD0_DSID_AGE_OUT BIT(25) +#define AMCC_WORD1_UNEXP_RT_HIT BIT(8) + +/* A coprocessor's ASC */ +#define ASC_CPU_CONTROL 0x44 +#define ASC_CPU_CONTROL_RUN BIT(4) +#define ASC_CPU_STATUS 0x48 + +#define PS_ANS 0x278 +#define PS_APCIE_SYS_GP 0x2a0 + +/* The coprocessors the last mile stops. Each driver idles its own first. */ +#define SLEEP_COPROCS 4 + +static struct t8140_sleep_coproc { + struct device *dev; + void __iomem *asc; + bool pcie_ans; + int (*last_check)(void *data); + int (*last_stop)(void *data); + void *data; + bool ready; + bool stopped; + u32 status; + /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ + bool first; + u32 ps; +} t8140_sleep_coprocs[SLEEP_COPROCS]; +static DEFINE_MUTEX(t8140_sleep_coprocs_lock); + +/* The enclave's sleep request, left to the last refusal. Under t8140_sleep_coprocs_lock. */ +static struct { + struct device *dev; + int (*tell)(void *data); + void *data; +} t8140_sleep_enclave; + +/* + * Written with bits 0 and 63 by every core right before its last WFI, measured + * from macOS. This is not CYC_OVRD, which is undefined on this SoC. + */ +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) + +/* The names of ADT pmgr/ap-wake-sources, for the log only */ +static const struct { + u32 id; + const char *name; +} t8140_wake_names[] = { + { 0, "AOP.AICWakeupAP" }, { 5, "AOP.CPUWakeupAP" }, + { 42, "AOP.OutboxNotEmpty" }, { 62, "AOP.SPMI0.SW3" }, + { 66, "AOP.SPMI1.SW3" }, { 70, "ATC0_PMGR_USB_WAKEUP" }, + { 80, "MTP.DOCK.CHANNELS.AP0.IRQ" }, { 99, "MTP.OUTBOX.N.EMPTY_IRQ" }, + { 109, "NUB.APWakeupTime" }, { 110, "NUB.APWatchdog" }, + { 125, "NUB.GPIO.IRQ6" }, { 131, "NUB.SPMI0.SW3" }, + { 136, "NUB.SPMI1.SW3" }, { 140, "NUB.WakeupAP" }, + { 145, "SMC.CPUWakeupAP" }, { 165, "SMC.OutboxNotEmpty" }, + { 211, "AOP.TrustedOutboxNotEmpty" }, { 227, "AOP2.OutboxNotEmpty" }, + { 231, "AOP2.TrustedOutboxNotEmpty" }, { 236, "AOP2.CPUWakeupAP" }, +}; + +#define T8140_DISPLAY_COUNT 2 + +/* CPU clusters: 0 is E, 1 is P. ADT /arm-io/pmgr reg[5], [7], [9] and [14], [16], [18]. */ +#define T8140_CLUSTERS 2 + +/* cpm-dvfm */ +#define DVFM_CTL 0x178 +#define DVFM_WORD_0 0x140 +#define DVFM_WORD_1 0x160 +#define DVFM_WORD_2 0x190 +#define DVFM_WORD_3 0x198 +#define DVFM_ENABLE BIT_ULL(63) + +/* In macOS's order, on the way down. */ +static const u32 t8140_dvfm_words[] = { + DVFM_WORD_0, DVFM_WORD_1, DVFM_WORD_2, DVFM_WORD_3, +}; + +struct t8140_sleep { + struct device *dev; + void __iomem *ps; + void __iomem *pwrgate; + void __iomem *cpu_start; + void __iomem *cllt; + void __iomem *cluster; + void __iomem *commit; + void __iomem *wdt; + void __iomem *aon_misc; + void __iomem *wake; + void __iomem *amcc[4]; + void __iomem *pmc_dcp; + void __iomem *spmi; + void __iomem *pmc_voters; + void __iomem *dcs; + void __iomem *ps_aon; + void __iomem *spmi_a0; + void __iomem *hpm_spmi; + /* The port controller whose cold reset ends the debug link. */ + struct device_node *hpm_np; + u64 siq_cfg; + u32 hpm_sid; + int usb_down_ret; + /* Cut, or found already down, for this sleep. */ + bool usb_down_ran; + void __iomem *clk_mux; + void __iomem *mcc[MCC_CHANNELS]; + void __iomem *ptd; + void __iomem *clk_perf; + void __iomem *cluster_dcs; + u32 ps_on_saved[10]; + u32 ps_off_saved[64]; + u32 ps_aon_saved[4]; + u32 ps_aon_auto_saved[2]; + u32 spmi_saved[2]; + bool matched; + void __iomem *dvfm[T8140_CLUSTERS]; + struct t8140_s2r_mailbox *mbox; + + u32 wake_enable[WAKE_SOURCES_MAX]; + int wake_enable_count; + u32 wake_saved[WAKE_WORDS]; + + struct { + struct device_node *np; + void __iomem *asc; + bool bound; + } display[T8140_DISPLAY_COUNT]; + + /* Filled in for the report after a return. */ + u32 wake_status[WAKE_WORDS]; + u64 dvfm_ctl[T8140_CLUSTERS]; + u64 dvfm_saved[T8140_CLUSTERS][ARRAY_SIZE(t8140_dvfm_words)]; + bool dvfm_parked; + u64 resumes_before; + u64 resumes_after; + int cluster_e_ret; + int cluster_ave_ret; + int cluster_ave_commit_ret; + unsigned int ps_rewoken; + unsigned int ps_rewake_timeouts; + u64 cntkctl; + int cpus_stop_ret; + /* The words t8140_sleep_lower_stray() lowered, one bit each */ + u8 stray_lowered; + /* What must be bound for a sleep, and what must have stopped by the last step */ + struct t8140_sleep_gate { + const char *name; + /* Compared only, never dereferenced: nodes outlive the gate. */ + const struct device_node *np; + void __iomem *asc; + bool bound; + bool must_stop; + bool sep; + u32 ps; + } gate[16]; + unsigned int gate_count; + bool returned; +}; + +static struct t8140_sleep *t8140_sleep; + +static struct t8140_sleep_coproc *t8140_sleep_coproc_find(struct device *dev) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (t8140_sleep_coprocs[i].dev == dev) + return &t8140_sleep_coprocs[i]; + + return NULL; +} + +/* A registered coprocessor its driver has readied, whose run bit the last step clears. */ +static bool t8140_sleep_coproc_ready_node(const struct device_node *np) +{ + int i; + + for (i = 0; np && i < SLEEP_COPROCS; i++) + if (t8140_sleep_coprocs[i].dev && t8140_sleep_coprocs[i].ready && + t8140_sleep_coprocs[i].dev->of_node == np) + return true; + return false; +} + +static bool t8140_usb_asleep; + +void apple_sleep_usb_set_asleep(bool asleep) +{ + WRITE_ONCE(t8140_usb_asleep, asleep); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_asleep); + +static unsigned long t8140_usb_wake_ports; + +void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ + WRITE_ONCE(t8140_usb_wake_ports, ports); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); + +bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return port < BITS_PER_LONG && (READ_ONCE(t8140_usb_wake_ports) & BIT(port)); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); + +static bool t8140_debug_link_cut; + +/* Only the debug port's controller took the cold reset. */ +static bool t8140_debug_port_is(struct device *dev) +{ + return t8140_sleep && t8140_sleep->hpm_np && dev->of_node == t8140_sleep->hpm_np; +} + +bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return t8140_debug_port_is(dev) && xchg(&t8140_debug_link_cut, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_was_cut); + +bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return t8140_debug_port_is(dev) && READ_ONCE(t8140_debug_link_cut); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_peek); + +/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data) +{ + struct t8140_sleep_coproc *c; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&t8140_sleep_coprocs_lock); + if (t8140_sleep_coproc_find(dev)) + return -EEXIST; + c = t8140_sleep_coproc_find(NULL); + if (!c) + return -ENOSPC; + c->asc = asc; + c->pcie_ans = pcie_ans; + c->last_check = NULL; + c->last_stop = last_stop; + c->data = data; + c->ready = false; + c->stopped = false; + c->first = false; + c->ps = 0; + c->dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); + +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data) +{ + struct t8140_sleep_coproc *c; + struct device_node *np; + u32 ps = 0; + int ret; + + if (!last_stop) + return -EINVAL; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np) { + of_property_read_u32(np, "reg", &ps); + of_node_put(np); + } + ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); + if (ret) + return ret; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (c) { + c->first = true; + c->ps = ps; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); + +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (!c) + return -ENOENT; + c->last_check = last_check; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); + +int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), void *data) +{ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&t8140_sleep_coprocs_lock); + if (t8140_sleep_enclave.dev) + return -EBUSY; + t8140_sleep_enclave.tell = tell; + t8140_sleep_enclave.data = data; + t8140_sleep_enclave.dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); + +void apple_sleep_enclave_unregister(struct device *dev) +{ + guard(mutex)(&t8140_sleep_coprocs_lock); + if (t8140_sleep_enclave.dev == dev) + memset(&t8140_sleep_enclave, 0, sizeof(t8140_sleep_enclave)); +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); + +void apple_sleep_coproc_unregister(struct device *dev) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (c) + memset(c, 0, sizeof(*c)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); + +void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (!c) + return; + c->ready = ready; + c->stopped = false; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); + +bool apple_sleep_coproc_stopped(struct device *dev) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + return c && c->stopped; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); + +/* + * The last thing that can fail on the way to the commit: no CPU but this one is up and + * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only + * through a real sleep, so every refusal has to be behind this, and nothing after it may + * go back. In the order measured on macOS, on ANS itself: whatever its driver does last, + * status and control read, then 0 written twice. + */ +static void t8140_sleep_coproc_run_off(struct t8140_sleep_coproc *c) +{ + /* Its own last stop cleared its run bits. */ + if (!c->asc) { + c->stopped = true; + return; + } + c->status = readl(c->asc + ASC_CPU_STATUS); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + c->stopped = true; +} + +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +static bool t8140_sleep_coproc_lowers(u32 ps) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (t8140_sleep_coprocs[i].dev && t8140_sleep_coprocs[i].ready && + t8140_sleep_coprocs[i].first && t8140_sleep_coprocs[i].ps == ps) + return true; + return false; +} + +/* The last refusals, each coprocessor's own: nothing has stopped yet. */ +static int t8140_sleep_coprocs_check(struct t8140_sleep *s) +{ + int i, ret; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->stopped || !c->last_check) + continue; + ret = c->last_check(c->data); + if (ret) { + dev_err(s->dev, "refused: %s: %d\n", dev_name(c->dev), ret); + return ret; + } + } + + return 0; +} + +/* Last of all the refusals: an enclave that gated comes back only through a real sleep. */ +static int t8140_sleep_enclave_tell(struct t8140_sleep *s) +{ + int ret; + + if (!t8140_sleep_enclave.tell) + return 0; + ret = t8140_sleep_enclave.tell(t8140_sleep_enclave.data); + if (ret) + dev_err(s->dev, "refused: the enclave didn't take the sleep request: %d\n", ret); + + return ret; +} + +/* + * In two passes, after the last refusal: from the first stop only a real sleep restarts the + * GPU, so a failure here goes on to the commit. Those registered first stop whole first, + * before macOS's off words drop under them: the GPU withdraws its PMP request there, and the + * PMP has to see that before its own stop. The rest after the match. + */ +static void t8140_sleep_coprocs_stop(struct t8140_sleep *s, bool first) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != first) + continue; + if (c->stopped) + continue; + if (c->last_stop) + c->last_stop(c->data); + t8140_sleep_coproc_run_off(c); + } + dsb(sy); +} + +/* MPIDR is cluster 0 cores 0-3 at 0.0.n and cluster 1 cores 0-1 at 1.1.n. */ +static unsigned int t8140_cpu_cluster(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 2); +} + +static unsigned int t8140_cpu_core(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 0); +} + +static u32 t8140_cpu_start_bit(u64 mpidr) +{ + return BIT(4 * t8140_cpu_cluster(mpidr) + t8140_cpu_core(mpidr)); +} + +static void t8140_deep_wfi_arm(void) +{ + /* A timer left running ends the WFI, and costs power until reset. */ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_ENABLE); +} + +/* A core's last instructions: nothing brings it back but a reset. */ +static void __noreturn t8140_deep_wfi(void) +{ + t8140_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void t8140_cpu_stop(u64 mpidr) +{ + writel(t8140_cpu_start_bit(mpidr), t8140_sleep->cpu_start + CPU_START_STOP); + dsb(sy); +} + +static void t8140_cpu_off(u64 mpidr) +{ + t8140_deep_wfi(); +} + +static int t8140_cpu_on(u64 mpidr) +{ + struct t8140_sleep *s = t8140_sleep; + unsigned int cluster = t8140_cpu_cluster(mpidr); + + writel(t8140_cpu_start_bit(mpidr), s->cpu_start + CPU_START_SYS); + writel(BIT(t8140_cpu_core(mpidr)), s->cpu_start + CPU_START_CLUSTER(cluster)); + + return 0; +} + +static const struct spin_table_power_ops t8140_cpu_power_ops = { + .cpu_stop = t8140_cpu_stop, + .cpu_off = t8140_cpu_off, + .cpu_on = t8140_cpu_on, +}; + +/* + * As macOS lowers a domain: AUTO_ENABLE off first if it's set, then target 0. + * Writing the flags clears them. Values measured, for instance ANS 0x1f0020ff, + * 0x0f0020ff, 0x0f0023f0. + */ +static u32 t8140_ps_lower_start(struct t8140_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + + return val; +} + +static void t8140_ps_lower_finish(struct t8140_sleep *s, u32 offset, u32 val) +{ + writel((val & ~PS_TARGET) | PS_FLAGS, s->ps + offset); +} + +static void t8140_ps_lower(struct t8140_sleep *s, u32 offset) +{ + t8140_ps_lower_finish(s, offset, t8140_ps_lower_start(s, offset)); +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Measured + * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. + */ +static int t8140_ps_raise_start(struct t8140_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 now; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + + return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, + PS_OFF_TIMEOUT_US); +} + +static void t8140_ps_raise_finish(struct t8140_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, + reg); +} + +static int t8140_ps_raise(struct t8140_sleep *s, u32 offset) +{ + int ret = t8140_ps_raise_start(s, offset); + + t8140_ps_raise_finish(s, offset); + + return ret; +} + +static int t8140_ps_wait_off(struct t8140_sleep *s, u32 offset) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, !FIELD_GET(PS_ACTUAL, val), + 10, PS_OFF_TIMEOUT_US); +} + +static bool t8140_ps_on(struct t8140_sleep *s, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)); +} + +/* + * After a refused sleep, the sleep engine's own restart of a firmware's CPU whose domain shares + * a power switch and can't gate alone, as SIO's: each of the device's own domains reset through + * target 4 and 0, released with the domain off, then raised again, its DARTs' state kept. The + * caller has cleared the run bit, and starts the firmware after. + */ +#define PS_BUSY BIT(11) +#define PS_RESET BIT(31) +#define PS_ENGINE_RESET_4 0x8f000244 +#define PS_ENGINE_RESET_0 0x80000000 +#define ASC_CPU_STATUS_READY BIT(1) +#define ENGINE_RESET_PS 4 +#define ENGINE_RESET_DARTS 2 + +static int t8140_engine_reset_wait(struct t8140_sleep *s, u32 offset, u32 actual) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, + FIELD_GET(PS_ACTUAL, val) == actual && !(val & PS_BUSY), + 10, PS_OFF_TIMEOUT_US); +} + +/* A domain of the PMGR block mapped here as "ps", and its offset in it. */ +static bool t8140_ps_domain(struct t8140_sleep *s, struct device_node *pd, u32 *offset) +{ + int idx = of_property_match_string(s->dev->of_node, "reg-names", "ps"); + struct device_node *parent = of_get_parent(pd); + struct resource ps, res; + bool ours; + + ours = idx >= 0 && parent && !of_address_to_resource(s->dev->of_node, idx, &ps) && + !of_address_to_resource(parent, 0, &res) && res.start == ps.start && + !of_property_read_u32(pd, "reg", offset); + of_node_put(parent); + + return ours; +} + +/* The @index-th DART in @np's iommus, a reference held, or NULL past the last. */ +static struct device *t8140_iommus_dart(struct device_node *np, int index) +{ + struct of_phandle_args args; + struct platform_device *pdev; + + if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, &args)) + return NULL; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + + return pdev ? &pdev->dev : NULL; +} + +int apple_sleep_engine_reset(struct device *dev) +{ + struct t8140_sleep *s = t8140_sleep; + struct device *darts[ENGINE_RESET_DARTS] = {}; + u32 ps[ENGINE_RESET_PS], saved[ENGINE_RESET_PS], val; + struct of_phandle_iterator it; + void __iomem *asc; + int i, n = 0, idx, err, ret = 0; + + if (!s) + return -ENODEV; + of_for_each_phandle(&it, err, dev->of_node, "power-domains", NULL, 0) { + if (!t8140_ps_domain(s, it.node, &val)) + continue; + if (n == ENGINE_RESET_PS) { + of_node_put(it.node); + return -E2BIG; + } + ps[n++] = val; + } + if (!n) + return -ENODEV; + + idx = of_property_match_string(dev->of_node, "reg-names", "coproc"); + if (idx < 0) + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return -ENODEV; + + for (i = 0; i < ENGINE_RESET_DARTS; i++) { + darts[i] = t8140_iommus_dart(dev->of_node, i); + if (!darts[i]) + break; + apple_dart_state_save(darts[i]); + } + + for (i = 0; i < n && !ret; i++) { + saved[i] = readl(s->ps + ps[i]); + writel(PS_ENGINE_RESET_4, s->ps + ps[i]); + ret = t8140_engine_reset_wait(s, ps[i], 4); + if (!ret) { + writel(PS_ENGINE_RESET_0, s->ps + ps[i]); + ret = t8140_engine_reset_wait(s, ps[i], 0); + } + } + if (!ret) { + usleep_range(1000, 2000); + for (i = 0; i < n; i++) + writel(0, s->ps + ps[i]); + for (i = n - 1; i >= 0 && !ret; i--) { + writel(PS_TARGET, s->ps + ps[i]); + ret = t8140_engine_reset_wait(s, ps[i], FIELD_MAX(PS_ACTUAL)); + if (!ret) + writel(saved[i] & ~(PS_FLAGS | PS_RESET | PS_ACTUAL), + s->ps + ps[i]); + } + } + if (!ret) + ret = readl_poll_timeout(asc + ASC_CPU_STATUS, val, val & ASC_CPU_STATUS_READY, 10, + 100 * USEC_PER_MSEC); + + for (i = 0; i < ENGINE_RESET_DARTS && darts[i]; i++) { + apple_dart_state_restore(darts[i]); + put_device(darts[i]); + } + iounmap(asc); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_sleep_engine_reset); + +/* The order macOS does it in, across clusters. */ +static void t8140_dvfm_park(struct t8140_sleep *s) +{ + int cluster, word; + + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) { + s->dvfm_ctl[cluster] = readq(s->dvfm[cluster] + DVFM_CTL); + for (word = 0; word < ARRAY_SIZE(t8140_dvfm_words); word++) + s->dvfm_saved[cluster][word] = + readq(s->dvfm[cluster] + t8140_dvfm_words[word]); + } + + writeq(0, s->dvfm[0] + DVFM_CTL); + writeq(0, s->dvfm[1] + DVFM_CTL); + for (word = 0; word < 2; word++) + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + t8140_dvfm_words[word]); + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + t8140_dvfm_words[word]); + s->dvfm_parked = true; +} + +static void t8140_dvfm_unpark(struct t8140_sleep *s) +{ + int cluster, word; + + if (!s->dvfm_parked) + return; + s->dvfm_parked = false; + + writeq(s->dvfm_ctl[0], s->dvfm[0] + DVFM_CTL); + writeq(s->dvfm_ctl[1], s->dvfm[1] + DVFM_CTL); + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + t8140_dvfm_words[word]); + for (word = 1; word >= 0; word--) + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + t8140_dvfm_words[word]); +} + +/* + * What XNU polls just before its complex words, and panics on if one is on, besides the + * display's, ANS's and ATC0's, which the commit lowers itself. Every device has suspended by + * the last step, so one still on has no user: it's lowered there, and a sleep with one that + * won't go off is refused. They come back on after the sleep or a refusal, for genpd's books. + */ +static const struct { + const char *name; + u32 offset; +} t8140_stray_words[] = { + { "AVD_SYS", 0x270 }, { "VENC0_SYS", 0x240 }, { "VENC1_SYS", 0x248 }, + { "GFX", 0x1f0 }, { "JPG", 0x230 }, +}; + +static void t8140_sleep_stray_restore(struct t8140_sleep *s) +{ + int i; + + for (i = ARRAY_SIZE(t8140_stray_words) - 1; i >= 0; i--) + if (s->stray_lowered & BIT(i)) + t8140_ps_raise(s, t8140_stray_words[i].offset); + s->stray_lowered = 0; +} + +static int t8140_sleep_lower_stray(struct t8140_sleep *s) +{ + int i, ret = 0; + + s->stray_lowered = 0; + for (i = 0; i < ARRAY_SIZE(t8140_stray_words); i++) { + u32 off = t8140_stray_words[i].offset; + + if (!t8140_ps_on(s, off)) + continue; + /* Its driver's own last stop lowers it, at the commit. */ + if (t8140_sleep_coproc_lowers(off)) { + dev_dbg(s->dev, "%s: left on for its driver's last stop\n", + t8140_stray_words[i].name); + continue; + } + t8140_ps_lower(s, off); + s->stray_lowered |= BIT(i); + if (t8140_ps_wait_off(s, off)) { + dev_err(s->dev, "refused: %s won't go off: %#x\n", + t8140_stray_words[i].name, readl(s->ps + off)); + ret = -EBUSY; + } else { + dev_info(s->dev, "%s was on at the last step, lowered\n", + t8140_stray_words[i].name); + } + } + if (ret) + t8140_sleep_stray_restore(s); + + return ret; +} + +/* + * T8140's coprocessors that must be bound for a sleep, the ones that must also have stopped + * by the last step (a commit with the PMP and SIO still running reset the SoC 5.5 s later), + * and the domains macOS has off by then. Taken in process context, since the checks run with + * interrupts off. A driver whose stop failed has already failed the suspend, naming itself, + * so what's left is whether it's bound at all, and what the hardware shows. @ps is the CPU's + * own domain where a driver may power it off, as the DCPs' at DPMS off. + */ +static const struct { + const char *path; + bool must_stop; + u32 ps; +} t8140_gate_coprocs[] = { + { "/soc/dcp@412e00000", true, PS_DISP_CPU }, + { "/soc/dcp@416e00000", true, PS_DISPEXT0_CPU }, + { "/soc/pmp@300500000", true }, { "/soc/dma-controller@385e00000", true }, + { "/soc/nvme@3cdcc0000" }, { "/soc/mtp@314600000" }, + { "/soc/aop@310c00000" }, { "/soc/smc@30c600000" }, + { "/soc/sep@282600000" }, { "/soc/gpu@482600000" }, +}; + +static const struct { + const char *name; + u32 offset; +} t8140_gate_off[] = { + { "ane_sys", 0x290 }, { "isp_sys", 0x268 }, { "prores", 0x250 }, + { "avd_sys", 0x270 }, { "jpg", 0x230 }, { "venc0_sys", 0x240 }, +}; + +static void t8140_sleep_gate_prepare(struct t8140_sleep *s) +{ + int i; + + s->gate_count = 0; + for (i = 0; i < ARRAY_SIZE(t8140_gate_coprocs) && s->gate_count < ARRAY_SIZE(s->gate); + i++) { + struct device_node *c = of_find_node_by_path(t8140_gate_coprocs[i].path); + struct platform_device *pdev = c ? of_find_device_by_node(c) : NULL; + struct t8140_sleep_gate *g; + + if (!c) { + dev_warn(s->dev, "%s isn't in this tree\n", t8140_gate_coprocs[i].path); + continue; + } + g = &s->gate[s->gate_count++]; + g->name = c->name; + g->np = c; + g->bound = pdev && device_is_bound(&pdev->dev); + g->sep = of_node_name_eq(c, "sep"); + g->must_stop = t8140_gate_coprocs[i].must_stop; + g->ps = t8140_gate_coprocs[i].ps; + if (g->must_stop && !g->asc) { + int idx = of_property_match_string(c, "reg-names", "coproc"); + + if (idx < 0) + idx = of_property_match_string(c, "reg-names", "asc"); + if (idx >= 0) + g->asc = of_iomap(c, idx); + } + if (pdev) + put_device(&pdev->dev); + of_node_put(c); + } +} + +/* + * Names everything wrong, so one boot shows the whole list. A coprocessor that must stop and + * still runs refuses the sleep; the rest only warns, as a real sleep is the only full test. + */ +static bool t8140_sleep_gate_failed(struct t8140_sleep *s) +{ + int i, bad = 0, running = 0; + + for (i = 0; i < s->gate_count; i++) { + struct t8140_sleep_gate *g = &s->gate[i]; + + if (!g->bound) { + dev_err(s->dev, "%s isn't bound\n", g->name); + bad++; + /* Nobody told it of the sleep; a commit with it running never returns. */ + if (g->sep && t8140_ps_on(s, PS_SEP)) { + dev_err(s->dev, "sep runs with no driver bound: %#x\n", + readl(s->ps + PS_SEP)); + running++; + } + continue; + } + /* A CPU whose domain is off isn't running, and its registers aren't there. */ + if (g->ps && !t8140_ps_on(s, g->ps)) + continue; + if (g->must_stop && (!g->asc || + ((readl(g->asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN) && + !t8140_sleep_coproc_ready_node(g->np)))) { + dev_err(s->dev, "%s didn't stop: control %#x status %#x\n", g->name, + g->asc ? readl(g->asc + ASC_CPU_CONTROL) : 0, + g->asc ? readl(g->asc + ASC_CPU_STATUS) : 0); + bad++; + running++; + } + /* Told later, as the last refusal: see t8140_sleep_enclave_tell(). */ + if (g->sep && t8140_ps_on(s, PS_SEP) && !t8140_sleep_enclave.tell) { + dev_err(s->dev, "sep hasn't gated itself: %#x\n", readl(s->ps + PS_SEP)); + bad++; + running++; + } + } + + for (i = 0; i < ARRAY_SIZE(t8140_gate_off); i++) { + if (t8140_ps_on(s, t8140_gate_off[i].offset)) { + dev_err(s->dev, "%s is on: %#x\n", t8140_gate_off[i].name, + readl(s->ps + t8140_gate_off[i].offset)); + bad++; + } + } + + if (running) { + dev_err(s->dev, "refused: %d coprocessors that must stop or gate are still running\n", + running); + return true; + } + if (bad) + dev_warn(s->dev, "going ahead with %d problems\n", bad); + else + dev_info(s->dev, "%u coprocessors up and stopped\n", s->gate_count); + + return false; +} + +static void t8140_wake_unmask(u32 *val, u32 id) +{ + if (id / 32 < WAKE_WORDS) + val[id / 32] &= ~BIT(id % 32); +} + +/* + * Mask every source but the ones asked for. These words read all zero until + * something writes them, which leaves every source enabled, listed in the ADT or + * not. macOS reads all ones outside the ADT's ap-wake-sources (measured). The AOP's + * source 0 is masked with the rest: that held under macOS with its handler hidden, + * as did leaving it enabled with the AMCC enables off. The two together are unmeasured. + */ +static void t8140_wake_masks(struct t8140_sleep *s, u32 *val) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + val[i] = i == WAKE_WORDS - 1 ? WAKE_WORD7_ALL : U32_MAX; + for (i = 0; i < s->wake_enable_count; i++) + t8140_wake_unmask(val, s->wake_enable[i]); +} + +/* The last checks, then the stray words lowered: what has stopped may not return until a wake. */ +static int t8140_sleep_refuse(struct t8140_sleep *s) +{ + bool refused = false, idled = false; + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; + + if (!c->dev) + continue; + if (c->ready) { + idled |= c->pcie_ans; + } else { + /* Nothing may lower a coprocessor's domain under its driver. */ + dev_err(s->dev, "refused: %s is running and its driver has not quiesced it\n", + dev_name(c->dev)); + refused = true; + } + } + + if (num_online_cpus() != 1 || + (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { + dev_err(s->dev, "refused: wrong CPU, or other CPUs online\n"); + refused = true; + } + + /* Only a driver takes the display to where its domain can be lowered. */ + if (!s->display[0].bound) { + dev_err(s->dev, "refused: no driver bound to the display coprocessor\n"); + refused = true; + } + + /* ANS lowers its PCIe block with it, which this can't bring back yet. */ + if (idled || t8140_ps_on(s, PS_ANS) || t8140_ps_on(s, PS_APCIE_SYS_GP)) { + dev_err(s->dev, "refused: ANS or its PCIe block is on\n"); + refused = true; + } + + /* The engine waits for DISP_CPU with no timeout, so this must never slip. */ + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + u32 ps = i ? PS_DISPEXT0_CPU : PS_DISP_CPU; + + if (s->display[i].bound && t8140_ps_on(s, ps) && + (readl(s->display[i].asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + dev_err(s->dev, "refused: a bound display coprocessor is still running\n"); + refused = true; + } + } + + if (t8140_sleep_gate_failed(s)) + refused = true; + + if (refused) + return -EBUSY; + + return t8140_sleep_lower_stray(s); +} + +static void t8140_wake_arm(struct t8140_sleep *s) +{ + u32 val[WAKE_WORDS]; + int i; + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_saved[i] = readl(s->wake + WAKE_MASK(i)); + t8140_wake_masks(s, val); + for (i = 0; i < WAKE_WORDS; i++) + writel(val[i], s->wake + WAKE_MASK(i)); +} + +static void t8140_wake_restore(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + writel(s->wake_saved[i], s->wake + WAKE_MASK(i)); +} + +/* + * The sleep engine waits on DISP_CPU with no timeout, so the display domains + * must be off. A coprocessor's driver stops its firmware first, and nothing here + * talks to the firmware. Otherwise no driver is bound, and the bootloader left the + * firmware running: on the DCP, stop its CPU cold. That's unproven. The coprocessor + * restarts from its reset vector after any sleep, and the engine itself clears the + * DCP's run bit on the way down. + * + * Domain order and values as macOS's: DCPEXT then DCP, CPU, front end, system. + * The rest of the display's domains go off with these, by nothing Linux or macOS + * writes. The coprocessors' secure mailboxes are not used here. Whether that + * holds is open. + */ +static int t8140_display_off(struct t8140_sleep *s) +{ + static const u32 domains[T8140_DISPLAY_COUNT][3] = { + { PS_DISP_CPU, PS_DISP_FE, PS_DISP_SYS }, + { PS_DISPEXT0_CPU, PS_DISPEXT0_FE, PS_DISPEXT0_SYS }, + }; + int i, ret = 0; + + for (i = T8140_DISPLAY_COUNT - 1; i >= 0; i--) { + void __iomem *asc = s->display[i].asc; + + /* The engine doesn't wait for DISPEXT0. */ + if (i == 1 && !s->display[i].bound) + continue; + + if (t8140_ps_on(s, domains[i][0]) && asc && + (readl(asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + writel(0, asc + ASC_CPU_CONTROL); + writel(0, asc + ASC_CPU_CONTROL); + } + + /* Reads as a count of the DCPs powered, and macOS ends at 0. */ + writel(i, s->pmc_dcp); + if (i == 0) { + if (readl(s->ps + PS_DISP_CPU) & PS_AUTO_ENABLE) + writel(readl(s->ps + PS_DISP_CPU) & + ~(PS_AUTO_ENABLE | PS_FLAGS), s->ps + PS_DISP_CPU); + writel(3, s->pwrgate + PWRGATE_DISP_0); + writel(0, s->pwrgate + PWRGATE_DISP_1); + } + t8140_ps_lower(s, domains[i][0]); + t8140_ps_lower(s, domains[i][1]); + t8140_ps_lower(s, domains[i][2]); + } + + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + int err; + + if (i == 1 && !s->display[i].bound) + continue; + + err = t8140_ps_wait_off(s, domains[i][0]); + + if (err) { + dev_err(s->dev, "refused: display domain %#x stayed on: %#x\n", + domains[i][0], readl(s->ps + domains[i][0])); + ret = err; + } + } + + return ret; +} + +/* One command on the port controller's bus, polled: no other CPU, and interrupts are off. */ +static int t8140_hpm_cmd(struct t8140_sleep *s, u32 op, u32 extra, const u32 *in, u32 *out) +{ + void __iomem *b = s->hpm_spmi; + u32 status, reply; + int ret; + + while (!(readl(b + SPMI_STATUS) & SPMI_STATUS_RX_EMPTY)) + readl(b + SPMI_REPLY); + + writel(extra << 16 | SPMI_CMD_ACTIVE | s->hpm_sid << 8 | op, b + SPMI_CMD); + if (in) + writel(*in, b + SPMI_CMD); + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, SPMI_POLL_US); + if (ret) + return ret; + reply = readl(b + SPMI_REPLY); + writel(1, b + SPMI_NOTIFY_ACK); + if (out) { + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, + SPMI_POLL_US); + if (ret) + return ret; + *out = readl(b + SPMI_REPLY); + } + + /* A zero write's reply carries the value written. */ + return op == SPMI_OP_ZERO_WRITE || (reply & 0xff) == op ? 0 : -EIO; +} + +static int t8140_hpm_select(struct t8140_sleep *s, u32 page) +{ + u32 val = ~0U; + int i, ret; + + ret = t8140_hpm_cmd(s, SPMI_OP_ZERO_WRITE, page << 8, NULL, NULL); + for (i = 0; !ret && i < 2000; i++) { + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_READ(1), HPM_REG_SELECT, NULL, &val); + if (!ret && (val & 0xff) == page) + return 0; + udelay(10); + } + + return ret ?: -ETIMEDOUT; +} + +/* + * What macOS sends the other port controller about a second before a sleep, measured + * in three runs: system power state 3, then the bus's sleep command. + */ +static int t8140_hpm_sleep(struct t8140_sleep *s) +{ + u32 data = HPM_SSPS_S3, cmd = HPM_CMD_SSPS, val = ~0U; + int i, ret; + + ret = t8140_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = t8140_hpm_select(s, HPM_PAGE_DATA1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_WRITE(1), HPM_REG_DATA_CMD, &data, NULL); + if (!ret) + ret = t8140_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA_CMD, &cmd, NULL); + /* Selected afresh for each read, as macOS does. */ + for (i = 0; !ret && val && i < 100; i++) { + udelay(1000); + ret = t8140_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); + } + if (!ret && val) + ret = -ETIMEDOUT; + /* Asleep either way: a failure here only means it wasn't told first. */ + t8140_hpm_cmd(s, SPMI_OP_SLEEP, 0, NULL, NULL); + + return ret; +} + +/* + * The debug link stays up with the ATC0 domains off, and it holds the debug domains + * active, which macOS has idle. The only known way to end it is the port controller's + * cold reset. To the SMC that's a cable pulled: it queues the port's events and records + * them as wake reasons. So it has to come before the SMC's AP idle notice, not after, or + * SMC.OutboxNotEmpty is up at the commit. tipd rearms the link after a refusal. + */ +static bool t8140_sleep_usb_down(struct t8140_sleep *s) +{ + u32 cmd = HPM_CMD_COLD_RESET; + u32 enable; + int ret; + + if (s->usb_down_ran) + return false; + s->usb_down_ran = true; + + if (!s->hpm_spmi || !FIELD_GET(PS_ACTUAL, readl(s->ps_aon + PS_AON_DEBUG_SWITCH))) + return false; + + dev_info(s->dev, "debug link: cutting it for the sleep\n"); + enable = readl(s->hpm_spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + ret = t8140_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = t8140_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA, &cmd, NULL); + if (!ret) { + WRITE_ONCE(t8140_debug_link_cut, true); + mdelay(HPM_RESET_MS); + ret = t8140_hpm_sleep(s); + } + writel(1, s->hpm_spmi + SPMI_NOTIFY_ACK); + writel(enable, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + s->usb_down_ret = ret; + + return true; +} + +bool apple_sleep_debug_link_cut(void) +{ + return t8140_sleep ? t8140_sleep_usb_down(t8140_sleep) : false; +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); + +/* About 4 ms after the other cores stop on macOS: deepest first. */ +static void t8140_sleep_lower_io(struct t8140_sleep *s) +{ + /* + * USB's complex, deepest first: macOS has it off here, and its drivers don't lower it. + * A host that sleeps in place keeps ATC0_USB_AON, as macOS's does. + */ + t8140_ps_lower(s, PS_ATC0_USB); + t8140_ps_lower(s, PS_ATC0_COMMON); + if (!READ_ONCE(t8140_usb_asleep)) + t8140_ps_lower(s, PS_ATC0_USB_AON); +} + +static void t8140_sleep_set_wd2(struct t8140_sleep *s, unsigned int secs) +{ + writel(0, s->wdt + WDT_WD2_CTL); + writel(0, s->wdt + WDT_WD2_COUNT); + writel(secs * WDT_TICKS_PER_S, s->wdt + WDT_WD2_ALARM); + writel(1, s->wdt + WDT_WD2_CTL); +} + +/* + * Every power word in the ADT, read just before ANS's stop so a run shows the whole chip. + */ +static const u32 t8140_ps_all[] = { + 0x0, 0x8, 0x10, 0x18, 0x30, 0x38, 0x50, 0x58, 0x60, 0x100, 0x108, 0x110, 0x118, + 0x120, 0x128, 0x130, 0x138, 0x140, 0x148, 0x150, 0x158, 0x160, 0x168, 0x170, 0x178, + 0x180, 0x188, 0x190, 0x198, 0x1a0, 0x1a8, 0x1b0, 0x1b8, 0x1c0, 0x1c8, 0x1d0, 0x1d8, + 0x1e0, 0x1e8, 0x1f0, 0x1f8, 0x200, 0x208, 0x210, 0x218, 0x220, 0x228, 0x230, 0x238, + 0x240, 0x248, 0x250, 0x258, 0x260, 0x268, 0x270, 0x278, 0x280, 0x288, 0x290, 0x298, + 0x2a0, 0x2a8, 0x2b0, 0x2b8, 0x2c0, 0x2c8, 0x2d0, 0x2d8, 0x2e0, 0x2e8, 0x2f0, 0x2f8, + 0x300, 0x308, 0x310, 0x318, 0x320, 0x328, 0x330, 0x338, 0x340, 0x348, 0x350, 0x358, + 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x390, 0x398, 0x3a0, 0x3a8, 0x3b0, 0x3b8, + 0x3c0, 0x3c8, 0x3d0, 0x3d8, 0x3e0, 0x3e8, 0x3f0, 0xc00, 0x4000, 0x4008, 0x4010, + 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, 0x4050, 0x4058, 0x4060, + 0x8000, 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, + 0xc000, 0xc008, 0xc010, 0x10000, 0x10008, 0x10010, 0x10018, 0x10020, 0x10028, + 0x10030, 0x10038, 0x10040, +}; + +static const u32 t8140_ps_aon_all[] = { + 0x0, 0x58, 0x60, 0x68, 0x70, 0x78, 0x80, 0x88, 0x90, 0x98, 0xa0, 0xa8, 0xb0, + 0x4000, 0x4008, 0x4010, 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, + 0x4050, 0x4058, 0x4060, 0x4068, 0x4070, 0x4078, 0x4080, 0x4088, 0x4090, 0x4098, + 0x40a0, 0x40a8, 0x40b0, 0x40b8, 0x40c0, 0x40c8, 0x40d0, 0x40d8, 0x40e0, 0x40e8, + 0x40f0, 0x40f8, 0x4100, 0x4108, 0x4110, 0x4118, 0x4120, 0x4128, 0x4130, 0x4138, + 0x4140, 0x4148, 0x4150, 0x4158, 0x4160, 0x4168, 0x4170, 0x4178, 0x4180, 0x8000, + 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, 0x8050, + 0x8058, 0x8060, 0x8068, 0x8070, 0x8078, +}; + +/* What macOS has on at the commit and Linux leaves off, in its order: nothing uses them here. */ +static const u32 t8140_ps_macos_on[] = { + 0x2f0, 0x2f8, 0x358, 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x118, +}; + +/* + * The power words macOS's wake writes back on before any driver runs, in its order + * (logs/wakerec-20261105-154137Z). The engine leaves domains like GPIO dark, and genpd still + * has them on, so nothing else would. + */ +static const u32 t8140_ps_wake_order[] = { + 0xc00, 0x110, 0x118, 0x120, 0x168, 0x178, 0x2a8, 0x2f0, 0x2f8, 0x300, 0x358, 0x360, + 0x368, 0x370, 0x378, 0x380, 0x388, 0x3d0, 0x3d8, +}; + +static const u32 t8140_ps_aon_wake_order[] = { 0x78, 0x88, 0x8030, 0x8038 }; + +/* Each word at the commit, and as the wake found it. */ +static u32 t8140_ps_commit[ARRAY_SIZE(t8140_ps_all)]; +static u32 t8140_ps_aon_commit[ARRAY_SIZE(t8140_ps_aon_all)]; +static u32 t8140_ps_woke[ARRAY_SIZE(t8140_ps_all)]; +static u32 t8140_ps_aon_woke[ARRAY_SIZE(t8140_ps_aon_all)]; + +#define PS_PARENT_MISSING BIT(11) +#define PS_AUTO_STATE GENMASK(27, 24) +#define PS_REWAKE_PASSES 2 + +static void t8140_ps_save(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + t8140_ps_commit[i] = readl(s->ps + t8140_ps_all[i]); + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + t8140_ps_aon_commit[i] = readl(s->ps_aon + t8140_ps_aon_all[i]); +} + +static int t8140_ps_find(const u32 *all, int n, u32 offset) +{ + int i; + + for (i = 0; i < n; i++) + if (all[i] == offset) + return i; + return -1; +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Never + * DEV_DISABLE, which keeps it off: the SEP's is set at the commit, and macOS's wake writes + * that word without it. The hardware owns the auto state field, so it stays as read. + */ +static int t8140_ps_restore(void __iomem *reg, u32 saved) +{ + u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_PARENT_MISSING | PS_DEV_DISABLE | + PS_FLAGS | PS_ACTUAL)) | (readl(reg) & PS_AUTO_STATE); + u32 now; + int ret; + + /* + * Off under AUTO_ENABLE at the commit, as the SEP's is: the hardware powers it when + * it's wanted, and forcing it on doesn't. macOS writes that word once, AUTO set. + */ + if ((saved & PS_AUTO_ENABLE) && !FIELD_GET(PS_ACTUAL, saved)) { + writel(keep | PS_AUTO_ENABLE, reg); + return 0; + } + + writel(keep, reg); + ret = readl_poll_timeout_atomic(reg, now, + FIELD_GET(PS_ACTUAL, now) == FIELD_GET(PS_TARGET, saved), 1, + PS_OFF_TIMEOUT_US); + if (saved & PS_AUTO_ENABLE) + writel(keep | PS_AUTO_ENABLE, reg); + + return ret; +} + +/* Linux had it on and the wake left it otherwise. With AUTO_ENABLE, off is allowed. */ +static bool t8140_ps_dropped(u32 saved, u32 woke) +{ + u32 control = PS_AUTO_ENABLE | PS_DEV_DISABLE | PS_TARGET; + + if (!FIELD_GET(PS_TARGET, saved)) + return false; + if ((saved ^ woke) & control) + return true; + return !(saved & PS_AUTO_ENABLE) && + FIELD_GET(PS_ACTUAL, woke) != FIELD_GET(PS_TARGET, saved); +} + +/* Words that didn't read on in time, for the report. */ +static bool t8140_ps_late[ARRAY_SIZE(t8140_ps_all)]; +static bool t8140_ps_aon_late[ARRAY_SIZE(t8140_ps_aon_all)]; + +/* + * Every domain back as it was at the commit, before any driver's resume reads its block: + * macOS's wake words first, in its order, whatever they read, if Linux had them on. Then + * any other domain the wake left otherwise, a second time for one whose parent came later. + */ +static void t8140_ps_rewake(struct t8140_sleep *s) +{ + bool *redo = t8140_ps_late, *redo_aon = t8140_ps_aon_late; + bool done[ARRAY_SIZE(t8140_ps_all)] = {}, done_aon[ARRAY_SIZE(t8140_ps_aon_all)] = {}; + int i, j, pass; + + memset(t8140_ps_late, 0, sizeof(t8140_ps_late)); + memset(t8140_ps_aon_late, 0, sizeof(t8140_ps_aon_late)); + s->ps_rewoken = 0; + s->ps_rewake_timeouts = 0; + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + t8140_ps_woke[i] = readl(s->ps + t8140_ps_all[i]); + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + t8140_ps_aon_woke[i] = readl(s->ps_aon + t8140_ps_aon_all[i]); + + for (j = 0; j < ARRAY_SIZE(t8140_ps_wake_order); j++) { + i = t8140_ps_find(t8140_ps_all, ARRAY_SIZE(t8140_ps_all), t8140_ps_wake_order[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, t8140_ps_commit[i])) + continue; + done[i] = true; + s->ps_rewoken++; + if (t8140_ps_restore(s->ps + t8140_ps_all[i], t8140_ps_commit[i])) + redo[i] = true; + } + for (j = 0; j < ARRAY_SIZE(t8140_ps_aon_wake_order); j++) { + i = t8140_ps_find(t8140_ps_aon_all, ARRAY_SIZE(t8140_ps_aon_all), + t8140_ps_aon_wake_order[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, t8140_ps_aon_commit[i])) + continue; + done_aon[i] = true; + s->ps_rewoken++; + if (t8140_ps_restore(s->ps_aon + t8140_ps_aon_all[i], t8140_ps_aon_commit[i])) + redo_aon[i] = true; + } + + for (pass = 0; pass < PS_REWAKE_PASSES; pass++) { + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) { + if (!redo[i] && + (done[i] || !t8140_ps_dropped(t8140_ps_commit[i], t8140_ps_woke[i]))) + continue; + if (!done[i]) { + done[i] = true; + s->ps_rewoken++; + } + redo[i] = t8140_ps_restore(s->ps + t8140_ps_all[i], t8140_ps_commit[i]); + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) { + if (!redo_aon[i] && + (done_aon[i] || + !t8140_ps_dropped(t8140_ps_aon_commit[i], t8140_ps_aon_woke[i]))) + continue; + if (!done_aon[i]) { + done_aon[i] = true; + s->ps_rewoken++; + } + redo_aon[i] = t8140_ps_restore(s->ps_aon + t8140_ps_aon_all[i], + t8140_ps_aon_commit[i]); + } + } + + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + s->ps_rewake_timeouts += redo[i]; + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + s->ps_rewake_timeouts += redo_aon[i]; +} + +static void t8140_ps_rewake_report(struct t8140_sleep *s) +{ + int i; + + dev_dbg(s->dev, "power words: %u written back, %u not on in time\n", s->ps_rewoken, + s->ps_rewake_timeouts); + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + if (t8140_ps_late[i]) + dev_err(s->dev, "power word %#x not on in time: %#x at the commit, %#x now\n", + t8140_ps_all[i], t8140_ps_commit[i], + readl(s->ps + t8140_ps_all[i])); + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + if (t8140_ps_aon_late[i]) + dev_err(s->dev, "power word aon %#x not on in time: %#x at the commit, %#x now\n", + t8140_ps_aon_all[i], t8140_ps_aon_commit[i], + readl(s->ps_aon + t8140_ps_aon_all[i])); +} + +/* What it has off, highest first, as it lowers them. Domains with their own stop aren't here. */ +static const u32 t8140_ps_macos_off[] = { + 0x4060, 0x4058, 0x4050, 0x4048, 0x4040, 0x4038, 0x4030, 0x4028, 0x4020, 0x4018, + 0x4010, 0x4008, 0x4000, 0x398, 0x390, 0x350, 0x348, 0x340, 0x338, 0x330, + 0x308, 0x2e8, 0x2e0, 0x2d8, 0x2d0, 0x2c8, 0x2c0, 0x2b8, 0x2b0, 0x298, + 0x290, 0x270, 0x268, 0x250, 0x248, 0x240, 0x238, 0x230, 0x1a0, 0x190, + 0x188, 0x180, 0x170, 0x160, 0x158, 0x150, 0x148, 0x140, 0x108, +}; + +static const u32 t8140_ps_aon_macos_off[] = { + 0x8028, 0x4140, 0x40f8, 0x80, +}; + +/* The AOP's SPMI words: on, with AUTO_ENABLE off on macOS. */ +static const u32 t8140_ps_aon_macos_no_auto[] = { + 0x4078, 0x4080, +}; + +/* As macOS writes the PMU: the reply's vector masked, then acknowledged, so none is left raised. */ +static void t8140_pmu_write(struct t8140_sleep *s, u16 reg, u8 val) +{ + u32 status, enable; + + enable = readl(s->spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->spmi + SPMI_NOTIFY_ENABLE); + writel(((u32)reg << 16) | SPMI_CMD_WRITE, s->spmi + SPMI_CMD); + writel(val, s->spmi + SPMI_CMD); + if (!readl_poll_timeout_atomic(s->spmi + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) + readl(s->spmi + SPMI_REPLY); + writel(1, s->spmi + SPMI_NOTIFY_ACK); + writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); +} + +static void t8140_sleep_match(struct t8140_sleep *s) +{ + int i, raised = 0, lowered = 0, failed = 0; + + writel(0, s->wdt + WDT_WD0_COUNT); + writel(0, s->wdt + WDT_WD1_COUNT); + t8140_pmu_write(s, PMU_BOOT_STAGE, PMU_BOOT_STAGE_SLEEP); + + s->spmi_saved[0] = readl(s->spmi + SPMI_NOTIFY_ENABLE); + s->spmi_saved[1] = readl(s->spmi_a0); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi + SPMI_NOTIFY_ENABLE); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi_a0); + + for (i = 0; i < ARRAY_SIZE(t8140_ps_macos_on); i++) { + u32 offset = t8140_ps_macos_on[i]; + + s->ps_on_saved[i] = readl(s->ps + offset); + /* State 4 reads as on, and macOS has these at f. */ + if (FIELD_GET(PS_ACTUAL, s->ps_on_saved[i]) == 0xf) + continue; + /* Measured at the commit with AUTO_ENABLE clear, unlike after a wake. */ + if (t8140_ps_raise_start(s, offset)) + failed++; + raised++; + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_macos_off); i++) { + u32 offset = t8140_ps_macos_off[i]; + + s->ps_off_saved[i] = readl(s->ps + offset); + if (!t8140_ps_on(s, offset)) + continue; + t8140_ps_lower(s, offset); + lowered++; + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_macos_off); i++) { + void __iomem *reg = s->ps_aon + t8140_ps_aon_macos_off[i]; + u32 val = readl(reg); + + s->ps_aon_saved[i] = val; + if (!FIELD_GET(PS_ACTUAL, val)) + continue; + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); + lowered++; + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_macos_no_auto); i++) { + void __iomem *reg = s->ps_aon + t8140_ps_aon_macos_no_auto[i]; + u32 val = readl(reg); + + s->ps_aon_auto_saved[i] = val; + writel(val & ~(PS_AUTO_ENABLE | PS_FLAGS), reg); + } + s->matched = true; + + for (i = 0; i < ARRAY_SIZE(t8140_ps_macos_off); i++) + if (t8140_ps_wait_off(s, t8140_ps_macos_off[i])) + dev_dbg(s->dev, "still on, macOS has it off: %x = %x\n", + t8140_ps_macos_off[i], readl(s->ps + t8140_ps_macos_off[i])); + dev_dbg(s->dev, "macOS's power words: raised %d, %d of them failed, lowered %d\n", + raised, failed, lowered); +} + +/* Back as they were, after a refusal past the match or a real wake. */ +static void t8140_sleep_match_undo(struct t8140_sleep *s) +{ + int i; + + if (!s->matched) + return; + s->matched = false; + + for (i = ARRAY_SIZE(t8140_ps_aon_macos_no_auto) - 1; i >= 0; i--) + writel(s->ps_aon_auto_saved[i] & ~PS_FLAGS, + s->ps_aon + t8140_ps_aon_macos_no_auto[i]); + for (i = ARRAY_SIZE(t8140_ps_aon_macos_off) - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_aon_saved[i])) + writel(s->ps_aon_saved[i] & ~PS_FLAGS, + s->ps_aon + t8140_ps_aon_macos_off[i]); + for (i = ARRAY_SIZE(t8140_ps_macos_off) - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_off_saved[i])) + t8140_ps_raise(s, t8140_ps_macos_off[i]); + for (i = ARRAY_SIZE(t8140_ps_macos_on) - 1; i >= 0; i--) { + u32 val = s->ps_on_saved[i]; + + if (!FIELD_GET(PS_ACTUAL, val)) + t8140_ps_lower(s, t8140_ps_macos_on[i]); + else if (FIELD_GET(PS_ACTUAL, val) != 0xf) + writel(val & ~PS_FLAGS, s->ps + t8140_ps_macos_on[i]); + } + writel(s->spmi_saved[0], s->spmi + SPMI_NOTIFY_ENABLE); + writel(s->spmi_saved[1], s->spmi_a0); +} + +/* The safety net, as macOS arms it. */ +static void t8140_sleep_arm_wd2(struct t8140_sleep *s) +{ + writel(0, s->wdt + WDT_WD0_CTL); + t8140_sleep_set_wd2(s, WDT_SLEEP_SECS); + writel(0, s->aon_misc + AON_AP_DEADLINE); +} + +static int t8140_perf_wait(void __iomem *reg, unsigned int state) +{ + u32 val; + + return readl_poll_timeout_atomic(reg, val, + (val & (DCS_BUSY | DCS_STATE)) == + FIELD_PREP(DCS_STATE, state), + 1, DCS_TIMEOUT_US); +} + +static int t8140_dcs_wait(struct t8140_sleep *s, unsigned int state) +{ + return t8140_perf_wait(s->dcs, state); +} + +/* + * As macOS does at about -18 ms, on each channel in turn: the enables down a bit at a time, + * then each of four lanes strobed and cleared, two lanes on every channel before the + * next two. Each write is built from a fresh read, as the trace has it. + */ +static void t8140_sleep_mcc_stop(struct t8140_sleep *s) +{ + int ch, pass, n; + u32 val; + + for (ch = 0; ch < MCC_CHANNELS; ch++) { + val = readl(s->mcc[ch] + MCC_ENABLES); + for (n = 0; val && n < 32; n++) { + val &= val - 1; + writel(val, s->mcc[ch] + MCC_ENABLES); + val = readl(s->mcc[ch] + MCC_ENABLES); + } + } + + for (pass = 0; pass < 2; pass++) { + for (ch = 0; ch < MCC_CHANNELS; ch++) { + void __iomem *mcc = s->mcc[ch]; + + for (n = 2 * pass; n < 2 * pass + 2; n++) { + writel(readl(mcc + MCC_LANES_MASK), mcc + MCC_LANES_MASK); + writel(readl(mcc + MCC_LANES) | MCC_LANE_STROBE(n), + mcc + MCC_LANES); + writel(readl(mcc + MCC_LANES) & ~MCC_LANE(n), mcc + MCC_LANES); + } + } + } +} + +/* + * The engine's first step polls for DCS at this state and has no way to get it there. + * macOS has every voter at 0 by now, which is how it lands there, and steps it down itself + * on the path without the PMC. Both, here, in that order. Nothing can fail by now. + */ +static void t8140_sleep_dcs_down(struct t8140_sleep *s) +{ + unsigned int state; + int agent; + + for (agent = 0; agent < PMC_VOTER_AGENTS; agent++) + if (readl(s->pmc_voters + agent * PMC_VOTER_STRIDE)) + writel(0, s->pmc_voters + agent * PMC_VOTER_STRIDE); + if (!t8140_dcs_wait(s, DCS_SLEEP_STATE)) + return; + + state = FIELD_GET(DCS_STATE, readl(s->dcs)); + if (state > DCS_STEP_STATE) { + writel(DCS_STEP_STATE, s->dcs); + t8140_dcs_wait(s, DCS_STEP_STATE); + } + if (state > DCS_SLEEP_STATE) { + writel(DCS_SLEEP_STATE, s->dcs); + t8140_dcs_wait(s, DCS_SLEEP_STATE); + } +} + +/* The boot core's last WFI. An event that ends it is taken, and the WFI taken again. */ +static void __noreturn t8140_boot_wfi(struct t8140_sleep *s) +{ + t8140_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + aic_sleep_ack(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +/* XNU polls these off just before its complex words. The sleep goes on whatever they read. */ +static void t8140_sleep_wait_still_on(struct t8140_sleep *s) +{ + static const u32 words[] = { + PS_ATC0_COMMON, PS_DISP_SYS, PS_DISPEXT0_SYS, PS_ANS, 0x270 /* AVD_SYS */, + 0x240 /* VENC0_SYS */, 0x248 /* VENC1_SYS */, 0x1f0 /* GFX */, 0x230 /* JPG */, + PS_APCIE_SYS_GP, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(words); i++) + t8140_ps_wait_off(s, words[i]); +} + +/* + * The last mile, in macOS's order. It starts after every refusal: this can't be + * undone, and returning now would leave the clusters armed off. No printing, and + * under 33 ms from the complex words to the WFI (the engine's bound for the + * cores to go idle). A hang from here ends at the watchdog. + * + * What's left out on purpose: + * - The two writes of 0 to 0x302800000: reads 0, changes nothing. + * - The RTC alarm: macOS arms it after the commit. rtcwake has armed it long + * before this, with no SPMI traffic near the engine. + */ +static int t8140_sleep_commit(unsigned long unused) +{ + struct t8140_sleep *s = t8140_sleep; + int plane, agent; + u32 val; + + /* The other cores stop first, as on macOS. They stay coherent until the commit. */ + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + s->cpus_stop_ret = spin_table_stop_parked(); + + t8140_sleep_arm_wd2(s); + t8140_sleep_wait_still_on(s); + + /* The SoC comes down before memory does, as on macOS. */ + if (t8140_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE)) { + writel(SOC_SLEEP_STATE, s->clk_perf + CLK_PERF_SOC); + t8140_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE); + } + t8140_sleep_dcs_down(s); + t8140_sleep_mcc_stop(s); + + /* Gates, PTD, the first voter twice, gates: macOS's order, measured once. */ + for (agent = 0; agent < 2; agent++) { + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + } + writeq(0, s->ptd + PTD_A); + writeq(0, s->ptd + PTD_B); + for (agent = 0; agent < 2; agent++) { + readl(s->pmc_voters); + writel(0, s->pmc_voters); + readl(s->pmc_voters + 4); + } + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + + /* Already off since .prepare, as on macOS. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_commit_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, 5000); + writel(0, s->cluster + CLUSTER_E); + writel(0, s->cluster + CLUSTER_P); + + writel(readl(s->ps + PS_SIO_CPU) & ~PS_AUTO_ENABLE, s->ps + PS_SIO_CPU); + + /* The SoC refuses the first, on macOS too. */ + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_38); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_3C); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_34); + + writel(readl(s->cllt + CLLT_0) & ~CLLT_0_CLEAR, s->cllt + CLLT_0); + writel(readl(s->cllt + CLLT_1) & ~CLLT_1_CLEAR, s->cllt + CLLT_1); + writel(readl(s->cllt + CLLT_2) & ~CLLT_2_CLEAR, s->cllt + CLLT_2); + + /* + * Two runs put these writes after the CLLT's: 5 to 6 ms before the commit + * in one, and the CLLT at 5.75 ms in the other. No run has hooked both. + */ + t8140_dvfm_park(s); + + t8140_ps_save(s); + for (plane = 0; plane < ARRAY_SIZE(s->amcc); plane++) { + void __iomem *amcc = s->amcc[plane]; + + writel(readl(amcc + AMCC_WORD0_ENABLE) & ~AMCC_WORD0_DSID_AGE_OUT, + amcc + AMCC_WORD0_ENABLE); + writel(readl(amcc + AMCC_WORD1_ENABLE) & ~AMCC_WORD1_UNEXP_RT_HIT, + amcc + AMCC_WORD1_ENABLE); + } + + /* aura reads this with the MMU off. */ + s->resumes_before = READ_ONCE(s->mbox->resumes); + WRITE_ONCE(s->mbox->resume_pa, __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dsb(sy); + + writel(1, s->commit); + + /* As on macOS: a raised line would end the last WFI, and the engine waits for it. */ + aic_sleep_mask_all(); + + /* Each core's deep WFI comes after the commit, this one last. No SPMI from here on. */ + spin_table_release_stopped(); + t8140_boot_wfi(s); +} + +/* Back on the boot core through the reset vector, MMU on, in macOS's order. */ +static void t8140_sleep_return(struct t8140_sleep *s) +{ + u32 val; + int i; + + /* + * The reset leaves the timer control at its EL2 default, without the event stream + * or EL0's counter. The timer driver's CPU PM exit puts it back too, later. + */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + + /* This core takes the AIC's IRQs, and a reset may have changed that. */ + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + + spin_table_stop_finished(); + aic_sleep_restore(); + + writel(CLUSTER_ON, s->cluster + CLUSTER_E); + s->cluster_e_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_E, val, + val & CLUSTER_E_READY, 1, + CLUSTER_TIMEOUT_US); + writel(CLUSTER_ON, s->cluster + CLUSTER_P); + + writel(PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_1); + + t8140_ps_rewake(s); + + val = readl(s->ps + PS_SEP); + if (val & PS_DEV_DISABLE) + writel(val & ~PS_DEV_DISABLE, s->ps + PS_SEP); + + /* The complex comes back on, and macOS turns it off. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, CLUSTER_TIMEOUT_US); + + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + + t8140_dvfm_unpark(s); + + /* macOS stops it here. Keep a longer one over device resume, until .end. */ + t8140_sleep_set_wd2(s, WDT_SUSPEND_SECS); + + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + s->resumes_after = READ_ONCE(s->mbox->resumes); + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_status[i] = readl(s->wake + WAKE_STATUS(i)); + s->returned = true; +} + +static int t8140_sleep_enter(suspend_state_t state) +{ + struct t8140_sleep *s = t8140_sleep; + int ret; + + ret = t8140_sleep_refuse(s); + if (ret) + return ret; + /* As on macOS, the display goes off before the GPU. */ + ret = t8140_display_off(s); + if (!ret) + ret = t8140_sleep_coprocs_check(s); + if (!ret) + ret = t8140_sleep_enclave_tell(s); + if (ret) { + t8140_sleep_stray_restore(s); + return ret; + } + t8140_sleep_coprocs_stop(s, true); + + t8140_wake_arm(s); + t8140_sleep_match(s); + + if (readl(s->wake + WAKE_SLP_DDR)) + writel(0, s->wake + WAKE_SLP_DDR); + /* Acknowledge a leftover done flag, as the DCS word has none on macOS. */ + if (readl(s->cluster_dcs) & CLUSTER_UP_DONE) + writel(readl(s->cluster_dcs), s->cluster_dcs); + t8140_sleep_coprocs_stop(s, false); + /* Done by now unless no SMC driver is bound. */ + t8140_sleep_usb_down(s); + + s->cntkctl = read_sysreg(cntkctl_el1); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, t8140_sleep_commit); + if (!ret) + t8140_sleep_return(s); + + t8140_sleep_match_undo(s); + t8140_sleep_stray_restore(s); + t8140_wake_restore(s); + + return ret; +} + +/* From here to .end a hang resets the machine. Nobody has to be there to recover it. */ +static int t8140_sleep_begin(suspend_state_t state) +{ + /* The last sleep's cut is its port controller's to take, by its .complete. */ + WRITE_ONCE(t8140_debug_link_cut, false); + if (t8140_sleep) + t8140_sleep->usb_down_ran = false; + t8140_sleep_set_wd2(t8140_sleep, WDT_SUSPEND_SECS); + pm_set_suspend_via_firmware(); + + return 0; +} + +static void t8140_sleep_end(void) +{ + writel(0, t8140_sleep->wdt + WDT_WD2_CTL); +} + +/* + * macOS turns this complex off at boot and again at the end of each wake, so at the commit + * its word is already 0 and the engine has no sequence to run for it. Turning it off at the + * commit instead puts that sequence, with its untimed wait, right before the CPU complexes'. + */ +static void t8140_sleep_ave_off(struct t8140_sleep *s) +{ + u32 val; + int ret; + + /* Target 0, and the leftover done flag from iBoot's up, as XNU's first write does. */ + writel(CLUSTER_UP_DONE, s->cluster + CLUSTER_AVE); + ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, 1, CLUSTER_TIMEOUT_US); + if (ret) + dev_warn(s->dev, "AVE complex didn't go off: %#x\n", val); +} + +static int t8140_sleep_prepare(void) +{ + struct t8140_sleep *s = t8140_sleep; + int i; + + t8140_sleep_ave_off(s); + + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + struct platform_device *pdev; + + s->display[i].bound = false; + if (!s->display[i].np) + continue; + pdev = of_find_device_by_node(s->display[i].np); + if (pdev) { + s->display[i].bound = device_is_bound(&pdev->dev); + put_device(&pdev->dev); + } + } + t8140_sleep_gate_prepare(s); + + return 0; +} + +static void t8140_sleep_report(void) +{ + struct t8140_sleep *s = t8140_sleep; + int i; + + if (!s->returned) + return; + s->returned = false; + + dev_info(s->dev, "back from sleep, boot object resumes %llu -> %llu\n", + s->resumes_before, s->resumes_after); + for (i = 0; i < ARRAY_SIZE(t8140_wake_names); i++) { + u32 id = t8140_wake_names[i].id; + + if (s->wake_status[id / 32] & BIT(id % 32)) + dev_info(s->dev, "wake source: %s\n", t8140_wake_names[i].name); + } + t8140_ps_rewake_report(s); + + if (s->cpus_stop_ret) + dev_err(s->dev, "other CPUs did not stop: %d\n", s->cpus_stop_ret); + if (s->cluster_e_ret || s->cluster_ave_ret || s->cluster_ave_commit_ret) + dev_err(s->dev, "cluster power timed out: E %d AVE %d, AVE at the commit %d\n", + s->cluster_e_ret, s->cluster_ave_ret, s->cluster_ave_commit_ret); + if (s->usb_down_ret) + dev_err(s->dev, "debug link: the stop failed: %d\n", s->usb_down_ret); +} + +static const struct platform_suspend_ops t8140_sleep_ops = { + .valid = suspend_valid_only_mem, + .begin = t8140_sleep_begin, + .prepare = t8140_sleep_prepare, + .enter = t8140_sleep_enter, + .wake = t8140_sleep_report, + .end = t8140_sleep_end, +}; + +static int t8140_sleep_read_wake_sources(struct t8140_sleep *s) +{ + struct device_node *np = s->dev->of_node; + int count; + + count = of_property_read_variable_u32_array(np, "apple,wake-enable", s->wake_enable, + 1, WAKE_SOURCES_MAX); + if (count < 0) + return dev_err_probe(s->dev, count, "no wake source enabled\n"); + s->wake_enable_count = count; + + return 0; +} + +static int t8140_sleep_map_displays(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + struct device_node *np; + int index; + + np = of_parse_phandle(s->dev->of_node, "apple,display-coprocessors", i); + if (!np) + return dev_err_probe(s->dev, -ENODEV, "no display coprocessor %d\n", i); + s->display[i].np = np; + + index = of_property_match_string(np, "reg-names", "coproc"); + if (index < 0) + return dev_err_probe(s->dev, index, "%pOF has no coproc registers\n", np); + s->display[i].asc = of_iomap(np, index); + if (!s->display[i].asc) + return dev_err_probe(s->dev, -ENOMEM, "cannot map %pOF\n", np); + } + + return 0; +} + +static void __iomem *t8140_sleep_map(struct device *dev, const char *name) +{ + struct device_node *np = dev->of_node; + struct resource res; + void __iomem *base; + int index; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return IOMEM_ERR_PTR(index); + if (of_address_to_resource(np, index, &res)) + return IOMEM_ERR_PTR(-EINVAL); + + /* Shared with the drivers of these blocks, so not claimed. */ + base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!base) + return IOMEM_ERR_PTR(-ENOMEM); + + return base; +} + +/* The register blocks every sleep writes, by their reg-names. */ +static const struct { + const char *name; + size_t offset; +} t8140_sleep_regs[] = { + { "ps", offsetof(struct t8140_sleep, ps) }, + { "pwrgate", offsetof(struct t8140_sleep, pwrgate) }, + { "cpu-start", offsetof(struct t8140_sleep, cpu_start) }, + { "cllt", offsetof(struct t8140_sleep, cllt) }, + { "cluster", offsetof(struct t8140_sleep, cluster) }, + { "commit", offsetof(struct t8140_sleep, commit) }, + { "wdt", offsetof(struct t8140_sleep, wdt) }, + { "aon-misc", offsetof(struct t8140_sleep, aon_misc) }, + { "wake", offsetof(struct t8140_sleep, wake) }, + { "pmc-dcp", offsetof(struct t8140_sleep, pmc_dcp) }, + { "spmi", offsetof(struct t8140_sleep, spmi) }, + { "pmc-voters", offsetof(struct t8140_sleep, pmc_voters) }, + { "dcs", offsetof(struct t8140_sleep, dcs) }, + { "ps-aon", offsetof(struct t8140_sleep, ps_aon) }, + { "spmi-a0", offsetof(struct t8140_sleep, spmi_a0) }, + { "clk-mux", offsetof(struct t8140_sleep, clk_mux) }, + { "mcc0", offsetof(struct t8140_sleep, mcc[0]) }, + { "mcc1", offsetof(struct t8140_sleep, mcc[1]) }, + { "mcc2", offsetof(struct t8140_sleep, mcc[2]) }, + { "ptd", offsetof(struct t8140_sleep, ptd) }, + { "clk-perf", offsetof(struct t8140_sleep, clk_perf) }, + { "cluster-dcs", offsetof(struct t8140_sleep, cluster_dcs) }, + { "dvfm-e", offsetof(struct t8140_sleep, dvfm[0]) }, + { "dvfm-p", offsetof(struct t8140_sleep, dvfm[1]) }, + { "amcc0", offsetof(struct t8140_sleep, amcc[0]) }, + { "amcc1", offsetof(struct t8140_sleep, amcc[1]) }, + { "amcc2", offsetof(struct t8140_sleep, amcc[2]) }, + { "amcc3", offsetof(struct t8140_sleep, amcc[3]) }, +}; + +static int t8140_sleep_map_regs(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t8140_sleep_regs); i++) { + void __iomem **field = (void __iomem **)((u8 *)s + t8140_sleep_regs[i].offset); + + *field = t8140_sleep_map(s->dev, t8140_sleep_regs[i].name); + if (IS_ERR(*field)) + return dev_err_probe(s->dev, PTR_ERR(*field), "no %s registers\n", + t8140_sleep_regs[i].name); + } + + return 0; +} + +static int t8140_sleep_map_mailbox(struct t8140_sleep *s) +{ + struct device_node *chosen; + u64 pa; + int ret; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return -ENODEV; + ret = of_property_read_u64(chosen, "aurora,s2r-mailbox", &pa); + of_node_put(chosen); + if (ret) { + dev_info(s->dev, "no aurora,s2r-mailbox from the boot object: no deep sleep\n"); + return -ENODEV; + } + + s->mbox = devm_memremap(s->dev, pa, sizeof(*s->mbox), MEMREMAP_WB); + if (IS_ERR(s->mbox)) + return PTR_ERR(s->mbox); + + /* aura wrote it with the MMU off, so nothing of it can be in a cache. */ + if (s->mbox->magic != T8140_S2R_MAGIC) + return dev_err_probe(s->dev, -EINVAL, "bad mailbox magic %llx\n", + s->mbox->magic); + /* Probe can run on any CPU. Logical CPU 0 is always the boot CPU. */ + if (s->mbox->boot_mpidr != cpu_logical_map(0)) + return dev_err_probe(s->dev, -EINVAL, + "mailbox is for CPU %llx, not this boot CPU\n", + s->mbox->boot_mpidr); + + return 0; +} + +static void t8140_sleep_map_debug_port(struct t8140_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,debug-port", 0); + struct device_node *bus; + + if (!np) + return; + bus = of_get_parent(np); + if (bus && !of_property_read_u32_index(np, "reg", 0, &s->hpm_sid)) + s->hpm_spmi = of_iomap(bus, 0); + of_node_put(bus); + /* Kept, for as long as the driver: no unbind. */ + s->hpm_np = np; +} + +static int t8140_sleep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct t8140_sleep *s; + int ret; + + /* Below EL2 a monitor owns all of this. */ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + s = devm_kzalloc(dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = dev; + + /* Without the boot object's half, a wake would rerun its boot flow. */ + ret = t8140_sleep_map_mailbox(s); + if (ret) + return ret; + + ret = t8140_sleep_map_regs(s); + if (ret) + return ret; + + ret = t8140_sleep_read_wake_sources(s); + if (ret) + return ret; + ret = t8140_sleep_map_displays(s); + if (ret) + return ret; + t8140_sleep_map_debug_port(s); + + platform_set_drvdata(pdev, s); + + ret = spin_table_set_power_ops(&t8140_cpu_power_ops); + if (ret) + return dev_err_probe(dev, ret, "cannot control CPU power\n"); + + /* Last: the exported helpers take it as a probed driver, and a failure frees s. */ + t8140_sleep = s; + /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ + suspend_set_ops(&t8140_sleep_ops); + + dev_info(dev, "deep sleep %s: boot object resumes %llu\n", + mem_sleep_current == PM_SUSPEND_MEM ? "is the default" : + "available, not the default", READ_ONCE(s->mbox->resumes)); + + return 0; +} + +static const struct of_device_id t8140_sleep_of_match[] = { + { .compatible = "apple,t8140-sleep" }, + { } +}; + +static struct platform_driver t8140_sleep_driver = { + .driver = { + .name = "t8140-sleep", + .of_match_table = t8140_sleep_of_match, + .suppress_bind_attrs = true, + }, + .probe = t8140_sleep_probe, +}; +builtin_platform_driver(t8140_sleep_driver); diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h new file mode 100644 index 0000000000000..a9f030d1ec7c3 --- /dev/null +++ b/include/linux/soc/apple/sleep.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What drivers tell the code that takes an Apple SoC into a firmware-driven + * suspend. + */ +#ifndef _LINUX_SOC_APPLE_SLEEP_H_ +#define _LINUX_SOC_APPLE_SLEEP_H_ + +#include +#include +#include +#include + +struct device; + +#if IS_ENABLED(CONFIG_APPLE_T8140_SLEEP) +/* + * A coprocessor that can't be stopped until a sleep is certain, and can't be + * restarted without one. While one is registered and not ready, a suspend is + * refused. Once ready, the last step before the commit runs @last_stop, with + * interrupts off and every refusal behind it, then clears its run bit. With + * @pcie_ans, that also lowers the PCIe and ANS domains, which nothing else may do + * under it. @last_stop can't refuse: a nonzero return means it failed, and the sleep + * goes on. + */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data); + +/* + * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power + * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. + */ +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data); +void apple_sleep_coproc_unregister(struct device *dev); + +/* + * For a registered coprocessor: the last of its stop that may still refuse the sleep, run + * with interrupts off before any coprocessor's @last_stop. A negative return refuses, and the + * coprocessor must still be able to carry on. + */ +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)); + +/* + * The enclave's sleep request, which it comes back from only through a real sleep: @tell + * runs with interrupts off as the very last refusal, after every other one and before any + * coprocessor stops. Nonzero refuses the sleep. + */ +int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), void *data); +void apple_sleep_enclave_unregister(struct device *dev); + +/* Its driver has quiesced it for that last step, or has taken it back. */ +void apple_sleep_coproc_set_ready(struct device *dev, bool ready); + +/* Whether the last step stopped it: after a refused sleep, false means it still runs. */ +bool apple_sleep_coproc_stopped(struct device *dev); + +/* + * The USB host sleeps in place for this sleep, as macOS's does: its always-on domain stays + * up through the commit. Its PHY sets it once the controller is in its hibernation state, + * and clears it at the wake. + */ +void apple_sleep_usb_set_asleep(bool asleep); + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The host that sleeps in place sets them before the freeze and clears them after. + */ +void apple_sleep_usb_set_wake_ports(unsigned long ports); + +/* For the port controller wired to the fixed hub's @port, as it arms USB wake. */ +bool apple_sleep_usb_port_can_wake(unsigned int port); + +/* The last sleep ended the debug link on @dev, whose driver may rearm it. */ +bool apple_sleep_debug_link_was_cut(struct device *dev); + +/* The same, without taking it: the port controller's own setup after its cold reset. */ +bool apple_sleep_debug_link_cut_peek(struct device *dev); + +/* + * End the debug link now, if this sleep hasn't: true if this call did. For the SMC's driver, + * ahead of its AP idle notice. It takes over a second, and doesn't sleep. + */ +bool apple_sleep_debug_link_cut(void); + +/* + * After a refused sleep, for a firmware that idled on a domain that can't gate alone: the + * sleep engine's PMGR reset of its own domains, its DARTs kept. The caller starts it after. + */ +int apple_sleep_engine_reset(struct device *dev); +#else +static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, + bool pcie_ans, int (*last_stop)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_coproc_register_first(struct device *dev, + int (*last_stop)(void *data), void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_coproc_unregister(struct device *dev) +{ +} + +static inline int apple_sleep_coproc_set_last_check(struct device *dev, + int (*last_check)(void *data)) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_enclave_unregister(struct device *dev) +{ +} + +static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ +} + +static inline bool apple_sleep_coproc_stopped(struct device *dev) +{ + return false; +} + +static inline void apple_sleep_usb_set_asleep(bool asleep) +{ +} + +static inline void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ +} + +static inline bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return false; +} + +static inline bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut(void) +{ + return false; +} + +static inline int apple_sleep_engine_reset(struct device *dev) +{ + return -EOPNOTSUPP; +} +#endif + +#endif From 932de6dc2b1f41bd09a0afd265aac0473971bdbf Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sun, 8 Nov 2026 03:32:01 -0500 Subject: [PATCH 31/73] arm64: smp, soc: apple: t8140-sleep: refuse a sleep a CPU won't stop for The parked CPUs were asked to stop only in the commit, after the GPU, the PMP and the enclave had gone down, when nothing could refuse any more. A CPU that didn't answer within 100 ms stayed awake in its park, and the commit went on. Measured on a J700 with a test switch that makes one CPU miss its stop, the sleep then never returned: the sleep engine waits on the awake core until WD2 resets the SoC, so the restart at the reset vector after the wake never gets to run. A parked CPU now answers a hold first. Right after the platform's first checks in .enter, before the display goes off, spin_table_hold_parked() asks every parked CPU to answer and waits up to 100 ms. A CPU that answers sets its held bit and waits, still parked, with nothing written to the hardware. If one doesn't answer, the hold is undone and the sleep refuses there, as it would for a CPU left online. A later refusal (the display, the last checks, the enclave) undoes the hold too, so every refusal finds the CPUs in their park and the PM core starts them as usual. The hold also refuses for a CPU that is offline but neither held nor powered off: one whose start failed after a wake, or one never started. Such a core may be awake anywhere, and the commit would wait on it the same way. At the commit, spin_table_stop_parked() stops only held CPUs. Each writes its stop bit and waits for the release at the same point as before, so the hardware sequence is unchanged. The stop request now stays up until the wake, so a held CPU that is slow still stops. macOS doesn't refuse here: its boot core waits for each secondary's token, which follows the secondary's stop-bit write, and panics ("failed to reach ARM_CPU_ON_SLEEP_PATH") when the wait runs out. No register tells a parked core from a running one before its deep WFI, so the answer is a software one here too. Signed-off-by: Diljit Singh --- arch/arm64/include/asm/cpu_ops.h | 9 +++ arch/arm64/kernel/smp_spin_table.c | 109 +++++++++++++++++++++++++---- drivers/soc/apple/t8140-sleep.c | 25 ++++++- 3 files changed, 127 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 1bebfa1c12efc..4ed50e0dc0b5c 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -66,11 +66,20 @@ struct spin_table_power_ops { #ifdef CONFIG_HOTPLUG_CPU int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +int spin_table_hold_parked(void); +void spin_table_unhold_parked(void); int spin_table_stop_parked(void); void spin_table_release_stopped(void); void spin_table_stop_finished(void); bool spin_table_cpus_stuck(void); #else +static inline int spin_table_hold_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_unhold_parked(void) {} + static inline int spin_table_stop_parked(void) { return -EOPNOTSUPP; diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index f6590af9d79df..2d8819416a489 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -146,12 +146,18 @@ const struct cpu_operations smp_spin_table_ops = { * It is then started again at its reset vector. Stopping and starting a CPU is * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug * fails cleanly, as on plain spin-table. + * + * A CPU left awake at the commit can hang the machine, and the commit can't be + * undone. So each parked CPU first answers a hold, while the suspend can still + * refuse, and only a held CPU stops. */ #ifdef CONFIG_HOTPLUG_CPU static const struct spin_table_power_ops *spin_table_power __read_mostly; static struct cpumask spin_table_cpus_parked; +static struct cpumask spin_table_cpus_held; static struct cpumask spin_table_cpus_unpark; static struct cpumask spin_table_cpus_off; +static bool spin_table_hold_requested; static bool spin_table_stop_requested; static bool spin_table_stop_released; @@ -162,9 +168,67 @@ int spin_table_set_power_ops(const struct spin_table_power_ops *ops) return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; } +/* A refused suspend: the held CPUs go back to the park. */ +void spin_table_unhold_parked(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + dsb(ish); + sev(); +} + +/* + * Every CPU not online is held or off. One that is neither, parked or not (a start that + * failed, a CPU never started), may be awake anywhere. + */ +static bool spin_table_all_held(bool report) +{ + unsigned int cpu; + bool all = true; + + for_each_present_cpu(cpu) { + if (cpu_online(cpu) || cpumask_test_cpu(cpu, &spin_table_cpus_held) || + cpumask_test_cpu(cpu, &spin_table_cpus_off)) + continue; + if (!report) + return false; + pr_err("CPU%u did not answer the hold for a suspend\n", cpu); + all = false; + } + + return all; +} + /* - * For a caller about to suspend the system, interrupts off: parked CPUs stop and - * wait for spin_table_release_stopped(). Returns -ETIMEDOUT if any didn't stop. + * Before a suspend's last refusal, interrupts off, other CPUs offline: every parked CPU + * answers and waits, still parked, for spin_table_stop_parked() or + * spin_table_unhold_parked(). Nothing is written to the hardware. Returns -ETIMEDOUT, + * with the hold undone, if any CPU not online is neither held nor off. + */ +int spin_table_hold_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_hold_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (spin_table_all_held(false)) + return 0; + udelay(1); + } + + if (spin_table_all_held(true)) + return 0; + spin_table_unhold_parked(); + + return -ETIMEDOUT; +} + +/* + * At the commit, after spin_table_hold_parked(): the held CPUs stop and wait for + * spin_table_release_stopped(). Returns -ETIMEDOUT if any didn't stop. The request + * stays up until the wake, so a late one still stops. */ int spin_table_stop_parked(void) { @@ -176,12 +240,11 @@ int spin_table_stop_parked(void) for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { if (cpumask_empty(&spin_table_cpus_parked)) - break; + return 0; udelay(1); } - WRITE_ONCE(spin_table_stop_requested, false); - return cpumask_empty(&spin_table_cpus_parked) ? 0 : -ETIMEDOUT; + return -ETIMEDOUT; } /* After the commit: the stopped CPUs take their last WFI. */ @@ -198,9 +261,12 @@ void spin_table_release_stopped(void) */ void spin_table_stop_finished(void) { + WRITE_ONCE(spin_table_hold_requested, false); + WRITE_ONCE(spin_table_stop_requested, false); WRITE_ONCE(spin_table_stop_released, false); cpumask_or(&spin_table_cpus_off, &spin_table_cpus_off, &spin_table_cpus_parked); cpumask_clear(&spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_held); cpumask_clear(&spin_table_cpus_unpark); } @@ -257,6 +323,28 @@ static void smp_rvbar_spin_table_restart(void) } #endif +/* Held until the commit's stop, or until a refusal sends it back to the park. */ +static void smp_rvbar_spin_table_hold(unsigned int cpu, u64 mpidr) +{ + cpumask_set_cpu(cpu, &spin_table_cpus_held); + + while (READ_ONCE(spin_table_hold_requested)) { + if (READ_ONCE(spin_table_stop_requested)) { + spin_table_power->cpu_stop(mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_off); + cpumask_clear_cpu(cpu, &spin_table_cpus_held); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + while (!READ_ONCE(spin_table_stop_released)) + wfe(); + spin_table_power->cpu_off(mpidr); + } + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_held); +} + static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) { u64 mpidr = cpu_logical_map(cpu); @@ -270,15 +358,8 @@ static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) cpumask_set_cpu(cpu, &spin_table_cpus_parked); while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { - if (READ_ONCE(spin_table_stop_requested)) { - spin_table_power->cpu_stop(mpidr); - cpumask_set_cpu(cpu, &spin_table_cpus_off); - cpumask_clear_cpu(cpu, &spin_table_cpus_parked); - - while (!READ_ONCE(spin_table_stop_released)) - wfe(); - spin_table_power->cpu_off(mpidr); - } + if (READ_ONCE(spin_table_hold_requested)) + smp_rvbar_spin_table_hold(cpu, mpidr); wfe(); } diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c index e787b3bede8aa..d1f4f3d65398c 100644 --- a/drivers/soc/apple/t8140-sleep.c +++ b/drivers/soc/apple/t8140-sleep.c @@ -639,6 +639,21 @@ static int t8140_sleep_coprocs_check(struct t8140_sleep *s) return 0; } +/* + * The other cores stop at the commit, and one left awake there keeps the engine waiting + * until WD2 resets the SoC. Each answers here, before the display goes off, and waits for the + * stop or a refusal. + */ +static int t8140_sleep_cpus_hold(struct t8140_sleep *s) +{ + int ret = spin_table_hold_parked(); + + if (ret) + dev_err(s->dev, "refused: a CPU didn't answer the hold: %d\n", ret); + + return ret; +} + /* Last of all the refusals: an enclave that gated comes back only through a real sleep. */ static int t8140_sleep_enclave_tell(struct t8140_sleep *s) { @@ -1958,7 +1973,10 @@ static int t8140_sleep_commit(unsigned long unused) int plane, agent; u32 val; - /* The other cores stop first, as on macOS. They stay coherent until the commit. */ + /* + * The other cores stop first, as on macOS. Each answered the hold in .enter, and they + * stay coherent until the commit. + */ s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); s->cpus_stop_ret = spin_table_stop_parked(); @@ -2101,13 +2119,16 @@ static int t8140_sleep_enter(suspend_state_t state) ret = t8140_sleep_refuse(s); if (ret) return ret; + ret = t8140_sleep_cpus_hold(s); /* As on macOS, the display goes off before the GPU. */ - ret = t8140_display_off(s); + if (!ret) + ret = t8140_display_off(s); if (!ret) ret = t8140_sleep_coprocs_check(s); if (!ret) ret = t8140_sleep_enclave_tell(s); if (ret) { + spin_table_unhold_parked(); t8140_sleep_stray_restore(s); return ret; } From 9d13f7b130a50cd02c04187b2faebfb95cd576fc Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:15 -0500 Subject: [PATCH 32/73] soc: apple: t8140-sleep: take ANS's PCIe block down and back up ANS's NAND link runs on the shared PCIe block, whether or not the PCIe host driver is bound, and macOS takes that block down with ANS's domains and brings it back from power-off after the wake. Until this, the sleep controller had to refuse any sleep with ANS on. At the commit, after every refusal and ANS's own last stop: the radio's link into L2 and its engine stopped and the port PHY's clocks off (see below), then the PHY, the clock generator and the AXI enables, each domain lowered in macOS's order, deepest first and ANS last. The PHY is only quieted with both its domains still on, as its registers need them. On the return, the domains come back in the reverse order and the block starts from power-off with the boot loader's tunables, measured on one macOS sleep: AXI enables, the CIO3 PLL and clock generator, both of the generator's off bits cleared, the PHY's clock requests, its first run bit, the PHY's PLL and lane tables, its second run bit and ready poll, then the RC's ready handshake. This happens on the waking core, well before ANS runs. A block whose registers were left off by the loader is skipped, as they fault with their domain off. The power loss takes most of the PHY's PLL settings with it, and ANS's firmware then never hears from its link. The tables go in as macOS writes them: each entry read-modify-write in table order, PLL then lanes, with no write where nothing changes, then the PLL's hold bit cleared. That takes the time the PHY needs between its run bits, so there's no separate wait. The radio's port is stopped only as the loader set it up. Nothing sets it up again after a wake yet, and its registers then abort at the next sleep's commit, so later sleeps leave it alone. The registers come from the PCIe node named by apple,apcie, where the loader publishes the tunables, even with that host disabled. A missing tunable refuses a sleep with ANS stopped, since the block couldn't come back. As that would be every real sleep, s2idle then stays the default unless the command line asks for deep. The PHY's tables, apple,tunable-phy-ip-pll and -auspma, need a loader that publishes them from the ADT. Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 1 + drivers/soc/apple/t8140-sleep.c | 417 +++++++++++++++++++++++++++++++- 2 files changed, 413 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index a3c46e7ba313e..ae3dc45793a7d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -83,6 +83,7 @@ config APPLE_T8140_SLEEP bool "Apple T8140 system sleep at EL2" depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) depends on HOTPLUG_CPU && SUSPEND + select APPLE_TUNABLE help Powers the secondary CPUs off and suspends the system to RAM on an Apple T8140 whose kernel runs directly at EL2, with no monitor to ask. diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c index d1f4f3d65398c..8f72c47ab3b8f 100644 --- a/drivers/soc/apple/t8140-sleep.c +++ b/drivers/soc/apple/t8140-sleep.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -233,7 +234,62 @@ struct t8140_s2r_mailbox { #define ASC_CPU_STATUS 0x48 #define PS_ANS 0x278 +#define PS_APCIE_GP 0x258 #define PS_APCIE_SYS_GP 0x2a0 +#define PS_APCIE_ST 0x310 +#define PS_APCIE_SYS_ST 0x3a0 +#define PS_APCIE_PHY_SW 0x3c0 + +/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ +#define APCIE_AXI_EN0 0x104 +#define APCIE_AXI_EN1 0x108 +#define APCIE_CLK_CTRL 0x0 +#define APCIE_CLK_BUSY BIT(31) +#define APCIE_CLK_RUN BIT(5) +#define APCIE_CLK_OFF BIT(1) +#define APCIE_CLK_OFF2 BIT(2) +#define APCIE_CLK_START 0x4 +#define APCIE_CLK_START_GO BIT(31) +#define APCIE_PHY_COMMON 0x4000 +#define APCIE_PHY_CTRL 0x8000 +#define APCIE_PHY_CTRL_CLK0REQ BIT(0) +#define APCIE_PHY_CTRL_CLK1REQ BIT(1) +#define APCIE_PHY_CTRL_CLK0ACK BIT(2) +#define APCIE_PHY_CTRL_CLK1ACK BIT(3) +#define APCIE_PHY_CTRL_RESET BIT(4) +#define APCIE_PHY_CTRL_UP BIT(27) +#define APCIE_PHY_RUN 0x8004 +#define APCIE_PHY_RUN_EN BIT(0) +#define APCIE_PHY_RUN_EN2 BIT(4) +#define APCIE_PHY_READY 0x8008 +#define APCIE_PHY_IP_PLL_CTL 0x90 +#define APCIE_PHY_IP_PLL_HOLD BIT(16) +#define APCIE_PORT_INTSTAT 0x100 +#define APCIE_PORT_INTMSK 0x104 +#define APCIE_PORT_INT_LINK GENMASK(7, 4) +#define APCIE_PORT_L2 0x88 +#define APCIE_PORT_L2_ENTER 0x11 +#define APCIE_PORT_L2_BUSY BIT(0) +#define APCIE_PORT_PERST_GUARD 0x13c +#define APCIE_PORT_LINKSTS 0x208 +#define APCIE_PORT_LINKSTS_UP BIT(0) +#define APCIE_PORT_APPCLK 0x800 +#define APCIE_PORT_APPCLK_EN BIT(0) +#define APCIE_PORT_APPCLK_CGDIS BIT(8) +#define APCIE_PORT_PERST 0x82c +#define APCIE_PORT_PHY_LINK BIT(31) +#define APCIE_PORT_PHY_EN0 BIT(9) +#define APCIE_PORT_PHY_EN1 BIT(10) +#define APCIE_ENGINE_CTRL 0x0 +#define APCIE_ENGINE_IRQ_MASK 0x8 +#define APCIE_ENGINE_STOP BIT(0) +#define APCIE_FABRIC_STOP 0x4 +#define APCIE_PORT_STOP_US 5000 +#define APCIE_RC_ZERO 0x04 +#define APCIE_RC_READY_START 0x50 +#define APCIE_RC_READY_CFG 0x54 +#define APCIE_RC_READY_STAT 0x58 +#define APCIE_POLL_US 50000 /* The coprocessors the last mile stops. Each driver idles its own first. */ #define SLEEP_COPROCS 4 @@ -371,6 +427,28 @@ struct t8140_sleep { unsigned int ps_rewake_timeouts; u64 cntkctl; int cpus_stop_ret; + int ans_off_ret; + int pcie_ans_on_ret; + void __iomem *apcie_axi; + void __iomem *apcie_cio3pll; + void __iomem *apcie_clkgen; + void __iomem *apcie_phy; + void __iomem *apcie_rc; + bool apcie_port_stopped; + /* A real sleep left the radio's port down, and nothing sets it up again */ + bool apcie_port_down; + void __iomem *apcie_port; + void __iomem *apcie_port_phy; + void __iomem *apcie_engine; + void __iomem *apcie_fabric; + struct apple_tunable *apcie_axi_tunable; + struct apple_tunable *apcie_cio3pll_tunable; + struct apple_tunable *apcie_clkgen_tunable; + struct apple_tunable *apcie_phy_tunable; + void __iomem *apcie_phy_ip; + struct apple_tunable *apcie_phy_ip_pll; + struct apple_tunable *apcie_phy_ip_lanes; + bool pcie_ans_stopped; /* The words t8140_sleep_lower_stray() lowered, one bit each */ u8 stray_lowered; /* What must be bound for a sleep, and what must have stopped by the last step */ @@ -678,11 +756,14 @@ static void t8140_sleep_coprocs_stop(struct t8140_sleep *s, bool first) { int i; + if (first) + s->pcie_ans_stopped = false; for (i = 0; i < SLEEP_COPROCS; i++) { struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; if (!c->dev || !c->ready || c->first != first) continue; + s->pcie_ans_stopped |= c->pcie_ans; if (c->stopped) continue; if (c->last_stop) @@ -1241,9 +1322,15 @@ static int t8140_sleep_refuse(struct t8140_sleep *s) refused = true; } - /* ANS lowers its PCIe block with it, which this can't bring back yet. */ - if (idled || t8140_ps_on(s, PS_ANS) || t8140_ps_on(s, PS_APCIE_SYS_GP)) { - dev_err(s->dev, "refused: ANS or its PCIe block is on\n"); + /* ANS's link needs these after a real power loss. */ + if (idled && !s->apcie_phy_tunable) { + dev_err(s->dev, "refused: no tunables to bring ANS's PCIe block back\n"); + refused = true; + } + + /* Only ANS's driver can stop it, and it doesn't come back without a real sleep. */ + if (!idled && (t8140_ps_on(s, PS_ANS) || t8140_ps_on(s, PS_APCIE_SYS_GP))) { + dev_err(s->dev, "refused: ANS or PCIe is on with no driver to stop it\n"); refused = true; } @@ -1480,7 +1567,225 @@ bool apple_sleep_debug_link_cut(void) } EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); -/* About 4 ms after the other cores stop on macOS: deepest first. */ +/* + * The radio's link and the engine that starts it, in macOS's order, measured on one + * sleep: the link into L2, the engine stopped, PERST# asserted, the PHY's clocks off. + * macOS does it 40 ms before ANS's idle. Here it follows, so that a refused sleep finds + * the link untouched. A link that's down or a radio that's gone only makes the poll time + * out, and the power goes next either way. + */ +static void t8140_apcie_port_stop(struct t8140_sleep *s) +{ + void __iomem *port = s->apcie_port; + void __iomem *phy = s->apcie_port_phy; + u32 val; + + writel(readl(port + APCIE_PORT_INTMSK) | APCIE_PORT_INT_LINK, port + APCIE_PORT_INTMSK); + writel(APCIE_PORT_INT_LINK, port + APCIE_PORT_INTSTAT); + + writel(readl(port + APCIE_PORT_APPCLK) | APCIE_PORT_APPCLK_CGDIS, port + APCIE_PORT_APPCLK); + /* A loader can leave the port ready with the link never trained. */ + if (readl(port + APCIE_PORT_LINKSTS) & APCIE_PORT_LINKSTS_UP) { + writel(APCIE_PORT_L2_ENTER, port + APCIE_PORT_L2); + readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), + 10, APCIE_PORT_STOP_US); + } + writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); + + if (s->apcie_engine && s->apcie_fabric) { + writel(0, s->apcie_engine + APCIE_ENGINE_IRQ_MASK); + writel(3, s->apcie_fabric + APCIE_FABRIC_STOP); + writel(readl(s->apcie_engine + APCIE_ENGINE_CTRL) | APCIE_ENGINE_STOP, + s->apcie_engine + APCIE_ENGINE_CTRL); + } + udelay(1000); + + /* What this bit does isn't known. macOS sets it around PERST#. */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(0, port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), port + APCIE_PORT_PERST_GUARD); + + writel(readl(phy) & ~APCIE_PORT_PHY_EN1, phy); + writel(readl(phy) & ~APCIE_PORT_PHY_EN0, phy); + writel(readl(phy) | APCIE_PHY_CTRL_RESET, phy); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK0REQ, phy); + readl_poll_timeout_atomic(phy, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK1REQ, phy); + + writel(readl(port + APCIE_PORT_APPCLK) & ~APCIE_PORT_APPCLK_EN, port + APCIE_PORT_APPCLK); + writel(readl(port + APCIE_PORT_INTSTAT), port + APCIE_PORT_INTSTAT); +} + +static void t8140_apcie_port_stop_once(struct t8140_sleep *s) +{ + /* Only as the loader set it up: after a wake its registers abort. */ + if (s->apcie_port_stopped || s->apcie_port_down || !s->apcie_port || !s->apcie_port_phy || + !t8140_ps_on(s, PS_APCIE_GP) || !t8140_ps_on(s, PS_APCIE_PHY_SW)) + return; + s->apcie_port_stopped = true; + t8140_apcie_port_stop(s); +} + +/* A poll here only paces the steps: the power goes next whatever it reads. */ +static void t8140_apcie_phy_off(struct t8140_sleep *s) +{ + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + + writel(readl(ctrl) | APCIE_PHY_CTRL_RESET, ctrl); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK1REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK1ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK0REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_UP, ctrl); + writel(readl(run) & ~APCIE_PHY_RUN_EN2, run); + writel(readl(run) & ~APCIE_PHY_RUN_EN, run); +} + +static void t8140_apcie_clk_off(struct t8140_sleep *s) +{ + void __iomem *clk = s->apcie_clkgen + APCIE_CLK_CTRL; + u32 val; + + writel(readl(clk) & ~APCIE_CLK_RUN, clk); + writel(readl(clk) | APCIE_CLK_OFF, clk); + readl_poll_timeout_atomic(clk, val, !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + writel(readl(clk) | APCIE_CLK_OFF2, clk); +} + +/* + * ANS's own link runs on this block, and macOS takes it down with the domains. Its + * order, measured on one sleep, in the same microseconds as the power words. A block's + * registers fault with its domain off, and a loader may have left any of these off. + */ +static void t8140_sleep_lower_apcie(struct t8140_sleep *s) +{ + u32 val; + + t8140_apcie_port_stop_once(s); + /* The PHY's registers need both domains on. */ + if (s->apcie_phy && t8140_ps_on(s, PS_APCIE_PHY_SW) && t8140_ps_on(s, PS_APCIE_SYS_ST)) + t8140_apcie_phy_off(s); + t8140_ps_lower(s, PS_APCIE_PHY_SW); + + if (s->apcie_clkgen && t8140_ps_on(s, PS_APCIE_SYS_ST)) + t8140_apcie_clk_off(s); + t8140_ps_lower(s, PS_APCIE_SYS_ST); + t8140_ps_lower(s, PS_APCIE_ST); + + val = t8140_ps_lower_start(s, PS_APCIE_SYS_GP); + if (s->apcie_axi && t8140_ps_on(s, PS_APCIE_SYS_GP)) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + t8140_ps_lower_finish(s, PS_APCIE_SYS_GP, val); + + t8140_ps_lower(s, PS_APCIE_GP); + t8140_ps_lower(s, PS_ANS); +} + +/* Read-modify-write in table order, with no write where nothing changes. */ +static void t8140_tunable_apply_changed(void __iomem *regs, const struct apple_tunable *t) +{ + size_t i; + + for (i = 0; i < t->sz; i++) { + void __iomem *reg = regs + t->values[i].offset; + u32 old = readl(reg); + u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); + + if (val != old) + writel(val, reg); + } +} + +/* + * The power loss takes the PHY's PLL and lane settings with it, and ANS's link can't + * train without them. Applied as macOS does: the PLL table, then the lanes, then the + * PLL's hold released. That takes about the settling time the PHY needs here. + */ +static void t8140_apcie_phy_ip_apply(struct t8140_sleep *s) +{ + void __iomem *ctl = s->apcie_phy_ip + APCIE_PHY_IP_PLL_CTL; + + t8140_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); + t8140_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); + writel(readl(ctl) & ~APCIE_PHY_IP_PLL_HOLD, ctl); +} + +/* + * Brought up from power-off in macOS's order, measured on one sleep: all of it follows + * the domains on the waking core, well before ANS runs. Whether ANS needs each step is + * inferred. Nothing else does this if no host driver is bound. + */ +static int t8140_sleep_raise_apcie(struct t8140_sleep *s) +{ + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + int ret; + + ret = t8140_ps_raise(s, PS_ANS); + ret |= t8140_ps_raise(s, PS_APCIE_GP); + ret |= t8140_ps_raise_start(s, PS_APCIE_SYS_GP); + if (!ret) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) | BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) | BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + t8140_ps_raise_finish(s, PS_APCIE_SYS_GP); + ret |= t8140_ps_raise(s, PS_APCIE_ST); + ret |= t8140_ps_raise(s, PS_APCIE_SYS_ST); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_axi, s->apcie_axi_tunable); + apple_tunable_apply(s->apcie_cio3pll, s->apcie_cio3pll_tunable); + apple_tunable_apply(s->apcie_clkgen, s->apcie_clkgen_tunable); + writel(readl(s->apcie_clkgen + APCIE_CLK_START) | APCIE_CLK_START_GO, + s->apcie_clkgen + APCIE_CLK_START); + /* Both off bits: the lowering sets them, and only a power-off clears OFF. */ + writel(readl(s->apcie_clkgen + APCIE_CLK_CTRL) & ~(APCIE_CLK_OFF2 | APCIE_CLK_OFF), + s->apcie_clkgen + APCIE_CLK_CTRL); + ret = readl_poll_timeout_atomic(s->apcie_clkgen + APCIE_CLK_CTRL, val, + !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + if (ret) + return ret; + + ret = t8140_ps_raise(s, PS_APCIE_PHY_SW); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_phy, s->apcie_phy_tunable); + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK0REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK1REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) & ~APCIE_PHY_CTRL_RESET, ctrl); + + writel(readl(run) | APCIE_PHY_RUN_EN, run); + t8140_apcie_phy_ip_apply(s); + writel(readl(run) | APCIE_PHY_RUN_EN2, run); + ret = readl_poll_timeout_atomic(s->apcie_phy + APCIE_PHY_READY, val, val & BIT(0), 1, + APCIE_POLL_US); + if (ret) + return ret; + writel(readl(s->apcie_phy + APCIE_PHY_COMMON) | BIT(0), s->apcie_phy + APCIE_PHY_COMMON); + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + + writel(0, s->apcie_rc + APCIE_RC_ZERO); + writel(0x140, s->apcie_rc + APCIE_RC_READY_CFG); + writel(1, s->apcie_rc + APCIE_RC_READY_START); + + return readl_poll_timeout_atomic(s->apcie_rc + APCIE_RC_READY_STAT, val, val & BIT(0), 1, + APCIE_POLL_US); +} + +/* About 4 ms after the other cores stop on macOS: deepest first, ANS last. */ static void t8140_sleep_lower_io(struct t8140_sleep *s) { /* @@ -1491,6 +1796,10 @@ static void t8140_sleep_lower_io(struct t8140_sleep *s) t8140_ps_lower(s, PS_ATC0_COMMON); if (!READ_ONCE(t8140_usb_asleep)) t8140_ps_lower(s, PS_ATC0_USB_AON); + if (s->pcie_ans_stopped) { + t8140_sleep_lower_apcie(s); + s->ans_off_ret = t8140_ps_wait_off(s, PS_ANS); + } } static void t8140_sleep_set_wd2(struct t8140_sleep *s, unsigned int secs) @@ -1980,6 +2289,7 @@ static int t8140_sleep_commit(unsigned long unused) s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); s->cpus_stop_ret = spin_table_stop_parked(); + t8140_sleep_lower_io(s); t8140_sleep_arm_wd2(s); t8140_sleep_wait_still_on(s); @@ -2103,6 +2413,15 @@ static void t8140_sleep_return(struct t8140_sleep *s) /* macOS stops it here. Keep a longer one over device resume, until .end. */ t8140_sleep_set_wd2(s, WDT_SUSPEND_SECS); + /* + * What the commit lowered for a driver comes back before anything uses it, in + * macOS's order, measured about 14 ms after this point. With no driver, genpd's + * books decide, as they do for every other domain. + */ + if (s->pcie_ans_stopped) + s->pcie_ans_on_ret = t8140_sleep_raise_apcie(s); + s->apcie_port_down = true; + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); s->resumes_after = READ_ONCE(s->mbox->resumes); @@ -2164,8 +2483,10 @@ static int t8140_sleep_begin(suspend_state_t state) { /* The last sleep's cut is its port controller's to take, by its .complete. */ WRITE_ONCE(t8140_debug_link_cut, false); - if (t8140_sleep) + if (t8140_sleep) { + t8140_sleep->apcie_port_stopped = false; t8140_sleep->usb_down_ran = false; + } t8140_sleep_set_wd2(t8140_sleep, WDT_SUSPEND_SECS); pm_set_suspend_via_firmware(); @@ -2242,8 +2563,13 @@ static void t8140_sleep_report(void) if (s->cluster_e_ret || s->cluster_ave_ret || s->cluster_ave_commit_ret) dev_err(s->dev, "cluster power timed out: E %d AVE %d, AVE at the commit %d\n", s->cluster_e_ret, s->cluster_ave_ret, s->cluster_ave_commit_ret); + if (s->ans_off_ret) + dev_err(s->dev, "ANS did not read off at the commit: %d\n", s->ans_off_ret); if (s->usb_down_ret) dev_err(s->dev, "debug link: the stop failed: %d\n", s->usb_down_ret); + if (s->pcie_ans_stopped && s->pcie_ans_on_ret) + dev_err(s->dev, "ANS's PCIe block did not come up after the wake: %d\n", + s->pcie_ans_on_ret); } static const struct platform_suspend_ops t8140_sleep_ops = { @@ -2398,6 +2724,83 @@ static int t8140_sleep_map_mailbox(struct t8140_sleep *s) return 0; } +static struct apple_tunable *t8140_sleep_apcie_tunable(struct t8140_sleep *s, + struct device_node *np, + const char *name, const char *prop) +{ + int idx = of_property_match_string(np, "reg-names", name); + struct apple_tunable *t; + struct resource res; + + if (idx < 0 || of_address_to_resource(np, idx, &res)) + return NULL; + t = devm_apple_tunable_parse(s->dev, np, prop, &res); + + return IS_ERR(t) ? NULL : t; +} + +static void __iomem *t8140_sleep_apcie_block(struct t8140_sleep *s, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable) +{ + int idx = of_property_match_string(np, "reg-names", name); + + if (idx < 0) + return NULL; + if (prop) { + *tunable = t8140_sleep_apcie_tunable(s, np, name, prop); + if (!*tunable) + return NULL; + } + + return of_iomap(np, idx); +} + +/* + * Its registers go with PCIe's node, which a test tree may have disabled, and a loader + * publishes the tunables there. Missing ones only refuse a real sleep with ANS stopped. + */ +static void t8140_sleep_map_apcie(struct t8140_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,apcie", 0); + struct device_node *engine; + + if (!np) + return; + s->apcie_axi = t8140_sleep_apcie_block(s, np, "axi2af", "apple,tunable-axi2af", + &s->apcie_axi_tunable); + s->apcie_cio3pll = t8140_sleep_apcie_block(s, np, "cio3pll", "apple,tunable-cio3pll", + &s->apcie_cio3pll_tunable); + s->apcie_clkgen = t8140_sleep_apcie_block(s, np, "clkgen", "apple,tunable-clkgen", + &s->apcie_clkgen_tunable); + s->apcie_phy = t8140_sleep_apcie_block(s, np, "phy-block", "apple,tunable-phy", + &s->apcie_phy_tunable); + s->apcie_rc = t8140_sleep_apcie_block(s, np, "rc", NULL, NULL); + s->apcie_phy_ip = t8140_sleep_apcie_block(s, np, "phy-ip", "apple,tunable-phy-ip-pll", + &s->apcie_phy_ip_pll); + s->apcie_phy_ip_lanes = t8140_sleep_apcie_tunable(s, np, "phy-ip", + "apple,tunable-phy-ip-auspma"); + if (!s->apcie_axi || !s->apcie_cio3pll || !s->apcie_clkgen || !s->apcie_phy || + !s->apcie_rc || !s->apcie_phy_ip || !s->apcie_phy_ip_lanes) { + dev_info(s->dev, + "no PCIe tunables for ANS: a real sleep with it stopped refuses, s2idle stays the default\n"); + s->apcie_phy_tunable = NULL; + s->apcie_phy = NULL; + s->apcie_clkgen = NULL; + s->apcie_axi = NULL; + } + + s->apcie_port = t8140_sleep_apcie_block(s, np, "port0", NULL, NULL); + s->apcie_port_phy = t8140_sleep_apcie_block(s, np, "phy0", NULL, NULL); + engine = of_parse_phandle(np, "apple,piodma", 0); + if (engine) { + s->apcie_engine = t8140_sleep_apcie_block(s, engine, "engine", NULL, NULL); + s->apcie_fabric = t8140_sleep_apcie_block(s, engine, "fabric", NULL, NULL); + of_node_put(engine); + } + of_node_put(np); +} + static void t8140_sleep_map_debug_port(struct t8140_sleep *s) { struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,debug-port", 0); @@ -2443,6 +2846,7 @@ static int t8140_sleep_probe(struct platform_device *pdev) ret = t8140_sleep_map_displays(s); if (ret) return ret; + t8140_sleep_map_apcie(s); t8140_sleep_map_debug_port(s); platform_set_drvdata(pdev, s); @@ -2455,6 +2859,9 @@ static int t8140_sleep_probe(struct platform_device *pdev) t8140_sleep = s; /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ suspend_set_ops(&t8140_sleep_ops); + /* Not when every real sleep would refuse because ANS couldn't come back. */ + if (!s->apcie_phy_tunable && mem_sleep_default == PM_SUSPEND_MAX) + mem_sleep_current = PM_SUSPEND_TO_IDLE; dev_info(dev, "deep sleep %s: boot object resumes %llu\n", mem_sleep_current == PM_SUSPEND_MEM ? "is the default" : From 1cc0c969301c54d630366cdf811627b074d1ad27 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:15 -0500 Subject: [PATCH 33/73] nvme-apple: take ANS through a firmware-driven suspend A deep sleep takes ANS's power away, and ANS doesn't come back from idle without losing it: on a dry run it acknowledged idle and then a wake timed out with no IOP acknowledgment, and the disk was gone until a reboot. So anything that can be taken back happens in .suspend, and everything else happens in the sleep controller's last mile, after the last thing that can refuse: - .suspend flushes the OS's namespace, freezes the queues and waits out what's in flight, refusing the sleep if requests are still there after the I/O timeout. Only namespace 1: on this controller 2 to 4 are small system namespaces, and a Flush to one sets CSTS.CFS. The flush is there because these controllers are known to revert fsynced writes after an abrupt reset. - The shutdown notification, as macOS has it (CC 0x46c001, CSTS 0x9), polled with interrupts off as the driver's last check: after the display is down and before any coprocessor stops. A failure still refuses, as the firmware hasn't been told, and the resume's controller reset takes the controller back, as after a shutdown timeout. - The last stop, polled with interrupts off: AP and IOP idle, the mailbox's receive bit, then the sleep controller clears the run bit and lowers ANS's PCIe block. From the first idle message only a real sleep brings ANS back, so a failure goes on to the commit: what a flush acknowledged is on flash, and the watchdog is the net. On a real wake the firmware restarts when the run bit is set. Measured on macOS: HELLO comes unprompted, the map is answered with the first part echoed and the last with endpoint 0x20 only, no endpoint is started and no buffer asked for, and about 68 ms later the firmware reports on. The run bit goes in the early resume phase so HELLO comes within milliseconds, as on macOS. The wait for "on" happens in the reset work, after the SEP's resume, since macOS rings the SEP a few ms after ANS's HELLO and before ANS reports on. If ANS doesn't come back, no reset is scheduled: the queues stay frozen, every request waits instead of failing, and no filesystem sees a write error. The SART's entries are written back once more right before the run bit, as macOS does. Only controllers whose sleep has been traced (T8140's) register with the sleep controller; others suspend as before. Signed-off-by: Diljit Singh --- drivers/nvme/host/apple.c | 307 +++++++++++++++++++++++++++++++++++++- 1 file changed, 301 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 732d7b63be3d4..a5124a299e4f2 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -39,17 +39,21 @@ #include #include #include +#include +#include #include #include #include "nvme.h" #define APPLE_ANS_BOOT_TIMEOUT USEC_PER_SEC +#define APPLE_ANS_SLEEP_SHN_TIMEOUT USEC_PER_SEC /* RTKit handshake timeout for the T6050 ANS, see apple_nvme_alloc() */ #define APPLE_NVME_T6050_HANDSHAKE_TIMEOUT_MS 10000 #define APPLE_ANS_COPROC_CPU_CONTROL 0x44 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_ANS_COPROC_CPU_STATUS 0x48 #define APPLE_ANS_ACQ_DB 0x1004 #define APPLE_ANS_IOCQ_DB 0x100c @@ -210,6 +214,14 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; + /* Firmware-driven suspend has been traced on this controller. */ + bool sleep_traced; + /* The last endpoint map reply from that trace */ + u32 sleep_epmap_last_mask; + /* Higher namespaces are the firmware's, and a Flush to one faults the controller. */ + u32 sleep_flush_max_nsid; + /* CC.SHN for a sleep, as read back from the OS the sleep was traced on. */ + u32 sleep_shn; u32 max_queue_depth; unsigned long quirks; }; @@ -234,6 +246,17 @@ struct apple_nvme { bool recover_to_live; bool suspended; bool final_shutdown; + /* This suspend is firmware-driven: ANS will lose power at its end. */ + bool sleep_stop; + bool sleep_idled; + u32 sleep_csts; + /* ANS came back across a sleep, or failed to. reset_work picks these up. */ + bool sleep_restarted; + bool sleep_failed; + /* Run bit set after a sleep, and not yet reported on. */ + bool sleep_restarting; + ktime_t sleep_run_time; + bool sleep_registered; unsigned long recovery_flags; void __iomem *mmio_coproc; @@ -1394,6 +1417,31 @@ static int apple_nvme_disable(struct apple_nvme *anv, bool shutdown) return apple_nvme_disable_locked(anv, shutdown); } +/* + * All of a sleep's stop that can be taken back. Nothing is in flight or can start, and + * the controller and its firmware keep running for a refused sleep to come back to. + */ +static int apple_nvme_sleep_quiesce(struct apple_nvme *anv) +{ + guard(mutex)(&anv->disable_lock); + if (READ_ONCE(anv->quarantined)) + return -EIO; + + if (!test_bit(NVME_CTRL_FROZEN, &anv->ctrl.flags)) + nvme_start_freeze(&anv->ctrl); + if (!nvme_wait_freeze_timeout(&anv->ctrl)) { + /* A reset cancels what's still in flight. Don't sleep on that. */ + dev_warn(anv->dev, "requests still in flight after the timeout\n"); + return -EBUSY; + } + nvme_quiesce_io_queues(&anv->ctrl); + nvme_quiesce_admin_queue(&anv->ctrl); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); + + return 0; +} + static void apple_nvme_recovery_work(struct work_struct *work) { struct apple_nvme *anv = container_of(work, struct apple_nvme, recovery_work); @@ -1612,6 +1660,8 @@ static void apple_nvme_wait_freeze(struct apple_nvme *anv) } } +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv); + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1632,6 +1682,17 @@ static void apple_nvme_reset_work(struct work_struct *work) return; } WRITE_ONCE(anv->abort_reset, false); + if (READ_ONCE(anv->sleep_restarting) && !apple_nvme_sleep_wait_on(anv)) + WRITE_ONCE(anv->sleep_failed, true); + if (READ_ONCE(anv->sleep_failed)) { + /* + * Starting it some other way can only reset it, which faults, and failing the + * controller fails every held write. Leave the queues quiesced and frozen. + */ + if (phase_locked) + mutex_unlock(&anv->disable_lock); + return; + } if (state != NVME_CTRL_RESETTING) { dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state); ret = -ENODEV; @@ -1688,6 +1749,12 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = -EIO; goto out; } + /* A real sleep took the controller's state with its power. */ + if (READ_ONCE(anv->sleep_restarted)) { + WRITE_ONCE(anv->sleep_restarted, false); + if (!(readl(anv->mmio_nvme + NVME_REG_CC) & NVME_CC_ENABLE)) + goto rtkit_ready; + } ret = apple_nvme_disable_locked(anv, false); if (ret) goto out; @@ -2528,6 +2595,57 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) return ERR_PTR(ret); } +/* + * A sleep's shutdown notification, with interrupts off, before anything else stops. macOS + * finds the controller enabled and shut down when it tells the firmware to idle. A failure + * refuses: the firmware hasn't been told, and resume's controller reset takes it back. + */ +static int apple_nvme_sleep_shutdown(void *data) +{ + struct apple_nvme *anv = data; + u32 csts; + int ret; + + anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; + anv->ctrl.ctrl_config |= anv->hw->sleep_shn; + writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); + ret = readl_poll_timeout_atomic(anv->mmio_nvme + NVME_REG_CSTS, csts, + (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, + 100, APPLE_ANS_SLEEP_SHN_TIMEOUT); + if (ret) + dev_err(anv->dev, "shutdown notification failed: CSTS %#x\n", csts); + anv->sleep_csts = csts; + + return ret; +} + +/* + * The rest of a sleep's stop, last of all. From the first idle message only a real sleep + * brings it back, so a failure goes on to the commit: what a flush acknowledged is on + * flash, and the watchdog is the net. + */ +static int apple_nvme_sleep_last_stop(void *data) +{ + struct apple_nvme *anv = data; + unsigned int us; + u32 cc, csts = anv->sleep_csts; + int ret; + + cc = readl(anv->mmio_nvme + NVME_REG_CC); + + anv->sleep_idled = true; + ret = apple_rtkit_sleep_idle_polled(anv->rtk, + anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL, &us); + if (ret) + dev_err(anv->dev, "ANS did not idle: %d\n", ret); + else + dev_dbg(anv->dev, "ANS idle, acknowledged in about %u us, CC %#x CSTS %#x\n", + us, cc, csts); + apple_rtkit_mailbox_recv_off(anv->rtk); + + return ret ? 1 : 0; +} + static int apple_nvme_probe(struct platform_device *pdev) { struct apple_nvme *anv; @@ -2556,6 +2674,19 @@ static int apple_nvme_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + if (anv->hw->sleep_traced) { + anv->sleep_registered = + !apple_sleep_coproc_register(anv->dev, anv->mmio_coproc, true, + apple_nvme_sleep_last_stop, anv); + /* Never idled without its shutdown notification. */ + if (anv->sleep_registered && + apple_sleep_coproc_set_last_check(anv->dev, apple_nvme_sleep_shutdown)) { + apple_sleep_coproc_unregister(anv->dev); + anv->sleep_registered = false; + } + apple_rtkit_set_warm_epmap_mask(anv->rtk, anv->hw->sleep_epmap_last_mask); + } + apple_nvme_set_reset_ready(anv, true); nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -2574,6 +2705,8 @@ static void apple_nvme_remove(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + if (anv->sleep_registered) + apple_sleep_coproc_unregister(anv->dev); WRITE_ONCE(anv->removing, true); apple_nvme_set_reset_ready(anv, false); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); @@ -2639,6 +2772,80 @@ static void apple_nvme_shutdown(struct platform_device *pdev) } } +/* macOS sees the firmware report on 68 ms after HELLO. Each try waits RTKit's timeout again. */ +#define APPLE_NVME_SLEEP_ON_TRIES 15 + +/* + * After a real sleep ANS has lost power, and its resident firmware restarts once the run bit + * is set. Measured on macOS under aura, the same in five runs: HELLO first and unprompted, + * then the endpoint map, the last part answered with endpoint 0x20 only. No endpoint is + * started and no buffer asked for. About 68 ms later the firmware reports on, and the AP state + * follows, with the controller's registers reading 0. A refused suspend never told the + * firmware anything, and takes a reset. + * + * The run bit goes in the early phase, so HELLO comes within milliseconds of the wake as on + * macOS. The wait for on doesn't: macOS rings the SEP a few ms after ANS's HELLO, before ANS + * reports on, so the reset work waits, after the SEP's own resume has had its turn. + */ +static int apple_nvme_resume_early(struct device *dev) +{ + struct apple_nvme *anv = dev_get_drvdata(dev); + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret; + + if (!anv->hw->needs_ioq_registers || !anv->suspended || !anv->sleep_idled) + return 0; + anv->sleep_idled = false; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + /* macOS writes the SART's entries back again right before the run bit. */ + if (anv->sart) + apple_sart_replay(anv->sart); + + dev_dbg(dev, "ANS: restarting after the sleep, CPU control %#x\n", readl(control)); + anv->sleep_run_time = ktime_get(); + ret = apple_rtkit_sleep_return_start(anv->rtk, control); + if (ret) { + dev_crit(dev, "ANS restart after sleep failed: %d\n", ret); + anv->sleep_failed = true; + return 0; + } + WRITE_ONCE(anv->sleep_restarting, true); + + return 0; +} + +/* From the reset work: wait for ANS to report on. False means hold every request. */ +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv) +{ + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret, tries = 0; + + do { + ret = apple_rtkit_boot(anv->rtk); + if (ret != -ETIME) + break; + dev_warn(anv->dev, "ANS: not on %lld ms after the run bit, waiting again\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + } while (++tries < APPLE_NVME_SLEEP_ON_TRIES); + + WRITE_ONCE(anv->sleep_restarting, false); + if (ret) { + dev_crit(anv->dev, "ANS did not come back after sleep: %d after %lld ms, CPU control %#x status %#x, crashed %d\n", + ret, ktime_ms_delta(ktime_get(), anv->sleep_run_time), readl(control), + readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_STATUS), + apple_rtkit_is_crashed(anv->rtk)); + dev_crit(anv->dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return false; + } + WRITE_ONCE(anv->sleep_restarted, true); + /* The one line per wake that says ANS is back. */ + dev_info(anv->dev, "ANS: on %lld ms after the run bit, resetting the controller before any I/O\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + + return true; +} + static int apple_nvme_resume(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2648,6 +2855,20 @@ static int apple_nvme_resume(struct device *dev) return -EIO; if (!anv->suspended) return 0; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + anv->sleep_idled = false; + anv->sleep_stop = false; + /* + * Hold every request rather than fail it: the queues stay frozen and quiesced + * from the suspend, so nothing reaches a controller that isn't there and no + * filesystem sees a write error. Only a reboot brings ANS back from this. + */ + if (READ_ONCE(anv->sleep_failed)) { + dev_crit(dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return 0; + } + /* The reset work waits for ANS first, if it's still coming up. */ WRITE_ONCE(anv->suspended, false); return nvme_try_sched_reset(&anv->ctrl); } @@ -2676,19 +2897,74 @@ static void apple_nvme_keep_link_powered(struct apple_nvme *anv) device_set_awake_path(anv->pd_dev[i]); } +/* + * What is written has to be on flash before power goes: an fsynced overwrite is + * known to revert after an abrupt reset on these controllers unless flushed. + */ +static int apple_nvme_sleep_flush(struct apple_nvme *anv) +{ + struct nvme_ctrl *ctrl = &anv->ctrl; + struct nvme_ns *ns; + int srcu_idx, ret = 0; + + srcu_idx = srcu_read_lock(&ctrl->srcu); + list_for_each_entry_srcu(ns, &ctrl->namespaces, list, + srcu_read_lock_held(&ctrl->srcu)) { + struct nvme_command c = { }; + + if (ns->head->ns_id > anv->hw->sleep_flush_max_nsid) + continue; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(ns->head->ns_id); + ret = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (ret) + break; + } + srcu_read_unlock(&ctrl->srcu, srcu_idx); + + return ret; +} + +static bool apple_nvme_sleep_wanted(struct apple_nvme *anv) +{ + return anv->sleep_registered && + pm_suspend_target_state == PM_SUSPEND_MEM && pm_suspend_via_firmware() && + nvme_ctrl_state(&anv->ctrl) == NVME_CTRL_LIVE && anv->owns_rtkit && + apple_rtkit_is_running(anv->rtk); +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); int ret; if (anv->hw->needs_ioq_registers) { - if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) + anv->sleep_stop = apple_nvme_sleep_wanted(anv); + anv->sleep_idled = false; + anv->sleep_failed = false; + anv->sleep_restarted = false; + if (anv->sleep_stop) { + ret = apple_nvme_sleep_flush(anv); + if (ret) { + dev_err(dev, "refusing to sleep: flush failed: %d\n", ret); + anv->sleep_stop = false; + return -EIO; + } + } + if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) { + anv->sleep_stop = false; return -EBUSY; + } WRITE_ONCE(anv->suspended, true); } - ret = apple_nvme_disable(anv, true); - if (ret) - return ret; + if (anv->sleep_stop) { + anv->sleep_failed = apple_nvme_sleep_quiesce(anv); + } else { + ret = apple_nvme_disable(anv, true); + if (ret) + return ret; + } /* * Only the NVMe controller was disabled. Its owned RTKit session stays * live, so resume rebuilds queues without clearing CPU_CONTROL.RUN. @@ -2696,6 +2972,19 @@ static int apple_nvme_suspend(struct device *dev) if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); apple_nvme_keep_link_powered(anv); + if (anv->sleep_failed) { + /* Requests are stuck, so bring it back and refuse. */ + dev_err(dev, "refusing to sleep: NVMe didn't quiesce\n"); + anv->sleep_failed = false; + anv->sleep_stop = false; + WRITE_ONCE(anv->suspended, false); + nvme_try_sched_reset(&anv->ctrl); + return -EIO; + } + if (anv->sleep_stop) { + apple_sleep_coproc_set_ready(dev, true); + dev_dbg(dev, "NVMe suspended for a sleep\n"); + } return 0; } @@ -2708,8 +2997,10 @@ static int apple_nvme_suspend(struct device *dev) return ret; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, - apple_nvme_resume); +static const struct dev_pm_ops apple_nvme_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_nvme_suspend, apple_nvme_resume) + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_nvme_resume_early) +}; static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, @@ -2742,6 +3033,10 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .has_separate_nvmmu = true, .needs_ioq_registers = true, .max_queue_depth = 64, + .sleep_traced = true, + .sleep_epmap_last_mask = BIT(0), + .sleep_flush_max_nsid = 1, + .sleep_shn = NVME_CC_SHN_MASK, /* * An fsynced overwrite completed and later reverted after an abrupt * reset; explicit flushes persisted the preceding data. Let the block From bc9540d806a9bfc7e76eea6e03bb1ba2c59efc39 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:16 -0500 Subject: [PATCH 34/73] soc: apple: t8140-sleep: put the AP pins back before ANS starts A deep sleep resets the AP pin controller: after macOS's wake every pin reads the same reset value. Right before ANS's run bit, macOS writes 1 at 0xc48, clears every interrupt group's pending bits and puts back 212 of the 216 pins, all but 49 to 51 and 134: each written back as it reads, then as it was, its lock bit last. The pin controller's driver puts back only the pins it configured itself. That was narrowed after an earlier restore of every pin took an SError; it wrote them without the 0xc48 write first and read all 216. It isn't enough for ANS: with only those back, ANS's firmware can't bring up its NAND link after the wake. Which of the other pins the link needs isn't known, so all of them go back, as macOS does it. Save the pins at the sleep's begin, before any device suspends, and put them back when ANS's driver asks, right before its run bit. The pin controller and the pins to leave alone come from the DT. On a J700, over three sleeps in a row, 113 of the 212 pins had changed each time and ANS came back each time. Signed-off-by: Diljit Singh --- drivers/nvme/host/apple.c | 2 + drivers/soc/apple/t8140-sleep.c | 99 +++++++++++++++++++++++++++++++++ include/linux/soc/apple/sleep.h | 7 +++ 3 files changed, 108 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index a5124a299e4f2..48b8433395de6 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2798,6 +2798,8 @@ static int apple_nvme_resume_early(struct device *dev) anv->sleep_idled = false; if (anv->sleep_stop) apple_sleep_coproc_set_ready(dev, false); + /* Its link doesn't come up with the pins as the sleep left them. */ + apple_sleep_restore_pins(); /* macOS writes the SART's entries back again right before the run bit. */ if (anv->sart) apple_sart_replay(anv->sart); diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c index 8f72c47ab3b8f..553eb7baa133c 100644 --- a/drivers/soc/apple/t8140-sleep.c +++ b/drivers/soc/apple/t8140-sleep.c @@ -26,6 +26,7 @@ */ #include +#include #include #include #include @@ -40,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -463,6 +465,14 @@ struct t8140_sleep { u32 ps; } gate[16]; unsigned int gate_count; + /* The AP pins, which a deep sleep resets */ + void __iomem *pins; + u32 *pins_saved; + unsigned long *pins_skip; + unsigned int npins; + unsigned int pin_irq_groups; + bool pins_saved_valid; + bool pins_lost; bool returned; }; @@ -533,6 +543,64 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev) } EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_peek); +#define PIN_REG(n) (4 * (n)) +#define PIN_LOCK BIT(21) +#define PIN_CTL 0xc48 +#define PIN_IRQ(g, w) (0x800 + 0x40 * (g) + 4 * (w)) + +/* Before any device suspends, with the pins' domain on. Skipped pins aren't read. */ +static void t8140_pins_save(struct t8140_sleep *s) +{ + unsigned int pin; + + s->pins_lost = false; + s->pins_saved_valid = false; + if (!s->pins) + return; + for (pin = 0; pin < s->npins; pin++) + if (!test_bit(pin, s->pins_skip)) + s->pins_saved[pin] = readl(s->pins + PIN_REG(pin)); + s->pins_saved_valid = true; +} + +/* + * A deep sleep resets every pin, and the pin controller's driver puts back only the + * ones it configured. macOS puts all of them back right before ANS's run bit, and + * ANS's link doesn't come up without that: 1 written at 0xc48, every interrupt + * group's pending bits cleared, then each pin written back as it reads, then as it + * was, its lock bit last. + */ +void apple_sleep_restore_pins(void) +{ + struct t8140_sleep *s = t8140_sleep; + unsigned int pin, grp, w, changed = 0; + + if (!s || !s->pins_lost) + return; + s->pins_lost = false; + + writel(1, s->pins + PIN_CTL); + for (grp = 0; grp < s->pin_irq_groups; grp++) + for (w = 0; w < DIV_ROUND_UP(s->npins, 32); w++) + writel(U32_MAX, s->pins + PIN_IRQ(grp, w)); + for (pin = 0; pin < s->npins; pin++) { + void __iomem *reg = s->pins + PIN_REG(pin); + u32 want = s->pins_saved[pin], cur; + + if (test_bit(pin, s->pins_skip)) + continue; + cur = readl(reg); + changed += cur != want; + writel(cur, reg); + if (want & PIN_LOCK) + writel(want & ~PIN_LOCK, reg); + writel(want, reg); + } + dev_info(s->dev, "%u of %u AP pins changed by the sleep, all put back\n", changed, + s->npins - bitmap_weight(s->pins_skip, s->npins)); +} +EXPORT_SYMBOL_GPL(apple_sleep_restore_pins); + /* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, int (*last_stop)(void *data), void *data) @@ -2421,6 +2489,7 @@ static void t8140_sleep_return(struct t8140_sleep *s) if (s->pcie_ans_stopped) s->pcie_ans_on_ret = t8140_sleep_raise_apcie(s); s->apcie_port_down = true; + s->pins_lost = s->pins_saved_valid; dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); s->resumes_after = READ_ONCE(s->mbox->resumes); @@ -2486,6 +2555,7 @@ static int t8140_sleep_begin(suspend_state_t state) if (t8140_sleep) { t8140_sleep->apcie_port_stopped = false; t8140_sleep->usb_down_ran = false; + t8140_pins_save(t8140_sleep); } t8140_sleep_set_wd2(t8140_sleep, WDT_SUSPEND_SECS); pm_set_suspend_via_firmware(); @@ -2801,6 +2871,34 @@ static void t8140_sleep_map_apcie(struct t8140_sleep *s) of_node_put(np); } +/* macOS leaves the skipped pins alone, and reading one may fault. */ +static void t8140_sleep_map_pins(struct t8140_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,pinctrl", 0); + u32 npins, pin; + int i, n; + + if (!np) + return; + if (!of_device_is_available(np) || of_property_read_u32(np, "apple,npins", &npins) || + !npins || npins > 512) + goto out; + s->pins_saved = devm_kcalloc(s->dev, npins, sizeof(*s->pins_saved), GFP_KERNEL); + s->pins_skip = devm_bitmap_zalloc(s->dev, npins, GFP_KERNEL); + if (!s->pins_saved || !s->pins_skip) + goto out; + n = of_property_count_u32_elems(s->dev->of_node, "apple,pinctrl-skip-pins"); + for (i = 0; i < n; i++) + if (!of_property_read_u32_index(s->dev->of_node, "apple,pinctrl-skip-pins", i, + &pin) && pin < npins) + set_bit(pin, s->pins_skip); + s->pin_irq_groups = of_irq_count(np); + s->npins = npins; + s->pins = of_iomap(np, 0); +out: + of_node_put(np); +} + static void t8140_sleep_map_debug_port(struct t8140_sleep *s) { struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,debug-port", 0); @@ -2847,6 +2945,7 @@ static int t8140_sleep_probe(struct platform_device *pdev) if (ret) return ret; t8140_sleep_map_apcie(s); + t8140_sleep_map_pins(s); t8140_sleep_map_debug_port(s); platform_set_drvdata(pdev, s); diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index a9f030d1ec7c3..62265b5ae32fd 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -83,6 +83,9 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev); */ bool apple_sleep_debug_link_cut(void); +/* After a deep sleep, put back the pins it reset. ANS's driver calls it before the run bit. */ +void apple_sleep_restore_pins(void); + /* * After a refused sleep, for a firmware that idled on a domain that can't gate alone: the * sleep engine's PMGR reset of its own domains, its DARTs kept. The caller starts it after. @@ -159,6 +162,10 @@ static inline bool apple_sleep_debug_link_cut(void) return false; } +static inline void apple_sleep_restore_pins(void) +{ +} + static inline int apple_sleep_engine_reset(struct device *dev) { return -EOPNOTSUPP; From c2549e05e1756e2ea49b91240be8044ab9817755 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:16 -0500 Subject: [PATCH 35/73] soc: apple: t8140-sleep: hold the radio in reset through a sleep macOS takes the radio's port through L2, then PERST#, then the port off, and keeps the radio's power and firmware. The port stop here only cleared the port's own reset register, and the radio's PERST# pin stayed driven released until the pin block lost its power. Drive it asserted after L2, with every refusal behind it. The pin comes from the enabled T8140 PCIe host's port, and is written only with the pins' domain on, which it reads at the commit. That stop also ran once per boot only: the first return marked the port down and nothing cleared that, so later commits skipped L2, the engine's stop and the PHY's clocks. The radio's host now says when it has its port back after a wake, with apple_sleep_radio_port_up(), and only then does the next commit stop it again. Signed-off-by: Diljit Singh --- drivers/soc/apple/t8140-sleep.c | 53 +++++++++++++++++++++++++++++++-- include/linux/soc/apple/sleep.h | 7 +++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c index 553eb7baa133c..dd37629042962 100644 --- a/drivers/soc/apple/t8140-sleep.c +++ b/drivers/soc/apple/t8140-sleep.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -235,6 +236,7 @@ struct t8140_s2r_mailbox { #define ASC_CPU_CONTROL_RUN BIT(4) #define ASC_CPU_STATUS 0x48 +#define PS_GPIO 0x120 #define PS_ANS 0x278 #define PS_APCIE_GP 0x258 #define PS_APCIE_SYS_GP 0x2a0 @@ -437,7 +439,7 @@ struct t8140_sleep { void __iomem *apcie_phy; void __iomem *apcie_rc; bool apcie_port_stopped; - /* A real sleep left the radio's port down, and nothing sets it up again */ + /* A real sleep left the radio's port down, until its host sets it up again */ bool apcie_port_down; void __iomem *apcie_port; void __iomem *apcie_port_phy; @@ -473,6 +475,9 @@ struct t8140_sleep { unsigned int pin_irq_groups; bool pins_saved_valid; bool pins_lost; + /* The radio port's PERST# among them, or -1 */ + int radio_perst; + bool radio_perst_low; bool returned; }; @@ -543,7 +548,16 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev) } EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_peek); +/* The radio's host set its port up again after a real sleep: the next commit may stop it. */ +void apple_sleep_radio_port_up(void) +{ + if (t8140_sleep) + WRITE_ONCE(t8140_sleep->apcie_port_down, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_port_up); + #define PIN_REG(n) (4 * (n)) +#define PIN_DATA BIT(0) #define PIN_LOCK BIT(21) #define PIN_CTL 0xc48 #define PIN_IRQ(g, w) (0x800 + 0x40 * (g) + 4 * (w)) @@ -1658,6 +1672,13 @@ static void t8140_apcie_port_stop(struct t8140_sleep *s) readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), 10, APCIE_PORT_STOP_US); } + /* The radio keeps its power and firmware, in reset until its host releases it. */ + if (s->radio_perst >= 0 && t8140_ps_on(s, PS_GPIO)) { + void __iomem *reg = s->pins + PIN_REG(s->radio_perst); + + val = readl(reg); + writel(s->radio_perst_low ? val & ~PIN_DATA : val | PIN_DATA, reg); + } writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); if (s->apcie_engine && s->apcie_fabric) { @@ -1686,7 +1707,7 @@ static void t8140_apcie_port_stop(struct t8140_sleep *s) static void t8140_apcie_port_stop_once(struct t8140_sleep *s) { - /* Only as the loader set it up: after a wake its registers abort. */ + /* Only with the port set up, by the loader or by its host after a wake, or it aborts. */ if (s->apcie_port_stopped || s->apcie_port_down || !s->apcie_port || !s->apcie_port_phy || !t8140_ps_on(s, PS_APCIE_GP) || !t8140_ps_on(s, PS_APCIE_PHY_SW)) return; @@ -2871,6 +2892,31 @@ static void t8140_sleep_map_apcie(struct t8140_sleep *s) of_node_put(np); } +/* The radio port's PERST# on the AP pins @np, from the enabled T8140 PCIe host's ports. */ +static void t8140_sleep_find_radio_perst(struct t8140_sleep *s, struct device_node *np) +{ + struct device_node *host, *port; + struct of_phandle_args args; + + host = of_find_compatible_node(NULL, NULL, "apple,t8140-pcie"); + if (!host) + return; + if (of_device_is_available(host)) { + for_each_available_child_of_node(host, port) { + if (of_parse_phandle_with_args(port, "reset-gpios", "#gpio-cells", 0, + &args)) + continue; + if (args.np == np && args.args_count >= 2 && args.args[0] < s->npins && + !test_bit(args.args[0], s->pins_skip)) { + s->radio_perst = args.args[0]; + s->radio_perst_low = args.args[1] & GPIO_ACTIVE_LOW; + } + of_node_put(args.np); + } + } + of_node_put(host); +} + /* macOS leaves the skipped pins alone, and reading one may fault. */ static void t8140_sleep_map_pins(struct t8140_sleep *s) { @@ -2878,6 +2924,7 @@ static void t8140_sleep_map_pins(struct t8140_sleep *s) u32 npins, pin; int i, n; + s->radio_perst = -1; if (!np) return; if (!of_device_is_available(np) || of_property_read_u32(np, "apple,npins", &npins) || @@ -2895,6 +2942,8 @@ static void t8140_sleep_map_pins(struct t8140_sleep *s) s->pin_irq_groups = of_irq_count(np); s->npins = npins; s->pins = of_iomap(np, 0); + if (s->pins) + t8140_sleep_find_radio_perst(s, np); out: of_node_put(np); } diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 62265b5ae32fd..39f2675178ebb 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -83,6 +83,9 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev); */ bool apple_sleep_debug_link_cut(void); +/* The radio's host has its port back after a real sleep, so the next commit may stop it. */ +void apple_sleep_radio_port_up(void); + /* After a deep sleep, put back the pins it reset. ANS's driver calls it before the run bit. */ void apple_sleep_restore_pins(void); @@ -162,6 +165,10 @@ static inline bool apple_sleep_debug_link_cut(void) return false; } +static inline void apple_sleep_radio_port_up(void) +{ +} + static inline void apple_sleep_restore_pins(void) { } From d6158d07a07be4f0761b3066e6d5239151c67366 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:17 -0500 Subject: [PATCH 36/73] mfd: macsmc: tell the SMC about a firmware-driven suspend The SMC keeps running through a firmware-driven suspend. macOS tells it first and stops it last, and on a wake it hears first. In order, measured on T8140: MBSE = slpw before any other coprocessor is told, 800 ms before the rest; MBSE = slpa, 6 ms, then the AP idle notice after all the others, then 30 ms; on a wake the AP on notice, MBSE and MBSW read, and MBSE = waka. Between the two notices the firmware takes no commands, so a command there answers -EAGAIN, which the SMC's clients take as "not now": a battery event that ran on the boot CPU after the wake and before the SMC's own resume logged failures from power_supply otherwise. Do the same: the warning in .prepare, the arm and the notice in suspend_noirq, the wake in resume_noirq, which gives that order whatever the device order is. Those phases have the mailbox's interrupt off, so commands there poll for their reply. The 800 ms after the warning is the time macOS's other drivers take to get ready for the sleep, not a wait the SMC asks for, so the warning doesn't wait: on a J700 that wait was 0.82 s of a 1.25 s entry. A refusal after the warning tells the SMC the sleep is off. KUnit tests cover the order and the refused cases. SMC.OutboxNotEmpty wakes the SoC while anything waits in the outbox. The sleep's debug link cut is a cable pulled to the SMC: it queues the port's and the charger's events and records them as wake reasons. With the SMC's wake sources armed, a sleep whose cut came after the AP idle notice committed and never returned. So suspend_noirq has the sleep driver cut the link before slpa and the notice, then polls the outbox until it has been quiet for 500 ms, up to 5 s. The events reach their handlers through the RTKit worker. A machine without a debug cable doesn't wait. An event that arrives later in the sleep's entry, the lid opened or a key pressed, is left in the outbox: the line stays raised and the sleep returns at once, as a wake would. The lid may have moved while the AP was off, with no event to say so, so macsmc-input reads it again after such a wake. Other sleeps don't change. Signed-off-by: Diljit Singh --- drivers/input/misc/macsmc-input.c | 11 ++ drivers/mfd/macsmc-test.c | 83 +++++++++++ drivers/mfd/macsmc.c | 235 +++++++++++++++++++++++++++++- include/linux/mfd/macsmc.h | 7 + 4 files changed, 335 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 853dfa73b7b8b..585ae57aca5e6 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -13,6 +13,7 @@ #include #include #include +#include /** * struct macsmc_input @@ -214,6 +215,16 @@ static void macsmc_input_pm_complete(struct device *dev) struct macsmc_input *smcin = dev_get_drvdata(dev); smcin->wakeup_mode = false; + + /* The lid may have moved while the AP was off and no event could arrive. */ + if (pm_resume_via_firmware() && apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD))) { + u8 val; + + if (apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { + input_report_switch(smcin->input, SW_LID, val); + input_sync(smcin->input); + } + } } static const struct dev_pm_ops macsmc_input_pm_ops = { diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index c4f9329dbc01c..2e0824a7c5825 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -9,8 +9,15 @@ struct macsmc_test_context { int result; unsigned int calls; bool require_sram_lock; + /* What the sleep tests saw, in order: keys read, values written, notices. */ + u32 seen[8]; + unsigned int seen_count; + int ap_power_result; }; +#define MACSMC_TEST_AP_IDLE 0 +#define MACSMC_TEST_AP_ON 1 + static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -18,6 +25,9 @@ static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); ctx->calls++; + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = cmd == SMC_MSG_WRITE_KEY ? + get_unaligned((u32 *)ctx->shmem) : arg; if (ret_data && ctx->result >= 0) *ret_data = ctx->data; return ctx->result; @@ -199,6 +209,76 @@ static void macsmc_key_info_lock_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); } +static int macsmc_test_ap_power(struct apple_smc *smc, bool on) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + /* The answer needs a worker that may be waiting for this mutex. */ + KUNIT_EXPECT_FALSE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = on ? MACSMC_TEST_AP_ON : MACSMC_TEST_AP_IDLE; + return ctx->ap_power_result; +} + +static void macsmc_sleep_order_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + apple_smc_sleep_warn(&ctx->smc); + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + + /* As in the trace, values included. */ + KUNIT_ASSERT_EQ(test, ctx->seen_count, 7U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[5], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[6], SMC_KEY(waka)); +} + +static void macsmc_sleep_refused_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->ap_power_result = -ETIME; + ctx->result = 4; + + /* No answer: refuse, undo the notice and leave commands working. */ + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), -ETIME); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_ON); +} + +static void macsmc_notice_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + /* The test trigger writes no MBSE value: those announce a real sleep. */ + KUNIT_EXPECT_EQ(test, apple_smc_notice_idle(&ctx->smc), 0); + apple_smc_notice_on(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(MBSW)); +} + static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_key_info_lock_test), KUNIT_CASE(macsmc_key_index_test), @@ -209,6 +289,9 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_request_bounds_test), KUNIT_CASE(macsmc_sram_value_test), KUNIT_CASE(macsmc_key_info_test), + KUNIT_CASE(macsmc_sleep_order_test), + KUNIT_CASE(macsmc_sleep_refused_test), + KUNIT_CASE(macsmc_notice_test), {} }; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 546e5cdbb72ad..0eb90e50b99d3 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -22,7 +22,10 @@ #include #include #include +#include #include +#include +#include #include #define SMC_ENDPOINT 0x20 @@ -75,6 +78,14 @@ static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) memcpy_fromio(buf, smc->shmem.iomem, size); } +/* In a noirq phase the mailbox's interrupt is off, so its reply has to be fetched. */ +static bool apple_smc_reply_arrived(struct apple_smc *smc) +{ + apple_rtkit_poll(smc->rtk); + + return completion_done(&smc->cmd_done); +} + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -91,6 +102,9 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return -EIO; if (smc->atomic_mode) return -EIO; + /* Between the AP power notices the firmware expects no commands. */ + if (smc->ap_notified) + return -EAGAIN; reinit_completion(&smc->cmd_done); @@ -107,7 +121,17 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return ret; } - if (wait_for_completion_timeout(&smc->cmd_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) <= 0) { + if (smc->polled) { + bool done; + + ret = read_poll_timeout(apple_smc_reply_arrived, done, done, 100, + SMC_TIMEOUT_MS * USEC_PER_MSEC, false, smc); + } else { + ret = wait_for_completion_timeout(&smc->cmd_done, + msecs_to_jiffies(SMC_TIMEOUT_MS)) > 0 ? + 0 : -ETIMEDOUT; + } + if (ret) { dev_err(smc->dev, "Command timed out (%llx)", msg); return -ETIMEDOUT; } @@ -591,10 +615,219 @@ static const struct of_device_id apple_smc_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_smc_of_match); +/* + * The SMC keeps running through a firmware-driven suspend. It is told first and + * stops last, and on a wake it hears first. In order, measured on T8140 and + * unverified on other SoCs: MBSE = slpw before any other coprocessor is told; + * MBSE = slpa and the AP power notice after all the others; on a wake the notice, + * MBSE and MBSW read, MBSE = waka. The noirq phases give that order whatever the + * device order is, so these commands poll. What the values mean is inferred + * from their letters. + * + * SMC.OutboxNotEmpty wakes the SoC while anything waits in the outbox. An event that arrives + * during the sleep's entry (the lid, a key) is left there, so the sleep returns at once. The + * debug link's cut fills it too, and the SMC counts what it reports after the notice as wake + * reasons, so the cut comes before the notice. + */ + +/* The gaps macOS leaves before the AP idle notice and after it. */ +#define SMC_SLEEP_ARM_MS 6 +#define SMC_SLEEP_IDLE_MS 30 + +/* + * After the debug link's cut. Its port and charger events keep coming for over a second, + * and any that lands after the AP idle notice wakes the SoC at once. + */ +#define SMC_OUTBOX_QUIET_MS 500 +#define SMC_OUTBOX_SETTLE_MS 5000 + +static int apple_smc_ap_power(struct apple_smc *smc, bool on) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_ap_power, smc, on); + + return smc->polled ? apple_rtkit_ap_power_noirq(smc->rtk, on) : + apple_rtkit_ap_power(smc->rtk, on); +} + +static void apple_smc_set_ap_notified(struct apple_smc *smc, bool notified) +{ + guard(mutex)(&smc->mutex); + + smc->ap_notified = notified; +} + +static void apple_smc_set_polled(struct apple_smc *smc, bool polled) +{ + guard(mutex)(&smc->mutex); + + smc->polled = polled; +} + +static void apple_smc_write_mbse(struct apple_smc *smc, smc_key value) +{ + int ret; + + ret = apple_smc_write_u32(smc, SMC_KEY(MBSE), value); + if (ret) + dev_err(smc->dev, "Failed to issue MBSE = %p4ch: %d\n", &value, ret); +} + +static void apple_smc_sleep_warn(struct apple_smc *smc) +{ + apple_smc_write_mbse(smc, SMC_KEY(slpw)); + smc->sleep_warned = true; +} + +static int apple_smc_notice_idle(struct apple_smc *smc) +{ + int ret; + + /* + * Commands fail from here on. The mutex isn't held across the notice: the + * answer comes through the same worker as the notifications, and those can + * be waiting for this mutex. + */ + apple_smc_set_ap_notified(smc, true); + + ret = apple_smc_ap_power(smc, false); + if (ret) { + /* Taken and not answered, perhaps, so undo it. */ + apple_smc_ap_power(smc, true); + apple_smc_set_ap_notified(smc, false); + dev_err(smc->dev, "SMC did not take the sleep notice: %d\n", ret); + } else { + dev_dbg(smc->dev, "AP idle notice answered\n"); + } + + return ret; +} + +static int apple_smc_sleep_commit(struct apple_smc *smc) +{ + int ret; + + apple_smc_write_mbse(smc, SMC_KEY(slpa)); + /* Device interrupts are off here. */ + mdelay(SMC_SLEEP_ARM_MS); + + ret = apple_smc_notice_idle(smc); + if (!ret) + mdelay(SMC_SLEEP_IDLE_MS); + + return ret; +} + +static void apple_smc_notice_on(struct apple_smc *smc) +{ + u8 record[8] = {}; + u32 event = 0; + int ret; + + ret = apple_smc_ap_power(smc, true); + if (ret) + dev_err(smc->dev, "SMC did not take the wake notice: %d\n", ret); + apple_smc_set_ap_notified(smc, false); + + ret = apple_smc_read_u32(smc, SMC_KEY(MBSE), &event); + if (ret) + dev_warn(smc->dev, "Failed to read MBSE after a wake: %d\n", ret); + ret = apple_smc_read(smc, SMC_KEY(MBSW), record, sizeof(record)); + if (ret < 0) + dev_warn(smc->dev, "Failed to read MBSW after a wake: %d\n", ret); + dev_dbg(smc->dev, "wake record: MBSE %#x, MBSW %8ph\n", event, record); +} + +static void apple_smc_sleep_wake(struct apple_smc *smc) +{ + apple_smc_notice_on(smc); + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; +} + +static int apple_smc_prepare(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (pm_suspend_via_firmware()) + apple_smc_sleep_warn(smc); + + return 0; +} + +/* Warned, and a refusal came before the notice: tell it the sleep is off. */ +static void apple_smc_complete(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (smc->sleep_warned) { + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + } +} + +/* The events go to their handlers as usual, through the RTKit worker. */ +static void apple_smc_outbox_settle(struct apple_smc *smc) +{ + unsigned int ms, quiet = 0, count = 0, last = 0; + int ret; + + for (ms = 0; ms < SMC_OUTBOX_SETTLE_MS && quiet < SMC_OUTBOX_QUIET_MS; ms++) { + ret = apple_rtkit_poll(smc->rtk); + if (ret > 0) { + count += ret; + last = ms; + quiet = 0; + } else { + quiet++; + } + usleep_range(1000, 1200); + } + dev_dbg(smc->dev, "debug link cut: %u messages, the last at %u ms, %s after %u ms\n", + count, last, quiet >= SMC_OUTBOX_QUIET_MS ? "quiet" : "still busy", ms); +} + +static int apple_smc_suspend_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + int ret; + + if (!smc->sleep_warned) + return 0; + + apple_smc_set_polled(smc, true); + if (apple_sleep_debug_link_cut()) + apple_smc_outbox_settle(smc); + ret = apple_smc_sleep_commit(smc); + apple_smc_set_polled(smc, false); + + return ret; +} + +static int apple_smc_resume_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (smc->ap_notified) { + apple_smc_set_polled(smc, true); + apple_smc_sleep_wake(smc); + apple_smc_set_polled(smc, false); + } + + return 0; +} + +static const struct dev_pm_ops apple_smc_pm_ops = { + .prepare = pm_sleep_ptr(apple_smc_prepare), + .complete = pm_sleep_ptr(apple_smc_complete), + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_smc_suspend_noirq, apple_smc_resume_noirq) +}; + static struct platform_driver apple_smc_driver = { .driver = { .name = "macsmc", .of_match_table = apple_smc_of_match, + .pm = pm_sleep_ptr(&apple_smc_pm_ops), }, .probe = apple_smc_probe, }; diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index fbc047b00afd8..0ce7d85e923c5 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -112,6 +112,13 @@ struct apple_smc { struct apple_rtkit *rtk; + /* Between the AP power notices of a firmware-driven suspend. */ + bool ap_notified; + /* Told of a coming sleep, and not yet of its end. */ + bool sleep_warned; + /* In a noirq phase: replies are polled for. */ + bool polled; + struct completion init_done; enum apple_smc_boot_stage boot_stage; From cad6fce0f349a945b2b6466a545abd18941794fc Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:17 -0500 Subject: [PATCH 37/73] mfd: macsmc: name what the SMC woke the AP for MBSW is a bitmap of why the SMC had the AP woken, little-endian. macOS names each set bit from smc-pmu's event_name properties and passes it on as the wake reason. Do the same with T8140's numbering, from its ADT: the names go on the wake record's line, and a set record is a wakeup event on the SMC's device, which is now wakeup capable. Measured on a J700: the power button wakes with pwrbtn, the lid with lid, a USB keyboard with USB2_wake and a Wi-Fi magic packet with wifibt. Signed-off-by: Diljit Singh --- drivers/mfd/macsmc-test.c | 13 +++++++++++++ drivers/mfd/macsmc.c | 37 ++++++++++++++++++++++++++++++++++++- include/linux/mfd/macsmc.h | 2 ++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index 2e0824a7c5825..5e38da0dabcfa 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -279,6 +279,18 @@ static void macsmc_notice_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(MBSW)); } +static void macsmc_wake_reason_test(struct kunit *test) +{ + char names[128]; + + KUNIT_EXPECT_EQ(test, apple_smc_wake_reason_names(0, names, sizeof(names)), 0); + KUNIT_EXPECT_STREQ(test, names, ""); + + /* A bit with no name is left out. */ + apple_smc_wake_reason_names(BIT_ULL(23) | BIT_ULL(42) | BIT_ULL(1), names, sizeof(names)); + KUNIT_EXPECT_STREQ(test, names, "wifibt USB-C_plug"); +} + static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_key_info_lock_test), KUNIT_CASE(macsmc_key_index_test), @@ -292,6 +304,7 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_sleep_order_test), KUNIT_CASE(macsmc_sleep_refused_test), KUNIT_CASE(macsmc_notice_test), + KUNIT_CASE(macsmc_wake_reason_test), {} }; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 0eb90e50b99d3..1b9ffc9bf2d3a 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -577,6 +577,10 @@ static int apple_smc_probe(struct platform_device *pdev) ARRAY_SIZE(apple_smc_telemetry_devs), NULL, 0, NULL); + ret = devm_device_init_wakeup(dev); + if (ret) + return ret; + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -718,9 +722,34 @@ static int apple_smc_sleep_commit(struct apple_smc *smc) return ret; } +/* MBSW's bits, as smc-pmu's event_name properties number them on T8140. Other SoCs differ. */ +static const struct { + u8 bit; + const char *name; +} apple_smc_wake_reasons[] = { + { 0, "pwrbtn" }, { 9, "ipd_touch" }, { 10, "lid" }, { 16, "ipd_kbd" }, + { 18, "charger" }, { 23, "wifibt" }, { 40, "acattach" }, { 41, "acdetach" }, + { 42, "USB-C_plug" }, { 43, "USB2_plug" }, { 44, "USB2_wake" }, { 45, "aupo" }, + { 48, "rtc" }, +}; + +static int apple_smc_wake_reason_names(u64 record, char *buf, size_t size) +{ + int i, len = 0; + + buf[0] = 0; + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (record & BIT_ULL(apple_smc_wake_reasons[i].bit)) + len += scnprintf(buf + len, size - len, "%s%s", len ? " " : "", + apple_smc_wake_reasons[i].name); + + return len; +} + static void apple_smc_notice_on(struct apple_smc *smc) { u8 record[8] = {}; + char names[128]; u32 event = 0; int ret; @@ -735,7 +764,13 @@ static void apple_smc_notice_on(struct apple_smc *smc) ret = apple_smc_read(smc, SMC_KEY(MBSW), record, sizeof(record)); if (ret < 0) dev_warn(smc->dev, "Failed to read MBSW after a wake: %d\n", ret); - dev_dbg(smc->dev, "wake record: MBSE %#x, MBSW %8ph\n", event, record); + + smc->wake_record = get_unaligned_le64(record); + apple_smc_wake_reason_names(smc->wake_record, names, sizeof(names)); + dev_info(smc->dev, "wake record: MBSE %#x, MBSW %8ph%s%s\n", event, record, + names[0] ? ": " : "", names); + if (smc->wake_record) + pm_wakeup_event(smc->dev, 0); } static void apple_smc_sleep_wake(struct apple_smc *smc) diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index 0ce7d85e923c5..bf8781ba4c52e 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -118,6 +118,8 @@ struct apple_smc { bool sleep_warned; /* In a noirq phase: replies are polled for. */ bool polled; + /* MBSW at the last wake: what the SMC had woken the AP for. */ + u64 wake_record; struct completion init_done; enum apple_smc_boot_stage boot_stage; From 7b03b5a45f2e75d1faa14ce33adc794789b96510 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:17 -0500 Subject: [PATCH 38/73] Input: macsmc-input: wake on the lid and button from a suspend's start A suspend's first steps, the PM notifiers and the freezer, run before .prepare, and until the SMC's AP idle notice the lid and the power button arrive as plain events. A lid opened or the button pressed in that window didn't count as a wake, and the machine went to sleep anyway: on a J700, closing and opening the lid back to back left it asleep. Count them as wakes from PM_SUSPEND_PREPARE to PM_POST_SUSPEND, so they end the suspend, as on macOS. Signed-off-by: Diljit Singh --- drivers/input/misc/macsmc-input.c | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 585ae57aca5e6..19ea8f552c6bf 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -21,6 +21,7 @@ * @smc: Pointer to apple_smc struct of the mfd parent * @input: Allocated input_dev; devres managed * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) + * @pm_nb: Notifier block for the start and end of a system suspend * @wakeup_mode: Set to true when system is suspended and power button events should wake it */ struct macsmc_input { @@ -28,6 +29,7 @@ struct macsmc_input { struct apple_smc *smc; struct input_dev *input; struct notifier_block nb; + struct notifier_block pm_nb; bool wakeup_mode; }; @@ -107,6 +109,30 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +/* + * A suspend's first steps run before .prepare, and the SMC reports the lid and the button as + * plain events until its AP idle notice. Treat them as wakes from the suspend's start, so a lid + * opened or a button pressed while the machine goes down ends the suspend, as on macOS. + */ +static int macsmc_input_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, pm_nb); + + if (action == PM_SUSPEND_PREPARE) + WRITE_ONCE(smcin->wakeup_mode, true); + else if (action == PM_POST_SUSPEND) + WRITE_ONCE(smcin->wakeup_mode, false); + + return NOTIFY_DONE; +} + +static void macsmc_input_unregister_pm_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + unregister_pm_notifier(&smcin->pm_nb); +} + static void macsmc_input_unregister_notifier(void *data) { struct macsmc_input *smcin = data; @@ -199,7 +225,12 @@ static int macsmc_input_probe(struct platform_device *pdev) if (error) return error; - return 0; + smcin->pm_nb.notifier_call = macsmc_input_pm_notify; + error = register_pm_notifier(&smcin->pm_nb); + if (error) + return error; + + return devm_add_action_or_reset(&pdev->dev, macsmc_input_unregister_pm_notifier, smcin); } static int macsmc_input_pm_prepare(struct device *dev) From a1ec5ed507f61f0344ddcb078744532312f302a3 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:09:37 -0500 Subject: [PATCH 39/73] Input: macsmc-input: report the lid once a sleep transition ends Lid events that arrive during a sleep transition went to the input core at once. A lid that closes during a sleep and opens again before user space runs then shows up as a closed-then-open pair, which logind or a desktop may act on. From PM_SUSPEND_PREPARE on, raise the wakeup event as before, so the lid opening still ends a suspend, but hold the report back. ->complete() now reads MSLD after every sleep, s2idle included, and reports where the lid ended up, and the PM notifier reports it again at PM_POST_SUSPEND, for a suspend that ended before ->prepare() and so never ran ->complete(). In a dark wake ->complete() reports nothing and leaves the lid and the button armed: the system sleeps again, and a lid opened or a button pressed before the next ->prepare() is a hard wakeup that aborts it, not a KEY_POWER for user space. That ->prepare() reads MSLD once and refuses with -EBUSY if the lid is open, in case its event came and went. PM_POST_SUSPEND clears that state, so a later suspend never inherits it. User space sees the lid as it is when it next runs. A wake that stays awake gives user space the same lid state as before. Signed-off-by: Diljit Singh --- drivers/input/misc/macsmc-input.c | 39 +++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 19ea8f552c6bf..fb3fe3fdb6920 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -23,6 +23,7 @@ * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) * @pm_nb: Notifier block for the start and end of a system suspend * @wakeup_mode: Set to true when system is suspended and power button events should wake it + * @dark: Going back to sleep from a dark wake: the lid is read again before it */ struct macsmc_input { struct device *dev; @@ -31,6 +32,7 @@ struct macsmc_input { struct notifier_block nb; struct notifier_block pm_nb; bool wakeup_mode; + bool dark; }; #define SMC_EV_BTN 0x7201 @@ -87,6 +89,10 @@ static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long eve u8 lid_state = !!((event >> 8) & 0xff); pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + /* Through a sleep, .complete reports where the lid ended up. */ + if (smcin->wakeup_mode) + return; + input_report_switch(smcin->input, SW_LID, lid_state); input_sync(smcin->input); } @@ -118,10 +124,20 @@ static int macsmc_input_pm_notify(struct notifier_block *nb, unsigned long actio { struct macsmc_input *smcin = container_of(nb, struct macsmc_input, pm_nb); - if (action == PM_SUSPEND_PREPARE) + if (action == PM_SUSPEND_PREPARE) { WRITE_ONCE(smcin->wakeup_mode, true); - else if (action == PM_POST_SUSPEND) + } else if (action == PM_POST_SUSPEND) { + u8 val; + WRITE_ONCE(smcin->wakeup_mode, false); + smcin->dark = false; + /* A suspend that ended before .prepare held its lid events back too. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { + input_report_switch(smcin->input, SW_LID, val); + input_sync(smcin->input); + } + } return NOTIFY_DONE; } @@ -236,8 +252,18 @@ static int macsmc_input_probe(struct platform_device *pdev) static int macsmc_input_pm_prepare(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); + u8 val; smcin->wakeup_mode = true; + /* A lid opened as the dark wake ended may have sent its event already. */ + if (smcin->dark) { + smcin->dark = false; + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0 && !val) { + dev_info(dev, "the lid opened during a dark wake: not sleeping again\n"); + return -EBUSY; + } + } return 0; } @@ -245,10 +271,15 @@ static void macsmc_input_pm_complete(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); + /* Still armed: a lid opened or a button pressed before the next .prepare wakes. */ + if (pm_dark_wake()) { + smcin->dark = true; + return; + } smcin->wakeup_mode = false; - /* The lid may have moved while the AP was off and no event could arrive. */ - if (pm_resume_via_firmware() && apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD))) { + /* The lid may have moved while the AP was off, or its events were held back. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD))) { u8 val; if (apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { From e7d20866f8d312f218fc5e0a7f7f45f6331d197d Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:18 -0500 Subject: [PATCH 40/73] power: supply: macsmc: report the supplies again after a sleep The battery's charge can change while the system sleeps, and nothing told userspace after a wake. An SMC event during the sleep can queue a uevent too, which is dropped if it runs before the SMC takes commands again. Send one from .complete, when the SMC is back and before the thaw. Signed-off-by: Diljit Singh --- drivers/power/supply/macsmc-power.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 1bc23703654a4..c71c48d146752 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -1212,6 +1212,24 @@ static void macsmc_power_remove(struct platform_device *pdev) blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); } +/* + * The charge moved while asleep, and an event's uevent that ran before the SMC + * was back found nothing to report. + */ +static void macsmc_power_complete(struct device *dev) +{ + struct macsmc_power *power = dev_get_drvdata(dev); + + if (power->batt) + power_supply_changed(power->batt); + if (power->ac) + power_supply_changed(power->ac); +} + +static const struct dev_pm_ops macsmc_power_pm_ops = { + .complete = pm_sleep_ptr(macsmc_power_complete), +}; + static const struct platform_device_id macsmc_power_id[] = { { .name = "macsmc-power" }, { /* sentinel */ } @@ -1221,6 +1239,7 @@ MODULE_DEVICE_TABLE(platform, macsmc_power_id); static struct platform_driver macsmc_power_driver = { .driver = { .name = "macsmc-power", + .pm = pm_sleep_ptr(&macsmc_power_pm_ops), }, .id_table = macsmc_power_id, .probe = macsmc_power_probe, From 4172e9773ead94406bf34766a30f15cdbc13b1d7 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:19 -0500 Subject: [PATCH 41/73] PCI: apple: keep the T8140 radio bound and powered across a deep sleep A deep sleep powers this host off, and until now only its boot loader could bring it up, so the bootstrap engine and the Wi-Fi driver refused any sleep but suspend-to-idle. Removing both radios for the sleep and starting over at the wake didn't hold up: the rail cycle at the wake undid the root port's setup, the one engine read failed, and Wi-Fi and Bluetooth were gone after every wake. macOS keeps the radio's power and firmware: the drivers stop, the link goes through L2 and PERST#, and at the wake the port is enabled before its functions power up. Do that: - Nothing is removed, so the functions' own suspend runs, and no rail is touched. - resume_noirq repeats the bring-up with the boot loader's tunables, before the PCI core restores the functions: PERST# driven, the shared block and the port started in macOS's order from power-off (skipped if ANS's restart already has the shared block ready), the port's RID to SID map and its DART restored, the root port restored, PERST# released, the link polled up, and one engine read. - The bootstrap engine has to read as it did before its first command, every word the driver already checks then. That's the evidence its DMA pointers are gone, which the retained-ownership rule requires. If any word differs the radios stay down. Only then does the retained state reopen for one more validated read, on the same memory, which the retrained link needs before native ECAM. That read isn't a firmware load. The engine's interrupt resumes early, and its handler stays out of the block until the port answers. - The radio port's DART runs on the port's clock, so the host restores it once the port is back, and the sleep controller is told it may stop the port again at the next commit. Radios that fail to come back don't cost the machine its deep sleep: an engine left failed no longer refuses one, and a port that's still down isn't touched by the next sleep. With the link down the functions' config space is closed, but the PCI core's MSI-X restore writes the table in the function's BAR with no check of its own, so msix_enabled is set aside until the link and the engine read succeed. If a function finds that its side didn't survive the sleep, or the link or the engine read failed, the host restarts both radios from cold, as macOS does with a dead chip: at PM_POST_SUSPEND, once firmware files can be read, the functions are removed, Bluetooth first, which resets the chip; with the link down the rail is cycled and the bring-up runs again; then a rescan probes them as at boot. Wi-Fi's stop runs before its removal, which would otherwise wait on a failed reset. A chip that doesn't answer, or an engine read already spent on this wake, waits for the next sleep's wake instead. A host that a failed wake left down stays down: the next suspend doesn't save the functions over their last good state, as its config space reads all ones, and a refused sleep leaves it down with its port interrupts off. Both early failures in the wake drop the saved config too, as a failed link does. Wi-Fi's stop is registered with its module, which the recovery holds while the stop runs, so an unload can't free it under the call. A stop whose module is already going isn't called: the recovery leaves Wi-Fi bound and the radios for the next sleep, and the unload's own remove stops the radio. A host with no pins or tunables from the loader can't do this, and its guard keeps refusing a deep sleep. Only a sleep where deep has been selected runs any of it, so s2idle doesn't change. Signed-off-by: Diljit Singh --- Documentation/PCI/controller/apple-t8140.rst | 13 + .../pcie-apple-t8140-bootstrap-state.h | 10 +- .../pcie-apple-t8140-bootstrap-test.c | 20 +- .../controller/pcie-apple-t8140-bootstrap.c | 156 +++- .../controller/pcie-apple-t8140-bootstrap.h | 24 + drivers/pci/controller/pcie-apple.c | 774 ++++++++++++++++++ include/linux/soc/apple/pci-apple-piodma.h | 14 + 7 files changed, 1005 insertions(+), 6 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 53948d60375dc..4908e46f58712 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -64,6 +64,19 @@ recovery remain refused. Supporting these operations requires a proven engine teardown/reset and endpoint recovery contract first. Renaming the provider does not qualify those operations. +Deep sleep keeps both functions bound and the radio powered, with its firmware +running. Their drivers stop for it, the link goes through L2 and PERST#, and the SoC +powers the engine off. On the way back the engine's whole register file must read as it +did before the first command, the state the bootstrap checks for; if anything differs, +the radios stay down. Only then does the retained state reopen for one more validated +read, on the same memory, which the retrained link needs before native ECAM. The +functions' configuration is then restored and their drivers resume, with no firmware +load. If a driver finds that its side didn't survive, the host removes both functions, +which resets the chip, and scans again, so that they probe and load their firmware as at +boot. If the link didn't come back, the radio's rail is cycled first. An abandoned sleep +never ran that power-off, so it replays nothing. This needs tunables +from a loader that publishes them, and is refused otherwise. + Suspend-to-idle is allowed only after radio admission, with no IRQ fault, idle command/FIFO checks and intact arena contents. Resume restores the configuration banks after genpd power restoration without submitting another diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h index 21627bfa2ad4a..46dddb9f42610 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h @@ -15,7 +15,10 @@ #define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) #define PIODMA_IRQ_LIMIT 16 -/* Only RELEASE before publication can end guard ownership. No reset event. */ +/* + * Only RELEASE before publication can end guard ownership, or POWER_LOST: the + * engine's whole register file reading as reset after the SoC powered it off. + */ enum apple_piodma_state { PIODMA_IDLE, PIODMA_GUARDED, @@ -37,6 +40,7 @@ enum apple_piodma_event { PIODMA_COMPLETE, PIODMA_ADMIT, PIODMA_FAIL, + PIODMA_POWER_LOST, PIODMA_EVENT_COUNT, }; @@ -105,6 +109,10 @@ static inline int apple_piodma_next(enum apple_piodma_state state, if (state == PIODMA_RETAINED) return PIODMA_FAILED_RETAINED; break; + case PIODMA_POWER_LOST: + if (state == PIODMA_ADMITTED) + return PIODMA_GUARDED; + break; default: break; } diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c index 941a1786c0483..9cccb3ca4bcd0 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c @@ -61,12 +61,12 @@ static void apple_piodma_transition_guards_test(struct kunit *test) { int state, event, next; - /* Exhaust all transitions: no retained state can release or retry. */ + /* Exhaust all transitions: only a power loss can end a retained state. */ for (state = PIODMA_IDLE; state < PIODMA_STATE_COUNT; state++) { for (event = PIODMA_GUARD; event < PIODMA_EVENT_COUNT; event++) { next = apple_piodma_next(state, event); if (apple_piodma_retained(state)) { - if (next >= 0) + if (next >= 0 && event != PIODMA_POWER_LOST) KUNIT_EXPECT_TRUE(test, apple_piodma_retained(next)); if (event == PIODMA_RELEASE || event == PIODMA_PRIME || event == PIODMA_GUARD) @@ -128,8 +128,24 @@ static void apple_piodma_irq_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 1, 2)); } +/* Losing power reopens the guard from an admitted radio, and from nothing else. */ +static void apple_piodma_power_lost_test(struct kunit *test) +{ + static const enum apple_piodma_state refused[] = { + PIODMA_IDLE, PIODMA_GUARDED, PIODMA_ATTEMPTED, PIODMA_FAILED, + PIODMA_REJECTED, PIODMA_RETAINED, PIODMA_READY, PIODMA_FAILED_RETAINED, + }; + int i; + + KUNIT_EXPECT_EQ(test, apple_piodma_next(PIODMA_ADMITTED, PIODMA_POWER_LOST), + PIODMA_GUARDED); + for (i = 0; i < ARRAY_SIZE(refused); i++) + KUNIT_EXPECT_LT(test, apple_piodma_next(refused[i], PIODMA_POWER_LOST), 0); +} + static struct kunit_case apple_piodma_cases[] = { KUNIT_CASE(apple_piodma_lifecycle_test), + KUNIT_CASE(apple_piodma_power_lost_test), KUNIT_CASE(apple_piodma_failure_test), KUNIT_CASE(apple_piodma_transition_guards_test), KUNIT_CASE(apple_piodma_arena_test), diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c index a95dd5112f201..9261d23abee16 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c @@ -92,6 +92,16 @@ struct apple_t8140_bootstrap { struct kexec_blocker kexec_blocker; struct delayed_work admission_retry; struct device *retry_host; + /* The host stops the radios for a sleep that takes this engine's power. */ + bool sleep_armed; + bool power_lost; + /* After a real sleep: read the engine once the host's port is up again. */ + bool power_check; + bool irq_held; + int (*radio_stop)(struct pci_dev *pdev); + struct module *radio_stop_owner; + void (*radio_recover)(void *data); + void *radio_recover_data; }; /* A successful supplier probe owns its unpublished arena until full reset. */ @@ -193,6 +203,8 @@ static int apple_piodma_pm_prepare(struct device *dev) return 0; if (apple_piodma_sleep_allowed(apple_piodma_state(bootstrap), pm_suspend_no_platform())) return 0; + if (READ_ONCE(bootstrap->sleep_armed)) + return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this * supplier probed. Its later image/restore/suspend ioctls go straight @@ -372,6 +384,10 @@ static irqreturn_t apple_t8140_bootstrap_irq(int irq, void *data) u32 fabric, status, tag, active; int count; + /* After a real sleep its block can't answer until the host's port is back. */ + if (READ_ONCE(bootstrap->irq_held)) + return IRQ_NONE; + /* Acknowledge order follows the attributed prior controller contract. */ fabric = readl(bootstrap->fabric); writel(fabric, bootstrap->fabric); @@ -744,6 +760,60 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + WRITE_ONCE(bootstrap->sleep_armed, armed); + /* A check still pending means the block never came back, and stays unread. */ + if (armed) + bootstrap->power_lost = false; +} + +/* The next bootstrap starts as the first did, on the same retained memory. */ +int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + void *arena = bootstrap->arena; + int i; + + /* + * Everything reading as it did before the first command is what shows the sleep took + * the engine's pointers with it. Read only now: the block answers once the host's + * port is set up, after resume_noirq. + */ + if (bootstrap->power_check) { + bootstrap->power_check = false; + bootstrap->power_lost = !apple_t8140_bootstrap_idle(bootstrap); + if (!bootstrap->power_lost) + dev_err(supplier, "engine kept state across sleep; radios stay down\n"); + WRITE_ONCE(bootstrap->irq_held, false); + } + if (!bootstrap->power_lost || READ_ONCE(bootstrap->irq_fault)) + return -EIO; + bootstrap->power_lost = false; + + memset(arena, 0, PIODMA_ARENA_SIZE); + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { + void *slot = arena + i * PIODMA_SLOT_SIZE; + + memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); + *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); + memset(bootstrap->slots[i].packet, 0, sizeof(bootstrap->slots[i].packet)); + memset(bootstrap->slots[i].completion, 0, sizeof(bootstrap->slots[i].completion)); + bootstrap->slots[i].result = 0; + WRITE_ONCE(bootstrap->slots[i].matched_status, 0); + bootstrap->slots[i].submitted = false; + bootstrap->slots[i].observed = false; + reinit_completion(&bootstrap->slots[i].event); + } + dma_wmb(); + WRITE_ONCE(bootstrap->active_tag, 0); + atomic_set(&bootstrap->irq_count, 0); + + return apple_piodma_transition(bootstrap, PIODMA_POWER_LOST); +} + bool apple_piodma_bootstrap_retained(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -753,6 +823,43 @@ bool apple_piodma_bootstrap_retained(struct device *supplier) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); +/* The stop and the module it's in, set and read together. */ +static DEFINE_SPINLOCK(radio_stop_lock); + +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + int (*stop)(struct pci_dev *pdev); + struct module *owner; + bool held; + int ret; + + /* Its module stays while it runs. One that is unloading: its remove stops the radio. */ + spin_lock(&radio_stop_lock); + stop = bootstrap->radio_stop; + owner = bootstrap->radio_stop_owner; + held = !stop || try_module_get(owner); + spin_unlock(&radio_stop_lock); + if (!held) + return -EBUSY; + if (!stop) + return 0; + + ret = stop(pdev); + module_put(owner); + + return ret; +} + +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + bootstrap->radio_recover_data = data; + WRITE_ONCE(bootstrap->radio_recover, recover); +} + void apple_piodma_bootstrap_admitted(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -790,6 +897,31 @@ int apple_piodma_radio_check(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(apple_piodma_radio_check); +void apple_piodma_radio_recover(void) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + void (*recover)(void *data) = bootstrap ? READ_ONCE(bootstrap->radio_recover) : NULL; + + if (recover) + recover(bootstrap->radio_recover_data); +} +EXPORT_SYMBOL_GPL(apple_piodma_radio_recover); + +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + + if (!bootstrap) + return; + spin_lock(&radio_stop_lock); + bootstrap->radio_stop = stop; + bootstrap->radio_stop_owner = owner; + spin_unlock(&radio_stop_lock); +} +EXPORT_SYMBOL_GPL(__apple_piodma_radio_set_stop); + static int apple_t8140_bootstrap_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -822,7 +954,8 @@ static int apple_t8140_bootstrap_probe(struct platform_device *pdev) ret = apple_t8140_bootstrap_resources(pdev, bootstrap); if (ret) goto err_arena; - ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, 0, + /* The host's resume_noirq waits for one completion, with device interrupts still off. */ + ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, IRQF_EARLY_RESUME, dev_name(dev), bootstrap); if (ret) goto err_arena; @@ -873,12 +1006,20 @@ static int apple_t8140_bootstrap_sleep_check(struct apple_t8140_bootstrap *boots static int apple_t8140_bootstrap_suspend_noirq(struct device *dev) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); + int ret; if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !READ_ONCE(bootstrap->sleep_armed)) return -EOPNOTSUPP; - return apple_t8140_bootstrap_sleep_check(bootstrap); + /* Radios that failed to come back don't cost the machine a deep sleep. */ + if (READ_ONCE(bootstrap->sleep_armed) && + (!apple_piodma_ready(apple_piodma_state(bootstrap)) || bootstrap->power_check)) + return 0; + ret = apple_t8140_bootstrap_sleep_check(bootstrap); + if (!ret && READ_ONCE(bootstrap->sleep_armed)) + WRITE_ONCE(bootstrap->irq_held, true); + return ret; } static int apple_t8140_bootstrap_resume_noirq(struct device *dev) @@ -890,6 +1031,15 @@ static int apple_t8140_bootstrap_resume_noirq(struct device *dev) return 0; if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; + /* + * Its block reads nothing until the host sets its port up again, so neither read it + * nor take its interrupt until then: apple_piodma_bootstrap_power_lost() checks it. + */ + if (READ_ONCE(bootstrap->sleep_armed) && pm_resume_via_firmware()) { + bootstrap->power_check = true; + return 0; + } + WRITE_ONCE(bootstrap->irq_held, false); /* Genpd may gate this island; restore configuration, never a command. */ ret = apple_t8140_bootstrap_initialize(bootstrap); if (ret) diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h index b7c4eeedb92c9..713ad546cbedc 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h @@ -17,6 +17,11 @@ void apple_piodma_bootstrap_guard_end(struct device *supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed); +int apple_piodma_bootstrap_power_lost(struct device *supplier); +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev); +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data); #else static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { @@ -46,5 +51,24 @@ static inline bool apple_piodma_bootstrap_retained(struct device *supplier) { return false; } + +static inline void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ +} + +static inline int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + return -ENODEV; +} + +static inline void apple_piodma_bootstrap_set_recover(struct device *supplier, + void (*recover)(void *data), void *data) +{ +} #endif #endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3173186c85209..edafbf0075c49 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -408,6 +409,19 @@ struct apple_pcie { bool neo_config_ready; u8 neo_secondary_bus; struct device *piodma_supplier; + /* What it takes to come back from a deep sleep, if the loader gave it. */ + struct apple_pcie_neo_cold *neo_cold; + struct notifier_block neo_pm_nb; + bool neo_sleep_armed; + struct work_struct neo_recover_work; + bool neo_recover; + /* The functions' state from the last sleep that found them answering */ + struct pci_saved_state *neo_fn_state[2]; + /* Their MSI-X, set aside while they're out of reach */ + bool neo_fn_msix[2]; + bool neo_rc_down; + /* Down already when the sleep began: a failed wake left it unprogrammed */ + bool neo_rc_was_down; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -436,6 +450,8 @@ struct apple_pcie_port { unsigned int link_irqs[2]; u32 saved_intmask; int sid_map_sz; + struct gpio_desc *perst_gpio; + struct gpio_desc *pwren_gpio; int idx; bool started; bool needs_stop; @@ -1591,6 +1607,13 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, u32 stat; int ret; + /* Bringing the link up again, after a deep sleep. */ + if (port->perst_gpio) { + reset = port->perst_gpio; + pwren = port->pwren_gpio; + goto have_gpios; + } + /* * Assert PERST# and configure the pin as output. * The Aquantia AQC113 10GB nic used desktop macs is sensitive to @@ -1628,7 +1651,12 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, else return PTR_ERR(pwren); } + if (!num_aux_resets) { + port->perst_gpio = reset; + port->pwren_gpio = pwren; + } +have_gpios: rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ @@ -2519,6 +2547,10 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); + /* Nothing answers on a host that has lost power and isn't set up again. */ + if (READ_ONCE(pcie->neo_rc_down)) + return NULL; + if (bus->number != cfg->busr.start) { /* Pair with ready publication before consuming the bus number. */ if (!smp_load_acquire(&pcie->neo_config_ready) || @@ -3069,6 +3101,396 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) return ret; } +/* + * What the bootloader did to this host at boot, on the same registers, in its + * order, with its tunables. On this SoC a deep sleep powers all of it off. + */ +struct apple_pcie_neo_cold { + void __iomem *axi2af; + void __iomem *cio3pll; + void __iomem *clkgen; + void __iomem *phy_block; + struct apple_tunable *axi2af_tunable; + struct apple_tunable *cio3pll_tunable; + struct apple_tunable *clkgen_tunable; + struct apple_tunable *phy_tunable; + struct apple_tunable *port_tunable; + struct apple_tunable *rc_tunable; + struct apple_tunable *rc_gen3_tunable; + struct apple_tunable *rc_gen4_tunable; + /* The port's DART, clocked by the port: restored after the port is set up. */ + struct device *dart; +}; + +#define NEO_PHY_CTRL 0x0 +#define NEO_PHY_CTRL_CLK0REQ BIT(0) +#define NEO_PHY_CTRL_CLK1REQ BIT(1) +#define NEO_PHY_CTRL_CLK0ACK BIT(2) +#define NEO_PHY_CTRL_CLK1ACK BIT(3) +#define NEO_PHY_CTRL_RESET BIT(4) +#define NEO_PHY_RUN 0x4 +#define NEO_PHY_READY 0x8 +#define NEO_PHY_CTRL_UP BIT(27) +#define NEO_PHY_SETTLE_US 300 +#define NEO_AXI_EN0 0x104 +#define NEO_AXI_EN1 0x108 +#define NEO_CLK_CTRL 0x0 +#define NEO_CLK_BUSY BIT(31) +#define NEO_CLK_OFF BIT(2) +#define NEO_CLK_START 0x4 +#define NEO_CLK_START_GO BIT(31) +#define NEO_PHY_BLOCK_COMMON 0x4000 +#define NEO_PHY_BLOCK_PHY 0x8000 +#define NEO_PHY_POLL_US 50000 +#define NEO_RC_WRITE_ZERO 0x04 +#define NEO_RC_READY_CFG 0x54 +#define NEO_RC_READY_START 0x50 +#define NEO_RC_READY_STAT 0x58 +#define NEO_RC_POLL_US 250000 +#define NEO_DBI_CAP_LNKCAP 0x7c +#define NEO_DBI_CAP_LNKCAP2 0x9c +#define NEO_DBI_LNKCAP2_SLS GENMASK(6, 1) +#define NEO_DBI_CAP_LNKCTL2 0xa0 +#define NEO_DBI_PORT_LINK_CONTROL 0x710 +#define NEO_DBI_PORT_LINK_MODE GENMASK(21, 16) +#define NEO_DBI_WIDTH_SPEED_CONTROL 0x80c +#define NEO_DBI_LINK_WIDTH GENMASK(12, 8) +#define NEO_DBI_SPEED_CHANGE BIT(17) +#define NEO_DBI_RO_WR 0x8bc +#define NEO_DBI_RO_WR_EN BIT(0) + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001000ff }, + { 0x82c, 0x00000000 }, +}; + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs2[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static int apple_pcie_neo_phy_clocks(void __iomem *phy) +{ + u32 stat; + int ret; + + rmw_set(NEO_PHY_CTRL_CLK0REQ, phy + NEO_PHY_CTRL); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK0ACK, + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(NEO_PHY_CTRL_CLK1REQ, phy + NEO_PHY_CTRL); + return readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK1ACK, + 10, NEO_PHY_POLL_US); +} + +/* + * The part of the block that ANS shares, from power-off in macOS's order, measured on + * one sleep. The sleep driver does it at the wake when ANS is bound, and it can't be + * redone under a running PHY. + */ +static int apple_pcie_neo_common_init(struct apple_pcie *pcie) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *phy = cold->phy_block + NEO_PHY_BLOCK_PHY; + u32 stat, val; + int ret; + + if (readl_relaxed(pcie->base + NEO_RC_READY_STAT) & BIT(0)) + return 0; + + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN0); + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN1); + apple_tunable_apply(cold->axi2af, cold->axi2af_tunable); + apple_tunable_apply(cold->cio3pll, cold->cio3pll_tunable); + apple_tunable_apply(cold->clkgen, cold->clkgen_tunable); + rmw_set(NEO_CLK_START_GO, cold->clkgen + NEO_CLK_START); + rmw_clear(NEO_CLK_OFF, cold->clkgen + NEO_CLK_CTRL); + ret = readl_relaxed_poll_timeout(cold->clkgen + NEO_CLK_CTRL, stat, !(stat & NEO_CLK_BUSY), + 1, NEO_PHY_POLL_US); + if (ret) + return ret; + apple_tunable_apply(cold->phy_block, cold->phy_tunable); + + ret = apple_pcie_neo_phy_clocks(phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, phy + NEO_PHY_CTRL); + udelay(1); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, !(stat & NEO_PHY_CTRL_RESET), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(BIT(0), phy + NEO_PHY_RUN); + udelay(NEO_PHY_SETTLE_US); + rmw_set(BIT(4), phy + NEO_PHY_RUN); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_READY, stat, stat & BIT(0), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + + val = readl_relaxed(cold->phy_block + NEO_PHY_BLOCK_COMMON); + writel_relaxed((val & ~GENMASK(1, 0)) | 1, cold->phy_block + NEO_PHY_BLOCK_COMMON); + rmw_set(NEO_PHY_CTRL_UP, phy + NEO_PHY_CTRL); + + writel_relaxed(0, pcie->base + NEO_RC_WRITE_ZERO); + writel_relaxed(0x140, pcie->base + NEO_RC_READY_CFG); + writel_relaxed(1, pcie->base + NEO_RC_READY_START); + + return readl_relaxed_poll_timeout(pcie->base + NEO_RC_READY_STAT, stat, stat & BIT(0), + 100, NEO_RC_POLL_US); +} + +static int apple_pcie_neo_cold_init(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + int speed = of_pci_get_max_link_speed(port->np); + u32 stat, val; + int i, ret; + + ret = apple_pcie_neo_common_init(pcie); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs[i].value, + apple_pcie_neo_port_regs[i].offset); + for (i = 0; i < 16; i++) + apple_pcie_port_writel(port, 0, 0x3000 + 4 * i); + for (i = 0; i < 512; i++) + apple_pcie_port_writel(port, 0, pcie->hw->port_msimap + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs2); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs2[i].value, + apple_pcie_neo_port_regs2[i].offset); + apple_tunable_apply(port->base, cold->port_tunable); + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* PERST# off before the PHY's clocks, as macOS has it coming from power-off. */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, NEO_RC_POLL_US); + if (ret) + return ret; + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, + !(stat & PORT_LINKSTS_BUSY), 100, NEO_RC_POLL_US); + if (ret) + return ret; + + rmw_clear(NEO_PHY_CTRL_CLK0REQ | NEO_PHY_CTRL_CLK1REQ, port->phy + NEO_PHY_CTRL); + ret = apple_pcie_neo_phy_clocks(port->phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, port->phy + NEO_PHY_CTRL); + rmw_set(BIT(9), port->phy + NEO_PHY_CTRL); + rmw_set(BIT(10), port->phy + NEO_PHY_CTRL); + + rmw_set(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + apple_tunable_apply(dbi, cold->rc_tunable); + apple_tunable_apply(dbi, cold->rc_gen3_tunable); + apple_tunable_apply(dbi, cold->rc_gen4_tunable); + if (speed > 0) { + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP); + writel_relaxed((val & ~PCI_EXP_LNKCAP_SLS) | speed, dbi + NEO_DBI_CAP_LNKCAP); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP2); + writel_relaxed((val & ~NEO_DBI_LNKCAP2_SLS) | + FIELD_PREP(NEO_DBI_LNKCAP2_SLS, (1 << speed) - 1), + dbi + NEO_DBI_CAP_LNKCAP2); + writew_relaxed((readw_relaxed(dbi + NEO_DBI_CAP_LNKCTL2) & ~PCI_EXP_LNKCTL2_TLS) | + speed, dbi + NEO_DBI_CAP_LNKCTL2); + rmw_set(NEO_DBI_SPEED_CHANGE, dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + } + val = readl_relaxed(dbi + NEO_DBI_PORT_LINK_CONTROL) & ~NEO_DBI_PORT_LINK_MODE; + writel_relaxed(val | FIELD_PREP(NEO_DBI_PORT_LINK_MODE, 1), + dbi + NEO_DBI_PORT_LINK_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_WIDTH_SPEED_CONTROL) & ~NEO_DBI_LINK_WIDTH; + writel_relaxed(val | FIELD_PREP(NEO_DBI_LINK_WIDTH, 1), dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP) & ~PCI_EXP_LNKCAP_MLW; + writel_relaxed(val | FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1), dbi + NEO_DBI_CAP_LNKCAP); + rmw_clear(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + + return 0; +} + +static void apple_pcie_neo_recover_work(struct work_struct *work); + +/* A radio left unusable by the wake, or its link. POST_SUSPEND runs this once all is thawed. */ +static void apple_pcie_neo_recover(void *data) +{ + struct apple_pcie *pcie = data; + + WRITE_ONCE(pcie->neo_recover, true); + /* Pairs with the barrier at PM_POST_SUSPEND: one side sees the other's write. */ + smp_mb(); + if (!READ_ONCE(pcie->neo_sleep_armed)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); +} + +/* Runs before the freezer. The radios stay bound, and their drivers do their own suspend. */ +static int apple_pcie_neo_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct apple_pcie *pcie = container_of(nb, struct apple_pcie, neo_pm_nb); + + switch (action) { + case PM_SUSPEND_PREPARE: + if (pm_suspend_default_s2idle()) + break; + flush_work(&pcie->neo_recover_work); + pcie->neo_sleep_armed = true; + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, true); + break; + case PM_POST_SUSPEND: + if (!pcie->neo_sleep_armed) + break; + WRITE_ONCE(pcie->neo_sleep_armed, false); + /* Pairs with the barrier in the recovery request. */ + smp_mb(); + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, false); + if (READ_ONCE(pcie->neo_recover)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +static void __iomem *apple_pcie_neo_map(struct platform_device *pdev, const char *name, + struct resource **res) +{ + *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + if (!*res) + return IOMEM_ERR_PTR(-ENOENT); + + /* These overlap each other and the port PHYs, so they can't be claimed. */ + return devm_ioremap(&pdev->dev, (*res)->start, resource_size(*res)) ?: + IOMEM_ERR_PTR(-ENOMEM); +} + +/* A shared block and the tunable the loader publishes for it. */ +static int apple_pcie_neo_get(struct platform_device *pdev, const char *name, const char *prop, + void __iomem **base, struct apple_tunable **tunable) +{ + struct resource *res; + + *base = apple_pcie_neo_map(pdev, name, &res); + if (IS_ERR(*base)) + return PTR_ERR(*base); + *tunable = devm_apple_tunable_parse(&pdev->dev, pdev->dev.of_node, prop, res); + + return PTR_ERR_OR_ZERO(*tunable); +} + +/* The DART in the host's iommu-map, which reads nothing until the port is set up. */ +static struct device *apple_pcie_neo_dart(struct device *dev) +{ + struct device_node *np = of_parse_phandle(dev->of_node, "iommu-map", 1); + struct platform_device *pdev = np ? of_find_device_by_node(np) : NULL; + int ret; + + of_node_put(np); + if (!pdev) + return NULL; + /* The reference is kept: this host can't be unbound. */ + ret = apple_dart_set_host_resumes(&pdev->dev); + /* The DART resumes, and gates itself, before this host restores it. */ + if (!ret && !device_link_add(dev, &pdev->dev, DL_FLAG_STATELESS)) + ret = -EINVAL; + if (ret) { + put_device(&pdev->dev); + dev_warn(dev, "no deep sleep: its DART can't wait for the port: %d\n", ret); + return NULL; + } + return &pdev->dev; +} + +/* Without all of this the host can't come back, and a deep sleep stays refused. */ +static void apple_pcie_neo_sleep_probe(struct platform_device *pdev, struct apple_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pcie_neo_cold *cold; + struct apple_pcie_port *port; + struct resource *res, *cfg; + int ret; + + port = list_first_entry_or_null(&pcie->ports, struct apple_pcie_port, entry); + cfg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!port || !cfg || !port->perst_gpio || !port->phy) { + dev_warn(dev, "no deep sleep: nothing here took the link up, so no pins to restart it\n"); + return; + } + cold = devm_kzalloc(dev, sizeof(*cold), GFP_KERNEL); + if (!cold) + return; + + if (apple_pcie_neo_get(pdev, "axi2af", "apple,tunable-axi2af", &cold->axi2af, + &cold->axi2af_tunable) || + apple_pcie_neo_get(pdev, "cio3pll", "apple,tunable-cio3pll", &cold->cio3pll, + &cold->cio3pll_tunable) || + apple_pcie_neo_get(pdev, "clkgen", "apple,tunable-clkgen", &cold->clkgen, + &cold->clkgen_tunable)) + goto absent; + cold->phy_block = apple_pcie_neo_map(pdev, "phy-block", &res); + if (IS_ERR(cold->phy_block)) + goto absent; + cold->phy_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-phy", res); + if (IS_ERR(cold->phy_tunable)) + goto absent; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "port0"); + if (!res) + goto absent; + cold->port_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-port", res); + if (IS_ERR(cold->port_tunable)) + goto absent; + cold->rc_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc", cfg); + if (IS_ERR(cold->rc_tunable)) + goto absent; + cold->rc_gen3_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen3", + cfg); + if (IS_ERR(cold->rc_gen3_tunable)) + goto absent; + cold->rc_gen4_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen4", + cfg); + if (IS_ERR(cold->rc_gen4_tunable)) + goto absent; + cold->dart = apple_pcie_neo_dart(dev); + if (!cold->dart) + goto absent; + + pcie->neo_cold = cold; + INIT_WORK(&pcie->neo_recover_work, apple_pcie_neo_recover_work); + apple_piodma_bootstrap_set_recover(pcie->piodma_supplier, apple_pcie_neo_recover, pcie); + pcie->neo_pm_nb.notifier_call = apple_pcie_neo_pm_notify; + ret = register_pm_notifier(&pcie->neo_pm_nb); + if (ret) + pcie->neo_cold = NULL; + else + dev_dbg(dev, "deep sleep: radios stay powered, link stopped and retrained\n"); + return; + +absent: + dev_warn(dev, "no deep sleep: the loader didn't publish what this host needs\n"); +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -3253,6 +3675,8 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) dev_err(dev, "radio admission failed=%d; host/supplier retained, no retry\n", ret); + else + apple_pcie_neo_sleep_probe(pdev, pcie); return 0; } @@ -3910,10 +4334,40 @@ static void apple_pcie_stop_for_sleep(struct device *dev) apple_pcie_tunnel_hierarchy_lost(dev); } +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie); + static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); + if (pcie->neo_sleep_armed) { + struct apple_pcie_port *port; + + pcie->neo_rc_was_down = READ_ONCE(pcie->neo_rc_down); + /* A port that didn't come back from the last sleep keeps what it saved then. */ + list_for_each_entry(port, &pcie->ports, entry) { + int i; + + apple_pcie_port_disable_irq(port); + if (pcie->neo_rc_was_down) + continue; + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + } + /* The functions too: the core read them through closed config space. */ + if (!pcie->neo_rc_was_down) + apple_pcie_neo_save_functions(pcie); + WRITE_ONCE(pcie->neo_rc_down, true); + /* + * Genpd would gate the host here, and an abandoned sleep then finds + * it unprogrammed with nothing to bring it up. The commit lowers it. + */ + device_set_awake_path(dev); + return 0; + } + /* * The managed radio has no suspend boundary of its own; suspend-to-idle * leaves it in place like any other internal port (no power domain). @@ -3970,12 +4424,332 @@ static int apple_pcie_reset_for_resume(struct apple_pcie *pcie) return 0; } +/* + * The radio kept its power and firmware, in reset since the commit. Its rail isn't touched: + * cycling it, or the port's own reset, is what a cold start needs. PERST# released, the link + * trains. Device interrupts are off in this phase, so this polls for it. + */ +static int apple_pcie_neo_link_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + usleep_range(1000, 2000); + ret = gpiod_set_value_cansleep(port->perst_gpio, 0); + if (ret) + return ret; + /* 100 ms after PERST# as at boot, and after the link (PCIe r5.0, 6.6.1). */ + msleep(100); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, 250000); + if (ret) + return ret; + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, stat & PORT_LINKSTS_UP, + 100, link_up_timeout * 1000); + if (ret) + return ret; + apple_pcie_port_writel(port, BIT(PORT_INT_LINK_UP), PORT_INTSTAT); + msleep(100); + + return 0; +} + +/* + * The PCI core saves the functions again at every sleep, and with the radio down it reads them + * through closed config space. Keep the last state they saved while they answered. + */ +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) + return; + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + kfree(pcie->neo_fn_state[i]); + pcie->neo_fn_state[i] = pci_store_saved_state(fn); + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* With the link back: what the core restores next is that state. */ +static void apple_pcie_neo_load_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int i; + + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (pcie->neo_fn_state[i]) + pci_load_saved_state(fn, pcie->neo_fn_state[i]); + pci_dev_put(fn); + } +} + +/* + * Closed config space keeps the PCI core's restore out of the functions, except for one thing: + * it writes an MSI-X table in their BAR, with no check of its own. With the functions out of + * reach, set that aside so that it has nothing to write; their reach back puts it in place. + */ +static void apple_pcie_neo_msix_gate(struct apple_pcie *pcie, bool reachable) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_msix); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (reachable && pcie->neo_fn_msix[i]) { + fn->msix_enabled = 1; + pcie->neo_fn_msix[i] = false; + } else if (!reachable && fn->msix_enabled) { + fn->msix_enabled = 0; + pcie->neo_fn_msix[i] = true; + } + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* + * The link back, and the one engine read that native ECAM needs on it. The root port's + * bus number is restored first, as that read crosses it. Not a firmware load. + */ +static int apple_pcie_neo_radio_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root; + int ret; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return -ENODEV; + pci_restore_state(root); + ret = apple_pcie_neo_link_up(pcie, port); + if (!ret) + ret = apple_piodma_bootstrap_power_lost(pcie->piodma_supplier); + if (!ret) + ret = apple_piodma_bootstrap_prime(pcie->piodma_supplier, root); + if (!ret) { + apple_piodma_bootstrap_admitted(pcie->piodma_supplier); + apple_pcie_neo_load_functions(pcie, root); + /* The functions' configuration before the flag that says it's there. */ + smp_store_release(&pcie->neo_config_ready, true); + apple_pcie_neo_msix_gate(pcie, true); + } + pci_dev_put(root); + + return ret; +} + +/* + * The rail is the one thing that brings back a chip whose link won't train: macOS's last resort + * for a dead chip too. A cold chip then has its firmware loaded by the probe that follows. + */ +static int apple_pcie_neo_rail_cycle(struct apple_pcie *pcie, struct apple_pcie_port *port, + struct pci_dev *root) +{ + struct pci_dev *fn; + int i, ret; + + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = gpiod_set_value_cansleep(port->pwren_gpio, 0); + if (ret) + return ret; + msleep(pcie->hw->pwren_off_ms); + ret = gpiod_set_value_cansleep(port->pwren_gpio, 1); + if (ret) + return ret; + msleep(pcie->hw->pwren_on_ms); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) + return ret; + /* The PCI core's restore found them gone. Wi-Fi's stop needs its BAR back. */ + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + pci_set_power_state(fn, PCI_D0); + pci_restore_state(fn); + pci_dev_put(fn); + } + return 0; +} + +/* The way a hotplug would. Bluetooth first: Wi-Fi's reset needs it gone. */ +static int apple_pcie_neo_stop_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int ret = 0; + + /* Stopping Wi-Fi writes its BAR, which needs the chip to answer. */ + fn = root->subordinate ? pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)) : NULL; + if (fn && !pci_device_is_present(fn)) + ret = -ENODEV; + pci_dev_put(fn); + if (ret) + return ret; + + pci_lock_rescan_remove(); + if (root->subordinate) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 1)); + if (fn) { + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)); + if (fn) { + ret = apple_piodma_bootstrap_radio_stop(pcie->piodma_supplier, fn); + if (!ret) + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + } + pci_unlock_rescan_remove(); + + return ret; +} + +static void apple_pcie_neo_scan_functions(struct pci_dev *root) +{ + pci_lock_rescan_remove(); + if (root->subordinate) { + pci_scan_child_bus(root->subordinate); + pci_assign_unassigned_bridge_resources(root); + pci_bus_add_devices(root->subordinate); + } + pci_unlock_rescan_remove(); +} + +/* + * Keeping the firmware across a sleep is what's wanted, and this is for when it didn't work: the + * radio lost power, its rings or its IPC. Removing the functions resets the chip, and their + * probes load the firmware again from ROM, as at boot. The link only needs its rail cycled if + * it didn't come back. Otherwise the next sleep tries the whole wake again. + */ +static void apple_pcie_neo_recover_work(struct work_struct *work) +{ + struct apple_pcie *pcie = container_of(work, struct apple_pcie, neo_recover_work); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + struct pci_dev *root; + int ret = 0; + + WRITE_ONCE(pcie->neo_recover, false); + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + if (READ_ONCE(pcie->neo_rc_down)) { + ret = -EIO; + goto out; + } + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) { + dev_warn(pcie->dev, "radio link down after sleep: cycling its rail\n"); + ret = apple_pcie_neo_rail_cycle(pcie, port, root); + if (ret) + goto out; + } + ret = apple_pcie_neo_stop_functions(pcie, root); + if (ret != -ENODEV) + apple_pcie_neo_scan_functions(root); +out: + if (ret) + dev_err(pcie->dev, "radios stay down until the next sleep: %d\n", ret); + else + dev_info(pcie->dev, "radios restarted after sleep, firmware loaded again\n"); + pci_dev_put(root); +} + +/* + * Before the PCI core restores the functions, which need the link. macOS's host bridge + * holds them back the same way until its radio port is enabled. + */ +static int apple_pcie_neo_resume_noirq(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i, ret = 0; + + /* Refused: as before the sleep. A host a failed wake left unprogrammed stays down. */ + if (!pm_resume_via_firmware()) { + if (pcie->neo_rc_was_down) + return 0; + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + WRITE_ONCE(pcie->neo_rc_down, false); + return 0; + } + + list_for_each_entry(port, &pcie->ports, entry) { + /* The pin controller just put it back as it was: released, with no clock yet. */ + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = apple_pcie_neo_cold_init(pcie, port); + if (ret) { + dev_err(pcie->dev, "host did not come back after sleep: %d\n", ret); + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + apple_pcie_tunnel_restore_irq_hw(port); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, port->saved_rid2sid[i]); + } + ret = apple_dart_host_resume(pcie->neo_cold->dart); + if (ret) { + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + WRITE_ONCE(pcie->neo_rc_down, false); + apple_sleep_radio_port_up(); + + /* A failed link stays clocked, and its functions read as gone until the next sleep. */ + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) { + /* The functions' restore then finds nothing, and their drivers say so. */ + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + dev_err(pcie->dev, "radios did not come back after sleep: %d\n", ret); + apple_pcie_neo_recover(pcie); + } else { + dev_dbg(pcie->dev, "radio link back after sleep, rail and firmware kept\n"); + } + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + + return 0; +} + static int apple_pcie_resume_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); struct apple_pcie_port *port; int ret; + if (pcie->neo_sleep_armed) + return apple_pcie_neo_resume_noirq(pcie); if (!pcie->hw->tunneled) return 0; /* The domain's hold for the sleep ends first, whatever follows. */ diff --git a/include/linux/soc/apple/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h index 3b2971b8b1db4..3a496a2040989 100644 --- a/include/linux/soc/apple/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -5,16 +5,30 @@ #include #include +struct module; struct pci_dev; #if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) /* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); +/* A function that a sleep left unusable asks for both to be restarted from cold. */ +void apple_piodma_radio_recover(void); +/* What the host calls before it removes Wi-Fi for that. A failure keeps it bound. */ +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner); +#define apple_piodma_radio_set_stop(stop) __apple_piodma_radio_set_stop(stop, THIS_MODULE) #else static inline int apple_piodma_radio_check(struct pci_dev *pdev) { return -ENODEV; } + +static inline void apple_piodma_radio_recover(void) +{ +} + +static inline void apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev)) +{ +} #endif #endif /* _LINUX_SOC_APPLE_PCI_PIODMA_H */ From 79ec11e390b321a6135446848b64273cd755b149 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:19 -0500 Subject: [PATCH 42/73] wifi: mt7932: keep the firmware across a deep sleep A deep sleep took the radio away, and its firmware had to be loaded again at the wake. macOS keeps the radio powered, stops the firmware's host interface and gives the firmware ownership, then restarts it at the wake with no download. Do the same. The command is CID 0xf6, a 36-byte set with no response requested: 02 01 01 01 to suspend, 02 01 00 00 to resume, the rest zero. The firmware answers with event 0xf6, whose suspend field reads 2 for idle. Suspend waits for that answer, stops DMA and hands over ownership. Resume first checks config space, takes ownership back, writes the PCIe MAC's interrupt enable back as the boot setup does, turns DMA back on and sends the resume. The rings are expected to survive PERST#, as macOS rebuilds them only after a firmware reset; if they don't, both ring bases are logged and the radio is restarted from cold through its host. New work waits from the start of the suspend: calibration requests and regulatory changes that arrive while stopped run again after the restart, rather than failing the RF for good on the closed BAR. A restart that fails stops DMA and the interrupt again, so nothing is left running into the next sleep. Trouble here costs Wi-Fi at the wake and never refuses the sleep. Without a WoWLAN configuration cfg80211 has left the network by then, so userspace reconnects after the wake. Signed-off-by: Diljit Singh --- .../net/wireless/mediatek/mt7932/frontend.c | 3 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 231 +++++++++++++++++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 6 + .../net/wireless/mediatek/mt7932/startup.c | 3 +- .../asahi/mt7932-association-events-tests.py | 3 + 5 files changed, 238 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 3a9db56813847..634ba05e2c027 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -564,7 +564,8 @@ static void mt_cal_work(struct work_struct *work) int ret; mutex_lock(&m->command_mutex); - if (!READ_ONCE(m->rf_ready)) + /* Stopped for a sleep, the requests wait for the restart. */ + if (!READ_ONCE(m->rf_ready) || READ_ONCE(m->hif_suspended)) goto unlock; ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 94388b56449e1..084c920a3157d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -44,7 +44,8 @@ static void mt_shutdown(struct pci_dev *pdev); bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && - !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed); + !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed) && + !READ_ONCE(m->hif_suspended); } int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name) @@ -130,6 +131,30 @@ static void mt_rx_post(struct mt7932_ring *q, unsigned int index) d->control = cpu_to_le32(MT7932_RX_CAPACITY << 16); } +/* + * The host interface control command and event, the same ID both ways. The command is + * type, direction, stop, suspend in 36 bytes; the event's body is type, TX traffic, RX + * traffic, suspend, with at least as many bytes. + */ +#define MT7932_CID_HIF_CTRL 0xf6 +#define MT7932_HIF_PCIE 2 +#define MT7932_HIF_IDLE 2 +#define MT7932_HIF_LENGTH 36 + +/* response_lock held. */ +static void mt_hif_event(struct mt7932 *m, const struct mt7932_event *event) +{ + const u8 *body = event->packet + 36; + + if (!m->hif_waiting || event->length < 36 + MT7932_HIF_LENGTH || + body[0] != MT7932_HIF_PCIE || !!body[3] != m->hif_suspending) + return; + m->hif_waiting = false; + m->hif_result = body[3] && (body[1] != MT7932_HIF_IDLE || body[2] != MT7932_HIF_IDLE) ? + -EBUSY : 0; + complete(&m->hif_done); +} + static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) { struct mt7932_event event; @@ -153,6 +178,10 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) return; } spin_lock_irqsave(&m->response_lock, flags); + if (event.eid == MT7932_CID_HIF_CTRL) { + mt_hif_event(m, &event); + goto unlock; + } if (event.eid == 0xee) { mt_link_event(m, &event); goto unlock; @@ -370,6 +399,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) return -ESHUTDOWN; + /* With the host interface stopped only its own control may reach the BAR. */ + if (READ_ONCE(m->hif_gate) && !READ_ONCE(m->hif_waiting)) + return -ESHUTDOWN; if (q->used >= q->slots) { /* Command buffers can wrap only after every published descriptor has * retired. Never reclaim by elapsed time or by a reply alone. @@ -1098,6 +1130,8 @@ static ssize_t removal_retry_store(struct device *dev, } static DEVICE_ATTR_WO(removal_retry); +static struct pci_driver mt_driver; + static int mt_retire_dma(struct mt7932 *m) { int ret; @@ -1120,11 +1154,35 @@ static int mt_retire_dma(struct mt7932 *m) return ret; } +/* + * Removal waits forever for a reset that fails, which a recovery in a worker can't do. So stop + * here first, where a failure keeps the function bound. A radio stopped this way stays down + * until it's removed, and remove then has nothing left to wait for. + */ +static int mt_radio_stop(struct pci_dev *pdev) +{ + struct mt7932 *m; + int ret = 0; + + device_lock(&pdev->dev); + m = pdev->dev.driver == &mt_driver.driver ? pci_get_drvdata(pdev) : NULL; + if (m) { + mt_stop_host(m); + ret = mt_retire_dma(m); + if (ret) + dev_err(&pdev->dev, "restart refused: DMA still owned: %d\n", ret); + } + device_unlock(&pdev->dev); + + return ret; +} + static void mt_remove(struct pci_dev *pdev) { struct mt7932 *m = pci_get_drvdata(pdev); int ret; + apple_piodma_radio_set_stop(NULL); mt_stop_host(m); while ((ret = mt_retire_dma(m))) { WRITE_ONCE(m->removal_error, ret); @@ -1207,6 +1265,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) init_completion(&m->response); init_completion(&m->cal_response); init_completion(&m->reset_retry); + init_completion(&m->hif_done); mutex_init(&m->command_mutex); spin_lock_init(&m->response_lock); ret = mt_ownership(m); @@ -1242,6 +1301,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = device_create_file(&pdev->dev, &dev_attr_removal_retry); if (ret) dev_warn(&pdev->dev, "removal retry attribute unavailable: %d\n", ret); + apple_piodma_radio_set_stop(mt_radio_stop); return 0; } @@ -1255,21 +1315,179 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* command_mutex held; the firmware answers once its host interface has changed state. */ +static int mt_hif_ctrl(struct mt7932 *m, bool suspend) +{ + u8 body[MT7932_HIF_LENGTH] = { MT7932_HIF_PCIE, 1, suspend, suspend }; + unsigned long flags; + int ret; + + reinit_completion(&m->hif_done); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_suspending = suspend; + m->hif_result = -ETIMEDOUT; + m->hif_waiting = true; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt_request(m, MT7932_CID_HIF_CTRL, true, true, false, body, sizeof(body)); + if (!ret) + wait_for_completion_timeout(&m->hif_done, msecs_to_jiffies(1000)); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_waiting = false; + if (!ret) + ret = m->hif_result; + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* Every published command and frame retired, ahead of the firmware's own stop. */ +static int mt_hif_tx_idle(struct mt7932 *m) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(500); + + do { + mt_data_clean(m); + mt_tx_clean(&m->tx[17]); + if (!READ_ONCE(m->tx[0].queued) && !m->tx[17].queued) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + return -EBUSY; +} + +/* Driver own again, DMA back on, then the firmware's host interface. Rings stay as they were. */ +static int mt_hif_restart(struct mt7932 *m) +{ + int ret; + + ret = mt_ownership(m); + if (ret) + return ret; + mt_write(m, 0x10188, 0xff); + if (mt_read(m, m->rx[0].reg) != lower_32_bits(m->rx[0].dma) || + mt_read(m, m->tx[17].reg) != lower_32_bits(m->tx[17].dma)) { + dev_err(&m->pdev->dev, "DMA rings lost across the sleep: RX0 %#x, TX17 %#x\n", + mt_read(m, m->rx[0].reg), mt_read(m, m->tx[17].reg)); + return -EIO; + } + mt_write(m, W + 0x208, m->hif_glo); + WRITE_ONCE(m->running, true); + mt_write(m, W + 0x204, m->irq_mask); + mutex_lock(&m->command_mutex); + ret = mt_hif_ctrl(m, false); + if (!ret) + WRITE_ONCE(m->hif_gate, false); + mutex_unlock(&m->command_mutex); + if (ret) { + /* Not left running into the next sleep. */ + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + mt_dma_stop(m); + return ret; + } + WRITE_ONCE(m->hif_suspended, false); + /* What arrived or was asked for while stopped, now that it can run. */ + if (READ_ONCE(m->cal_request_count)) + schedule_work(&m->cal_work); + if (READ_ONCE(m->interface_registered) && READ_ONCE(m->reg_pending)) + schedule_work(&m->startup_work); + if (m->netdev) + netif_device_attach(m->netdev); + return 0; +} + +/* + * What macOS does at sleep, without its wake-on-LAN setup: cfg80211 has already left + * the network, as no wakeup is configured. The firmware stops its host interface and is + * given ownership, and keeps running. PERST# then resets the link, not the firmware. + * + * Trouble here costs Wi-Fi at the wake, never the machine its sleep. + */ +static int mt_hif_suspend(struct mt7932 *m) +{ + int ret; + + if (!READ_ONCE(m->ready) || READ_ONCE(m->stopping) || READ_ONCE(m->hif_suspended)) + return 0; + /* New work waits from here, and what already runs finishes before the gate. */ + WRITE_ONCE(m->hif_suspended, true); + if (m->interface_registered) { + if (m->netdev) + netif_device_detach(m->netdev); + flush_work(&m->startup_work); + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + flush_work(&m->cal_work); + flush_work(&m->power_work); + mt_abort_scan(m->wiphy, &m->wdev); + ret = mt_scan_quiesce(m); + if (ret) + dev_warn(&m->pdev->dev, "scan still retiring at the sleep: %d\n", ret); + } + mutex_lock(&m->command_mutex); + WRITE_ONCE(m->hif_gate, true); + ret = mt_hif_tx_idle(m); + if (!ret) + ret = mt_hif_ctrl(m, true); + mutex_unlock(&m->command_mutex); + if (ret) + dev_err(&m->pdev->dev, "firmware did not stop its host interface: %d\n", ret); + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + m->hif_glo = mt_read(m, W + 0x208) & ~(BIT(1) | BIT(3)); + ret = mt_dma_stop(m); + if (ret) + dev_err(&m->pdev->dev, "DMA did not stop for the sleep: %d\n", ret); + mt_write(m, 0xe0010, 1); + ret = mt_poll(m, 0xe0010, BIT(2), BIT(2), 50000); + if (ret) + dev_err(&m->pdev->dev, "firmware did not take ownership for the sleep: %d\n", ret); + else + dev_dbg(&m->pdev->dev, "host interface stopped for the sleep, firmware kept\n"); + return 0; +} + /* - * Stopping the firmware for sleep would take the whole remove and probe - * sequence, so leave it running across suspend-to-idle. A saved state keeps - * the PCI core from changing the function's power state or command register, - * and the awake path keeps the PCIe controller powered until resume. + * Stopping the firmware for sleep takes the whole remove and probe sequence, so + * leave it running across suspend-to-idle. A saved state keeps the PCI core from + * changing the function's power state or command register, and the awake path + * keeps the PCIe controller powered until resume. */ static int mt_suspend(struct device *dev) { if (!pm_suspend_no_platform()) - return -EOPNOTSUPP; + return mt_hif_suspend(pci_get_drvdata(to_pci_dev(dev))); pci_save_state(to_pci_dev(dev)); device_set_awake_path(dev); return 0; } +static int mt_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mt7932 *m = pci_get_drvdata(pdev); + unsigned long flags; + int ret; + + if (!READ_ONCE(m->hif_suspended)) + return 0; + /* As macOS does: a chip that no longer answers isn't touched further. */ + ret = pci_device_is_present(pdev) ? mt_hif_restart(m) : -ENODEV; + if (!ret) { + dev_dbg(dev, "back after the sleep, firmware kept\n"); + return 0; + } + dev_err(dev, "Wi-Fi didn't keep its firmware across the sleep: %d; restarting it\n", ret); + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + apple_piodma_radio_recover(); + return ret; +} + /* * Hibernation is refused. Its freeze phase does not call .suspend, so without * this the firmware would keep writing to memory while the image is saved, @@ -1286,6 +1504,7 @@ static int mt_freeze(struct device *dev) static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .resume = mt_resume, .freeze = mt_freeze, .poweroff = mt_freeze, }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index f267ffca4d496..a00bca998037c 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -92,6 +92,12 @@ struct mt7932 { bool dma_owned, module_pinned; struct completion reset_retry; int removal_error; + /* The firmware keeps running across a deep sleep with its host interface stopped. */ + struct completion hif_done; + int hif_result; + u32 hif_glo; + /* hif_suspended stops new work; hif_gate then closes the BAR to all but the control. */ + bool hif_waiting, hif_suspending, hif_suspended, hif_gate; bool ram_config; bool runtime_epoch; bool runtime_cal_valid; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 0a27e86995f99..d534a0b7cdc00 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -561,7 +561,8 @@ static void mt_startup_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); generation = m->reg_generation; *reg = m->reg_desired; - if (m->stopping || generation == m->reg_attempted) { + /* Stopped for a sleep, the restart runs this generation again. */ + if (m->stopping || READ_ONCE(m->hif_suspended) || generation == m->reg_attempted) { spin_unlock_irqrestore(&m->response_lock, flags); kfree(reg); return; diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index d5905d4d71e34..4bd4a26f6928a 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -124,6 +124,7 @@ def function(source, name): static void mt_bss_presence(struct mt7932*m,const struct mt7932_event*e){} static void mt_data_complete(struct mt7932*m,const u8*p,size_t n){} static void mt_packet_receive(struct mt7932*m,const u8*p,size_t n){} +static void mt_hif_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_rf_fail_locked(struct mt7932*m,int e){m->rf_ready=false;m->link_failed=true;m->cal_state.error=e;m->cal_state.active=false;m->connect_error=e;complete_all(&m->assoc_done);complete_all(&m->cal_response);} CHANNEL LINK @@ -252,6 +253,8 @@ def main(): headers.append(header) cal=(root/DRIVER/"calibration.h").read_text() headers.append(cal[cal.index("struct mt7932_cal_completion {"):cal.index("struct mt7932_cal_segment {")]) + # The receive path's other events, by their production IDs: the firmware's HIF report. + headers.append(re.search(r"^#define MT7932_CID_HIF_CTRL\b.*$", rx, re.M).group(0)) parts["HEADERS"]="\n".join(headers) candidates={"production":parts} if args.old_source: From 6c752dd6f23af7cd08cacb32c356fb32c9554c25 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:21 -0500 Subject: [PATCH 43/73] wifi: mt7932: stay associated and wake on WLAN across a deep sleep Implement cfg80211 WoWLAN with magic packet and disconnect triggers, for example with "iw phy phy0 wowlan enable magic-packet disconnect" or NetworkManager's wake-on-wlan. With a configuration set, a deep sleep keeps the association, and before the host interface stops the firmware is programmed as macOS does it: 1. CID 0x4f, the WoW configuration: a zeroed 32-byte header and TLV 0x06, a magic packet entry for this station's address; 2. CID 0x05, power-save mode 2; 3. CID 0x58, suspend mode: 68 bytes of BSS, enable, DTIM multiple 1 and a WoW flag of 1; 4. CID 0x4a, WoW start: 228 bytes with the detect types, filter 0x63, scenario 0, and the wake signal, the radio's GPIO 13 at level 5. The radio then wakes the host through the SMC's wifibt wake: nothing in-band can wake the SoC from a deep sleep. Once the host interface runs again after the wake, WoW is stopped (CID 0x4a, stop), its configuration cleared with TLV 0x18, suspend mode disabled and the requested power-save mode sent again. The firmware's wake reason, the first byte of event 0x62, goes to cfg80211. If a WoW command can't be queued, the driver takes its reset path, as a firmware left filtering can't be used. Without a configuration nothing changes: cfg80211 leaves the network first as before, as do s2idle and hibernation. Patterns aren't offered, since the firmware's pattern TLV doesn't match cfg80211's fixed-offset ones, and neither ARP nor GTK rekey offload is done. The command layouts are pure builders and decoders, with KUnit tests. Measured on a J700: a magic packet woke the system twice out of two, with the station still associated after each wake. Signed-off-by: Diljit Singh --- drivers/net/wireless/mediatek/mt7932/Makefile | 3 +- .../net/wireless/mediatek/mt7932/frontend.c | 1 + .../net/wireless/mediatek/mt7932/lifecycle.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.c | 28 ++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 18 ++ drivers/net/wireless/mediatek/mt7932/power.c | 2 +- .../net/wireless/mediatek/mt7932/startup.c | 7 + .../net/wireless/mediatek/mt7932/wow-test.c | 221 ++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/wow.c | 199 ++++++++++++++++ drivers/net/wireless/mediatek/mt7932/wow.h | 157 +++++++++++++ .../asahi/mt7932-association-events-tests.py | 4 +- 11 files changed, 633 insertions(+), 8 deletions(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/wow-test.c create mode 100644 drivers/net/wireless/mediatek/mt7932/wow.c create mode 100644 drivers/net/wireless/mediatek/mt7932/wow.h diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 99aa39372b49b..175f605b33cf1 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o wow.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += power-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o @@ -8,5 +8,6 @@ mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += move-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += wow-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 634ba05e2c027..a8ad970fd3fc8 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -634,6 +634,7 @@ int mt_register_interface(struct mt7932 *m) INIT_WORK(&m->connect_work, mt_connect_work); INIT_WORK(&m->disconnect_work, mt_disconnect_work); INIT_WORK(&m->power_work, mt_power_work); + INIT_WORK(&m->wow_work, mt_wow_work); init_completion(&m->assoc_start); init_completion(&m->assoc_done); init_completion(&m->disconnect_done); diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index e0ed45dccdae8..5d4bc8074bc66 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -49,6 +49,7 @@ void mt_stop_host(struct mt7932 *m) flush_work(&m->disconnect_work); cancel_work_sync(&m->cal_work); cancel_work_sync(&m->power_work); + cancel_work_sync(&m->wow_work); cancel_delayed_work_sync(&m->scan_timeout_work); cancel_work_sync(&m->scan_finish_work); /* With all producers joined, this is the sole owner of any remainder. */ diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 084c920a3157d..36eb79125166d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -7,6 +7,7 @@ #include #include "mt7932.h" +#include "wow.h" #include #include @@ -186,6 +187,10 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) mt_link_event(m, &event); goto unlock; } + if (event.eid == MT7932_EID_WOW_REASON) { + mt_wow_reason_event(m, &event); + goto unlock; + } if (event.eid == 0x0d) { mt_scan_event(m, &event); goto unlock; @@ -1354,9 +1359,13 @@ static int mt_hif_tx_idle(struct mt7932 *m) return -EBUSY; } -/* Driver own again, DMA back on, then the firmware's host interface. Rings stay as they were. */ +/* + * Driver own again, DMA back on, then the firmware's host interface, and WoW + * off if the sleep had it. Rings stay as they were. + */ static int mt_hif_restart(struct mt7932 *m) { + bool wow; int ret; ret = mt_ownership(m); @@ -1373,9 +1382,14 @@ static int mt_hif_restart(struct mt7932 *m) WRITE_ONCE(m->running, true); mt_write(m, W + 0x204, m->irq_mask); mutex_lock(&m->command_mutex); + wow = m->wow_armed; ret = mt_hif_ctrl(m, false); - if (!ret) + if (!ret) { WRITE_ONCE(m->hif_gate, false); + ret = mt_wow_disarm(m); + if (ret) + WRITE_ONCE(m->hif_gate, true); + } mutex_unlock(&m->command_mutex); if (ret) { /* Not left running into the next sleep. */ @@ -1390,6 +1404,8 @@ static int mt_hif_restart(struct mt7932 *m) /* What arrived or was asked for while stopped, now that it can run. */ if (READ_ONCE(m->cal_request_count)) schedule_work(&m->cal_work); + if (wow) + schedule_work(&m->power_work); if (READ_ONCE(m->interface_registered) && READ_ONCE(m->reg_pending)) schedule_work(&m->startup_work); if (m->netdev) @@ -1398,9 +1414,10 @@ static int mt_hif_restart(struct mt7932 *m) } /* - * What macOS does at sleep, without its wake-on-LAN setup: cfg80211 has already left - * the network, as no wakeup is configured. The firmware stops its host interface and is - * given ownership, and keeps running. PERST# then resets the link, not the firmware. + * What macOS does at sleep. Without a WoWLAN configuration cfg80211 has already left + * the network; with one the firmware is given WoW first and stays associated. The firmware + * stops its host interface and is given ownership, and keeps running. PERST# then resets + * the link, not the firmware. * * Trouble here costs Wi-Fi at the wake, never the machine its sleep. */ @@ -1426,6 +1443,7 @@ static int mt_hif_suspend(struct mt7932 *m) dev_warn(&m->pdev->dev, "scan still retiring at the sleep: %d\n", ret); } mutex_lock(&m->command_mutex); + mt_wow_arm(m); WRITE_ONCE(m->hif_gate, true); ret = mt_hif_tx_idle(m); if (!ret) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index a00bca998037c..2c534c331a82e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -161,6 +161,14 @@ struct mt7932 { /* Desired/submitted PS configuration uses command_mutex; TIM uses response_lock. */ u8 power_mode, power_applied; bool power_applied_valid, power_tim; + /* WoWLAN detectors cfg80211 asked for at this sleep; 0 leaves the network. */ + u8 wow_detect; + /* The firmware was given WoW at the sleep; command_mutex. */ + bool wow_armed; + /* A wake reason is awaited, and the one received; response_lock. */ + bool wow_report; + u8 wow_reason; + struct work_struct wow_work; struct completion assoc_start, assoc_done, disconnect_done; struct completion discovery_done; struct completion scan_done; @@ -350,5 +358,15 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); +int mt_power_submit(struct mt7932 *m, u8 mode); +#ifdef CONFIG_PM +extern const struct wiphy_wowlan_support mt_wowlan_support; +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow); +int mt_wow_resume(struct wiphy *wiphy); +#endif +void mt_wow_arm(struct mt7932 *m); +int mt_wow_disarm(struct mt7932 *m); +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event); +void mt_wow_work(struct work_struct *work); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index f3c1c42863c9c..03653ba97eede 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -5,7 +5,7 @@ * submission is accepted configuration, not proof that the radio is asleep. * cfg80211 implements GET_POWER_SAVE from the accepted requested setting. */ -static int mt_power_submit(struct mt7932 *m, u8 mode) +int mt_power_submit(struct mt7932 *m, u8 mode) { u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index d534a0b7cdc00..4441150dcef4a 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -805,6 +805,10 @@ static const struct cfg80211_ops mt_cfg_ops = { .dump_station = mt_dump_station, .set_power_mgmt = mt_set_power_mgmt, .set_cqm_rssi_config = mt_set_cqm_rssi_config, +#ifdef CONFIG_PM + .suspend = mt_wow_suspend, + .resume = mt_wow_resume, +#endif }; int mt_register_regulatory_gate(struct mt7932 *m) @@ -841,6 +845,9 @@ int mt_register_regulatory_gate(struct mt7932 *m) wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_MFP_OPTIONAL); m->wiphy->max_scan_ssids = MT7932_SCAN_SSIDS; m->wiphy->max_scan_ie_len = 600; +#ifdef CONFIG_PM + m->wiphy->wowlan = &mt_wowlan_support; +#endif m->wiphy->reg_notifier = mt_regulatory_notify; /* World roaming may relax NO_IR after observing an AP beacon without * changing alpha2. Request the callback so firmware's CID0f domain is diff --git a/drivers/net/wireless/mediatek/mt7932/wow-test.c b/drivers/net/wireless/mediatek/mt7932/wow-test.c new file mode 100644 index 0000000000000..1301fa93b7bbb --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow-test.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "mt7932.h" +#include "wow.h" + +static bool mt_wow_test_zero(const u8 *p, size_t from, size_t to) +{ + size_t i; + + for (i = from; i < to; i++) + if (p[i]) + return false; + return true; +} + +static void mt_wow_detect_test(struct kunit *test) +{ + struct cfg80211_wowlan wow = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(NULL), 0); + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), MT7932_WOW_DETECT_MAGIC); + wow.magic_pkt = false; + wow.disconnect = true; + /* Losing the AP's beacons ends the association as surely as a deauth. */ + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x14); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x15); +} + +static void mt_wow_ctrl_test(struct kunit *test) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)228); + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, true, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_START); + KUNIT_EXPECT_EQ(test, body[1], 0x15); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), 0x63); + KUNIT_EXPECT_EQ(test, body[4], 0); /* scenario */ + KUNIT_EXPECT_EQ(test, body[5], 1); /* pins */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 6, 8)); + KUNIT_EXPECT_EQ(test, body[8], 3); /* GPIO */ + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + /* Interval, second wake signal, port counts and lists. */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); + + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, false, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_STOP); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 1, 5)); + KUNIT_EXPECT_EQ(test, body[5], 1); + KUNIT_EXPECT_EQ(test, body[8], 3); + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); +} + +static void mt_wow_suspend_mode_test(struct kunit *test) +{ + u8 body[MT7932_SUSPEND_MODE_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)68); + memset(body, 0xa5, sizeof(body)); + mt7932_suspend_mode(body, true); + KUNIT_EXPECT_EQ(test, body[0], 0); /* BSS */ + KUNIT_EXPECT_EQ(test, body[1], 1); + KUNIT_EXPECT_EQ(test, body[2], 1); /* DTIM multiple */ + KUNIT_EXPECT_EQ(test, body[3], 1); /* WoW suspend */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 4, sizeof(body))); + mt7932_suspend_mode(body, false); + KUNIT_EXPECT_EQ(test, body[1], 0); + KUNIT_EXPECT_EQ(test, body[2], 1); + KUNIT_EXPECT_EQ(test, body[3], 1); +} + +static void mt_wow_config_test(struct kunit *test) +{ + static const u8 addr[ETH_ALEN] = { 0x02, 0x11, 0x22, 0x33, 0x44, 0x55 }; + u8 magic[MT7932_WOW_MAGIC_LENGTH], clear[MT7932_WOW_CLEAR_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(magic), (size_t)52); + KUNIT_EXPECT_EQ(test, sizeof(clear), (size_t)44); + memset(magic, 0xa5, sizeof(magic)); + mt7932_wow_magic(magic, addr); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 32), 0x06); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 34), 20); + KUNIT_EXPECT_EQ(test, magic[36], 0); /* entry */ + KUNIT_EXPECT_EQ(test, magic[37], 1); /* enabled */ + KUNIT_EXPECT_EQ(test, magic[38], 1); /* forward */ + KUNIT_EXPECT_EQ(test, magic[39], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 40), 0); + KUNIT_EXPECT_MEMEQ(test, magic + 42, addr, ETH_ALEN); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 48, sizeof(magic))); + + memset(clear, 0xa5, sizeof(clear)); + mt7932_wow_clear(clear); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 32), 0x18); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 34), 12); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 36, sizeof(clear))); +} + +static void mt_wow_reason_test(struct kunit *test) +{ + struct cfg80211_wowlan_wakeup wakeup; + u8 body[MT7932_WOW_REASON_LENGTH] = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body) - 1), -EINVAL); + body[0] = 9; + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body)), 9); + + KUNIT_EXPECT_FALSE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_NONE, &wakeup)); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_MAGIC, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_DISCONNECT, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.disconnect); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + /* A firmware reason with no cfg80211 trigger is still a wireless wake. */ + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(9, &wakeup)); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); +} + +#ifdef CONFIG_PM +static void mt_wow_test_work(struct work_struct *work) +{ +} + +static void mt_wow_test_device_release(struct device *dev) +{ +} + +static void mt_wow_test_put(void *data) +{ + put_device(data); +} + +static void mt_wow_reason_event_test(struct kunit *test) +{ + struct mt7932 *m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + u8 *packet = kunit_kzalloc(test, 36 + MT7932_WOW_REASON_LENGTH, GFP_KERNEL); + struct mt7932_event event = { + .eid = MT7932_EID_WOW_REASON, + .packet = packet, + .length = 36 + MT7932_WOW_REASON_LENGTH, + }; + unsigned long flags; + + KUNIT_ASSERT_NOT_NULL(test, m); + KUNIT_ASSERT_NOT_NULL(test, packet); + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, m->pdev); + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_wow_test_device_release; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mt_wow_test_put, &m->pdev->dev), 0); + KUNIT_ASSERT_EQ(test, dev_set_name(&m->pdev->dev, "mt7932-wow-test"), 0); + spin_lock_init(&m->response_lock); + INIT_WORK(&m->wow_work, mt_wow_test_work); + m->wow_reason = MT7932_WOW_REASON_NONE; + packet[36] = MT7932_WOW_REASON_MAGIC; + + /* Not a sleep WoW was armed for: ignored. */ + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* A short event leaves the wait open. */ + m->wow_report = true; + event.length--; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_TRUE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* One report per sleep. */ + event.length++; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_FALSE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); + packet[36] = MT7932_WOW_REASON_DISCONNECT; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); +} +#endif + +static struct kunit_case mt_wow_test_cases[] = { + KUNIT_CASE(mt_wow_detect_test), + KUNIT_CASE(mt_wow_ctrl_test), + KUNIT_CASE(mt_wow_suspend_mode_test), + KUNIT_CASE(mt_wow_config_test), + KUNIT_CASE(mt_wow_reason_test), +#ifdef CONFIG_PM + KUNIT_CASE(mt_wow_reason_event_test), +#endif + {} +}; + +static struct kunit_suite mt_wow_test_suite = { + .name = "mt7932-wow", + .test_cases = mt_wow_test_cases, +}; + +kunit_test_suite(mt_wow_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/wow.c b/drivers/net/wireless/mediatek/mt7932/wow.c new file mode 100644 index 0000000000000..279a0c01a7b1d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Wake-on-WLAN for a deep sleep. The firmware keeps the association while the + * host sleeps and wakes it through the radio's GPIO, which the SMC turns into + * an AP wake. Nothing in-band can wake the SoC from S2R. + */ +#include + +#include "mt7932.h" +#include "wow.h" + +#ifdef CONFIG_PM +const struct wiphy_wowlan_support mt_wowlan_support = { + .flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT, +}; + +/* response_lock held. */ +static bool mt_wow_associated_locked(struct mt7932 *m) +{ + return m->connected && m->peer_valid && !m->connecting && !m->disconnecting && + !m->link_failed && !m->stopping && m->rf_ready; +} + +/* + * cfg80211 calls this before the device's own suspend. Returning 1 makes it + * leave the network and call again without a configuration. + */ +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + u8 detect = mt7932_wow_detect(wow); + unsigned long flags; + bool associated; + + WRITE_ONCE(m->wow_detect, 0); + if (!wow) + return 0; + /* Only S2R hands the radio to the firmware; s2idle and hibernation leave. */ + if (!detect || pm_suspend_target_state != PM_SUSPEND_MEM || pm_suspend_no_platform()) + return 1; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) + return 1; + WRITE_ONCE(m->wow_detect, detect); + return 0; +} + +int mt_wow_resume(struct wiphy *wiphy) +{ + WRITE_ONCE(mt_from_wiphy(wiphy)->wow_detect, 0); + return 0; +} + +/* + * command_mutex held, before the host interface stops. The firmware takes the + * commands from the ring before it answers the stop. + */ +void mt_wow_arm(struct mt7932 *m) +{ + u8 detect = READ_ONCE(m->wow_detect); + u8 body[MT7932_WOW_CTRL_LENGTH]; + unsigned long flags; + bool associated; + int ret = 0; + + lockdep_assert_held(&m->command_mutex); + WRITE_ONCE(m->wow_detect, 0); + if (!detect) + return; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + m->wow_report = false; + m->wow_reason = MT7932_WOW_REASON_NONE; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) { + dev_dbg(&m->pdev->dev, "not associated at the sleep: Wi-Fi wake not armed\n"); + return; + } + /* Whatever reached the firmware is undone at the wake. */ + m->wow_armed = true; + if (detect & MT7932_WOW_DETECT_MAGIC) { + mt7932_wow_magic(body, m->mac); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_MAGIC_LENGTH); + } + if (!ret) + ret = mt_power_submit(m, MT7932_WOW_POWER_MODE); + if (!ret) { + mt7932_suspend_mode(body, true); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + } + if (!ret) { + mt7932_wow_ctrl(body, true, detect); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (ret) { + dev_err(&m->pdev->dev, "Wi-Fi wake not armed: %d\n", ret); + return; + } + spin_lock_irqsave(&m->response_lock, flags); + m->wow_report = true; + spin_unlock_irqrestore(&m->response_lock, flags); + dev_dbg(&m->pdev->dev, "Wi-Fi wake armed:%s%s, staying associated\n", + detect & MT7932_WOW_DETECT_MAGIC ? " magic packet" : "", + detect & MT7932_WOW_DETECT_DISCONNECT ? " disconnect" : ""); +} + +/* + * command_mutex held, once the host interface runs again. A command that + * cannot be queued leaves the firmware filtering, so the caller resets it. + */ +int mt_wow_disarm(struct mt7932 *m) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + int ret; + + lockdep_assert_held(&m->command_mutex); + if (!m->wow_armed) + return 0; + m->wow_armed = false; + mt7932_suspend_mode(body, false); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + if (!ret) { + mt7932_wow_ctrl(body, false, 0); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (!ret) { + mt7932_wow_clear(body); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_CLEAR_LENGTH); + } + /* The sleep's profile stays until power_work submits the requested one. */ + m->power_applied_valid = false; + if (ret) + dev_err(&m->pdev->dev, "Wi-Fi wake not stopped: %d\n", ret); + return ret; +} + +/* response_lock held. The firmware reports why it woke the host. */ +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ + int reason = event->length < 36 ? -EINVAL : + mt7932_wow_reason(event->packet + 36, event->length - 36); + + if (reason < 0) { + dev_dbg(&m->pdev->dev, "short wake reason event: %zu bytes\n", event->length); + return; + } + if (!m->wow_report) + return; + m->wow_report = false; + m->wow_reason = reason; + schedule_work(&m->wow_work); +} + +void mt_wow_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, wow_work); + struct cfg80211_wowlan_wakeup wakeup; + unsigned long flags; + u8 reason; + + spin_lock_irqsave(&m->response_lock, flags); + reason = m->wow_reason; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!mt7932_wow_wakeup(reason, &wakeup)) { + dev_dbg(&m->pdev->dev, "Wi-Fi did not wake the host\n"); + cfg80211_report_wowlan_wakeup(&m->wdev, NULL, GFP_KERNEL); + return; + } + dev_info(&m->pdev->dev, "woken by Wi-Fi: firmware reason %u%s\n", reason, + wakeup.magic_pkt ? " (magic packet)" : + wakeup.disconnect ? " (disconnect)" : ""); + cfg80211_report_wowlan_wakeup(&m->wdev, &wakeup, GFP_KERNEL); +} +#else +void mt_wow_arm(struct mt7932 *m) +{ +} + +int mt_wow_disarm(struct mt7932 *m) +{ + return 0; +} + +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ +} + +void mt_wow_work(struct work_struct *work) +{ +} +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/wow.h b/drivers/net/wireless/mediatek/mt7932/wow.h new file mode 100644 index 0000000000000..2bbe7d4529a1b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_WOW_H +#define MT7932_WOW_H + +#include +#include +#include +#include +#include +#include + +/* + * Wake-on-WLAN commands, in the order the stock driver sends them for an + * associated sleep: the magic packet entry on the WoW configuration command, + * the power-save profile, suspend mode, then WoW start. Waking reverses it. + */ +#define MT7932_CID_WOW_CTRL 0x4a +#define MT7932_CID_WOW_CONFIG 0x4f +#define MT7932_CID_SUSPEND_MODE 0x58 +#define MT7932_EID_WOW_REASON 0x62 + +#define MT7932_WOW_CTRL_LENGTH 228 +#define MT7932_SUSPEND_MODE_LENGTH 68 +/* The configuration command is a zeroed 32-byte header and one TLV. */ +#define MT7932_WOW_CONFIG_HEADER 32 +#define MT7932_WOW_MAGIC_LENGTH (MT7932_WOW_CONFIG_HEADER + 20) +#define MT7932_WOW_CLEAR_LENGTH (MT7932_WOW_CONFIG_HEADER + 12) +#define MT7932_WOW_TLV_MAGIC 0x06 +#define MT7932_WOW_TLV_CLEAR 0x18 + +#define MT7932_WOW_START 1 +#define MT7932_WOW_STOP 2 + +#define MT7932_WOW_DETECT_MAGIC BIT(0) +#define MT7932_WOW_DETECT_DISCONNECT BIT(2) +#define MT7932_WOW_DETECT_BEACON_LOST BIT(4) + +/* + * Drop everything but magic packets, EAPOL and ARP requests to this station, + * as the stock driver filters for a sleep. + */ +#define MT7932_WOW_FILTER 0x63 + +/* The stock configuration wakes the host on the radio's GPIO 13, level 5. */ +#define MT7932_WOW_HIF_GPIO 3 +#define MT7932_WOW_GPIO_PIN 13 +#define MT7932_WOW_GPIO_LEVEL 5 +#define MT7932_WOW_PIN_COUNT 1 + +/* Power-save profile for the sleep, and the suspend mode's DTIM multiple. */ +#define MT7932_WOW_POWER_MODE 2 +#define MT7932_WOW_MDTIM 1 + +#define MT7932_WOW_REASON_MAGIC 0 +#define MT7932_WOW_REASON_DISCONNECT 8 +#define MT7932_WOW_REASON_NONE 0xff +/* The stock driver ignores a shorter reason event. */ +#define MT7932_WOW_REASON_LENGTH 132 + +/* The firmware detectors for the triggers cfg80211 asked for; 0 if none applies. */ +static inline u8 mt7932_wow_detect(const struct cfg80211_wowlan *wow) +{ + u8 detect = 0; + + if (!wow) + return 0; + if (wow->magic_pkt) + detect |= MT7932_WOW_DETECT_MAGIC; + if (wow->disconnect) + detect |= MT7932_WOW_DETECT_DISCONNECT | MT7932_WOW_DETECT_BEACON_LOST; + return detect; +} + +/* Start, with these detectors, or stop. */ +static inline void mt7932_wow_ctrl(u8 out[MT7932_WOW_CTRL_LENGTH], bool start, u8 detect) +{ + memset(out, 0, MT7932_WOW_CTRL_LENGTH); + out[0] = start ? MT7932_WOW_START : MT7932_WOW_STOP; + if (start) { + out[1] = detect; + put_unaligned_le16(MT7932_WOW_FILTER, out + 2); + } + /* Scenario 0, then the wake signal: one pin, GPIO, which pin, level. */ + out[5] = MT7932_WOW_PIN_COUNT; + out[8] = MT7932_WOW_HIF_GPIO; + out[9] = MT7932_WOW_GPIO_PIN; + out[10] = MT7932_WOW_GPIO_LEVEL; + /* No UDP or TCP wake ports. */ +} + +/* BSS 0 into or out of suspend mode; the DTIM multiple and WoW flag stay set. */ +static inline void mt7932_suspend_mode(u8 out[MT7932_SUSPEND_MODE_LENGTH], bool enable) +{ + memset(out, 0, MT7932_SUSPEND_MODE_LENGTH); + out[1] = enable; + out[2] = MT7932_WOW_MDTIM; + out[3] = 1; +} + +/* Magic packet entry 0 for this address, matched from the start of the frame. */ +static inline void mt7932_wow_magic(u8 out[MT7932_WOW_MAGIC_LENGTH], const u8 *addr) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_MAGIC_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_MAGIC, tlv); + put_unaligned_le16(MT7932_WOW_MAGIC_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); + tlv[4] = 0; /* entry */ + tlv[5] = 1; /* enabled */ + tlv[6] = 1; /* offset counts forward */ + put_unaligned_le16(0, tlv + 8); + ether_addr_copy(tlv + 10, addr); +} + +/* Remove every WoW configuration entry. */ +static inline void mt7932_wow_clear(u8 out[MT7932_WOW_CLEAR_LENGTH]) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_CLEAR_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_CLEAR, tlv); + put_unaligned_le16(MT7932_WOW_CLEAR_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); +} + +/* + * The reason from a wake reason event body, or -EINVAL if it is short. + * MT7932_WOW_REASON_NONE means the radio did not wake the host. + */ +static inline int mt7932_wow_reason(const u8 *body, size_t length) +{ + if (length < MT7932_WOW_REASON_LENGTH) + return -EINVAL; + return body[0]; +} + +/* + * What to tell cfg80211 for a reason. False if the radio did not wake the + * host. A reason without a cfg80211 trigger reports a wake with none set. + */ +static inline bool mt7932_wow_wakeup(u8 reason, struct cfg80211_wowlan_wakeup *wakeup) +{ + memset(wakeup, 0, sizeof(*wakeup)); + wakeup->pattern_idx = -1; + switch (reason) { + case MT7932_WOW_REASON_NONE: + return false; + case MT7932_WOW_REASON_MAGIC: + wakeup->magic_pkt = true; + break; + case MT7932_WOW_REASON_DISCONNECT: + wakeup->disconnect = true; + break; + } + return true; +} + +#endif diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 4bd4a26f6928a..d3b4d3a2ca787 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -125,6 +125,7 @@ def function(source, name): static void mt_data_complete(struct mt7932*m,const u8*p,size_t n){} static void mt_packet_receive(struct mt7932*m,const u8*p,size_t n){} static void mt_hif_event(struct mt7932*m,const struct mt7932_event*e){} +static void mt_wow_reason_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_rf_fail_locked(struct mt7932*m,int e){m->rf_ready=false;m->link_failed=true;m->cal_state.error=e;m->cal_state.active=false;m->connect_error=e;complete_all(&m->assoc_done);complete_all(&m->cal_response);} CHANNEL LINK @@ -253,8 +254,9 @@ def main(): headers.append(header) cal=(root/DRIVER/"calibration.h").read_text() headers.append(cal[cal.index("struct mt7932_cal_completion {"):cal.index("struct mt7932_cal_segment {")]) - # The receive path's other events, by their production IDs: the firmware's HIF report. + # The receive path's other events, by their production IDs: the firmware's HIF and WoW reports. headers.append(re.search(r"^#define MT7932_CID_HIF_CTRL\b.*$", rx, re.M).group(0)) + headers.append(re.search(r"^#define MT7932_EID_WOW_REASON\b.*$", (root/DRIVER/"wow.h").read_text(), re.M).group(0)) parts["HEADERS"]="\n".join(headers) candidates={"production":parts} if args.old_source: From 1864a50964115f8ba5d2e562bca569cb6c410e00 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:21 -0500 Subject: [PATCH 44/73] Bluetooth: mt7932: keep the controller across a deep sleep and wake from it Bluetooth's transport now stays bound through a deep sleep, as the radio keeps its power and firmware, and the controller should be able to wake the system as it can from suspend-to-idle. - The wake path existed for suspend-to-idle only: HCI suspended with its wake filters, the vendor host-sleep command, the transport kept and the MSI armed. A deep sleep takes the link down, so the controller has to wake the system out of band, through the SMC, and macOS puts it into host sleep for that. Its Bluetooth driver (27.2, read statically) takes the host-sleep bytes (01 6f fc 07, then seven 0xff) as a request to its own PCIe transport and never sends them to the controller: it writes 3 to the sleep control register at 0x3382c, rings doorbell 12 and waits up to 1 s for sleep notification 3, which is this driver's quiesce. In host sleep it ignores the controller's sleep notifications, and at the wake it writes control 0 and waits for notification 0. The wake scans are bluetoothd's HCI setup, which on Linux is hci_suspend_dev() with its event filters and accept list. So with wakeup enabled a deep sleep suspends HCI with its wake setup and then quiesces the controller, and the resume ends the host sleep with the existing restore. Nothing is published while the controller sleeps, so there are no doorbells left to ring after it, and a quiesced controller's notifications are left to the restore rather than faulting the transport. Wakeup is enabled on the function by default. If HCI didn't set its wake up, or the quiesce fails, a deep sleep goes in with no Bluetooth wake rather than being refused; suspend-to-idle keeps its running transport, its MSI and its refusals. - The link is retrained under this function at the wake. If that failed its BAR no longer answers, so the restore checks config space first and faults the transport, and asks the host for a cold restart of the radios. A faulted transport doesn't refuse a deep sleep either, as that would cost the machine every later one until a reboot. - The host removes Bluetooth to restart the radios from cold, and that removal waited out HCI_CMD_TIMEOUT for a Reset that HCI_QUIRK_RESET_ON_CLOSE queued after the device's command work was already disabled. Drop the quirk before unregistering; a close from userspace still resets the controller. Measured on a J700: a Bluetooth keyboard wakes the system from a deep sleep. Signed-off-by: Diljit Singh --- drivers/bluetooth/mt7932_bt_pcie.c | 101 ++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 25 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 72910ef976f8e..87b4436a65a91 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -520,11 +520,13 @@ static int bt7932_sleep_locked(struct bt7932 *bt) } if (value > 1) return -EPROTO; - /* A previous host-sleep notification must not cancel control 3. */ - if (bt->pm_state == BT7932_QUIESCING) + /* + * A previous host-sleep notification must not cancel control 3. Once + * quiesced, only the host ends the host sleep, so leave the rest to + * bt7932_restore_transport(), as macOS ignores it then too. + */ + if (bt->pm_state == BT7932_QUIESCING || bt->pm_state == BT7932_QUIESCED) return 0; - if (bt->pm_state == BT7932_QUIESCED) - return -EPROTO; if (bt->pm_state == BT7932_RESTORING) { if (value) { if (!bt->waking) { @@ -1125,7 +1127,7 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) * Follows the wakeup setting of the PCI function, see bt7932_probe(). * Only while it is on does hci_suspend_sync() leave page scan and LE passive * scan set up for the devices flagged for remote wakeup, and only then does - * bt7932_suspend() keep the transport running with its interrupt armed. + * bt7932_suspend() leave the controller able to wake the system. */ static bool bt7932_wakeup(struct hci_dev *hdev) { @@ -1487,14 +1489,15 @@ static bool bt7932_hci_pm(struct bt7932 *bt) } /* - * Suspend-to-idle can keep the transport running so that the controller can - * wake the system: with wakeup enabled on the PCI function, the controller - * up and HCI suspended by this driver. + * The controller can wake the system with wakeup enabled on the PCI function, the controller + * up and HCI suspended by this driver. In suspend-to-idle the transport stays up and the wake + * is its MSI. A deep sleep takes the link down: the controller goes into host sleep, which is + * the quiesce, and wakes the system through the SMC, as macOS does. */ static bool bt7932_may_wake(struct bt7932 *bt) { - return device_may_wakeup(&bt->pdev->dev) && pm_suspend_no_platform() && - bt7932_hci_pm(bt) && test_bit(HCI_UP, &bt->hdev->flags); + return device_may_wakeup(&bt->pdev->dev) && bt7932_hci_pm(bt) && + test_bit(HCI_UP, &bt->hdev->flags); } static void bt7932_track_wake(struct bt7932 *bt, bool on) @@ -1505,11 +1508,12 @@ static void bt7932_track_wake(struct bt7932 *bt, bool on) } /* - * Before system sleep, the host tells the controller that it is going to - * sleep with a vendor command whose parameters are all ones; nothing is sent - * when the host is back. It is answered by a vendor event, if at all, so - * place it on the command ring directly rather than through the HCI command - * flow, and wait until the controller has taken it. + * For suspend-to-idle with the transport kept running: a vendor command whose + * parameters are all ones. macOS uses the same bytes, but only as a request to + * its own transport, which turns them into the host sleep of a deep sleep (see + * bt7932_suspend()) and never sends them on. It is answered by a vendor event, + * if at all, so place it on the command ring directly rather than through the + * HCI command flow, and wait until the controller has taken it. * * Returns -EAGAIN if nothing was published (the controller could not be * reached without a fault, or the ring was full), -EBUSY if the command was @@ -1637,6 +1641,12 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) * The PCIe link and host bridge stay up in suspend-to-idle anyway, as the * Wi-Fi function on the same link is never stopped, and ASPM stays off on * this link as the transport was admitted. + * + * A deep sleep with wakeup enabled quiesces the transport after HCI has set + * up its wake scans. The quiesce (sleep control 3, notification 3) is the + * host sleep that macOS enters at S2R. The controller keeps scanning and + * wakes the system through the SMC. bt7932_resume() ends the host sleep + * with control 0, as macOS does. */ static int bt7932_suspend(struct device *dev) { @@ -1644,8 +1654,9 @@ static int bt7932_suspend(struct device *dev) bool wake; int ret; + /* A transport that's down has nothing to stop, and it can't cost a deep sleep. */ if (READ_ONCE(bt->fault)) - return -EIO; + return pm_suspend_no_platform() ? -EIO : 0; if (!bt->registered) return 0; wake = bt7932_may_wake(bt); @@ -1656,16 +1667,19 @@ static int bt7932_suspend(struct device *dev) ret = hci_suspend_dev(bt->hdev); if (ret) { bt7932_track_wake(bt, false); - return ret; + if (pm_suspend_no_platform()) + return ret; + dev_warn(dev, "HCI did not suspend: %d\n", ret); } } - if (wake) { + if (wake && pm_suspend_no_platform()) { ret = bt7932_arm_wakeup(bt); if (!ret) { - dev_dbg(dev, "system sleep with wakeup armed\n"); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } bt7932_track_wake(bt, false); + wake = false; if (ret == -EBUSY) { /* * The host sleep command may still take effect, so the @@ -1676,8 +1690,7 @@ static int bt7932_suspend(struct device *dev) hci_resume_dev(bt->hdev); bt7932_reset(bt->hdev); return ret; - } - if (ret != -EAGAIN) { + } else if (ret != -EAGAIN) { dev_err(dev, "transport failed while arming wakeup\n"); hci_resume_dev(bt->hdev); return ret; @@ -1686,9 +1699,24 @@ static int bt7932_suspend(struct device *dev) ret = bt7932_quiesce(bt); if (!ret) { dev_dbg(dev, "system sleep quiesce acknowledged\n"); + if (!wake) + return 0; + /* hci_suspend_dev() may have stopped short of the wake setup. */ + if (!bt->hdev->suspended || + bt->hdev->suspend_state != BT_SUSPEND_CONFIGURE_WAKE) { + bt7932_track_wake(bt, false); + dev_warn(dev, "no Bluetooth wake for this sleep\n"); + return 0; + } + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } + if (wake) + bt7932_track_wake(bt, false); dev_warn(dev, "system sleep quiesce failed: %d\n", ret); + /* The wake finds out what's left. A deep sleep takes the power regardless. */ + if (!pm_suspend_no_platform()) + return 0; /* Undo our request before the PM core rolls back the rest of the bus. */ if (!READ_ONCE(bt->fault)) bt7932_restore_transport(bt); @@ -1734,21 +1762,38 @@ static int bt7932_resume(struct device *dev) struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); int ret; + /* A deep sleep retrains the link. With the chip gone, nothing here may touch it. */ + if (!pci_device_is_present(bt->pdev)) { + bt7932_track_wake(bt, false); + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "radio gone after the sleep"); + mutex_unlock(&bt->lock); + apple_piodma_radio_recover(); + return -ENODEV; + } + if (bt->wake_armed) { bt7932_disarm_wakeup(bt); if (READ_ONCE(bt->fault)) return -EIO; return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } - if (READ_ONCE(bt->fault)) + if (READ_ONCE(bt->fault)) { + bt7932_track_wake(bt, false); return -EIO; + } if (!bt->registered) return 0; if (READ_ONCE(bt->pm_state) != BT7932_RUNNING) { ret = bt7932_restore_transport(bt); - if (ret) + if (ret) { + bt7932_track_wake(bt, false); + apple_piodma_radio_recover(); return ret; + } } + /* Reports now are no longer wake events. */ + bt7932_track_wake(bt, false); dev_dbg(dev, "transport restored after system sleep\n"); return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } @@ -1778,9 +1823,15 @@ static void bt7932_remove(struct pci_dev *pdev) u16 command = U16_MAX; bool quiesced, drained; - /* Closing HCI may send Reset: keep the transport and IRQ alive for it. */ - if (bt->registered) + /* + * The HCI core stops its command and event work before it closes a device it + * unregisters, so a Reset on that close goes unsent and waits out its 2 s timeout. + * The quiesce below stops the firmware. Close still runs with the transport alive. + */ + if (bt->registered) { + hci_clear_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_unregister_dev(bt->hdev); + } quiesced = bt->ipc_ready && !READ_ONCE(bt->fault) && !bt7932_quiesce(bt); mutex_lock(&bt->lock); WRITE_ONCE(bt->stopping, true); From c473ad585c4e2889d53842002de0568fc5d057a9 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 06:27:41 -0500 Subject: [PATCH 45/73] Bluetooth: mt7932: log what woke the host from a sleep After a sleep with Bluetooth wake set up, the controller delivers the event that woke the host once the transport runs again. Two things went wrong with it: - The HCI core records a wake reason only while hdev->suspended is set (hci_store_wake_reason()). The resume called hci_resume_dev() straight after the restore, which clears that flag before the RX work had seen the event, so mgmt reported no wake reason. - The interrupt thread doesn't touch the rings while the transport is restoring, so an event the controller delivered then waited for the next interrupt. At resume the driver now takes the rings itself and waits up to 200 ms for a wake report. It logs the first one, flushes the core's RX work and only then resumes HCI. The log lines: woken by Bluetooth: connection complete from woken by Bluetooth: LE advertising report from The second form is for LE. A wake by anything else logs that Bluetooth didn't wake the host at debug level, as the Wi-Fi driver does. The SMC record names one wifibt bit for both radios, so this line and Wi-Fi's are what tell them apart. Suspend-to-idle takes the reason the same way after its interrupt is re-enabled. The 200 ms is an estimate of how long the controller takes to hand the event over once host sleep ends. A wake by something else costs that wait once per resume. The resume runs asynchronously, so other devices don't wait for it. KUnit covers the names and the address offsets. Signed-off-by: Diljit Singh --- drivers/bluetooth/mt7932_bt_pcie.c | 121 +++++++++++++++++-- drivers/bluetooth/mt7932_bt_pcie_wake_test.c | 46 +++++++ 2 files changed, 160 insertions(+), 7 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 87b4436a65a91..d60458d69c590 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -65,6 +65,8 @@ /* Vendor command telling the controller that the host goes to sleep. */ #define BT7932_OP_HOST_SLEEP 0xfc6f #define BT7932_HOST_SLEEP_PLEN 7 +/* How long a wake reason may take to arrive once the transport is back. */ +#define BT7932_WAKE_REASON_MS 200 #define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" #define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" @@ -153,6 +155,8 @@ struct bt7932 { bool tx_blocked; /* nothing is sent until resume */ bool irq_held; /* IRQ off until the PCI state is restored */ bool pci_saved; /* saved PCI state not yet restored */ + bool wake_quiesced; /* in host sleep with HCI set up to wake, deep sleep */ + bool wake_expected; /* the next wake report is what woke the host */ enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; }; @@ -300,6 +304,70 @@ static bool bt7932_wake_report(struct bt7932 *bt, const u8 *data, size_t len) } } +/* + * Names a wake report (see bt7932_wake_event()) for the log, and points @addr + * at the remote address it carries, or NULL. + */ +static const char *bt7932_wake_name(const u8 *data, size_t len, const u8 **addr) +{ + /* Where the address starts: after status and handle, or LE report fields. */ + size_t offset = 0; + const char *name = "event"; + + switch (data[0]) { + case HCI_EV_CONN_REQUEST: + name = "connection request"; + offset = HCI_EVENT_HDR_SIZE; + break; + case HCI_EV_CONN_COMPLETE: + name = "connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_SYNC_CONN_COMPLETE: + name = "synchronous connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_HARDWARE_ERROR: + name = "hardware error"; + break; + case HCI_EV_LE_META: + if (len < HCI_EVENT_HDR_SIZE + 1) + break; + /* Subevent, report count, event type (two bytes if extended), address type. */ + switch (data[HCI_EVENT_HDR_SIZE]) { + case HCI_EV_LE_ADVERTISING_REPORT: + name = "LE advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_DIRECT_ADV_REPORT: + name = "LE directed advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_EXT_ADV_REPORT: + name = "LE extended advertising report"; + offset = HCI_EVENT_HDR_SIZE + 5; + break; + } + break; + } + *addr = offset && len >= offset + sizeof(bdaddr_t) ? data + offset : NULL; + return name; +} + +/* Caller owns lock and expects the wake reason. */ +static void bt7932_log_wake(struct bt7932 *bt, const u8 *data, size_t len) +{ + const u8 *addr; + const char *name = bt7932_wake_name(data, len, &addr); + + if (addr) + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s from %pMR\n", name, addr); + else + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s\n", name); + bt->wake_expected = false; + wake_up_all(&bt->wake_wait); +} + static int bt7932_validate_ptx(const u8 *data, size_t size) { static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; @@ -727,8 +795,11 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) * remember that it arrived. */ if (geo->packet == HCI_EVENT_PKT && bt->wake_tracking && - bt7932_wake_report(bt, descriptor + 16, length)) + bt7932_wake_report(bt, descriptor + 16, length)) { + if (bt->wake_expected) + bt7932_log_wake(bt, descriptor + 16, length); pm_wakeup_dev_event(&bt->pdev->dev, 0, true); + } hci_recv_frame(bt->hdev, skb); } slot = (slot + 1) & BT7932_MASK; @@ -745,9 +816,9 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) return 0; } -static irqreturn_t bt7932_irq_thread(int irq, void *data) +/* Takes what the firmware published: sleep notifications, completions, packets. */ +static void bt7932_service(struct bt7932 *bt) { - struct bt7932 *bt = data; u16 producer; int channel, ret; @@ -787,6 +858,11 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) queue_work(bt->tx_wq, &bt->tx_work); out: mutex_unlock(&bt->lock); +} + +static irqreturn_t bt7932_irq_thread(int irq, void *data) +{ + bt7932_service(data); return IRQ_HANDLED; } @@ -1504,6 +1580,8 @@ static void bt7932_track_wake(struct bt7932 *bt, bool on) { mutex_lock(&bt->lock); bt->wake_tracking = on; + if (!on) + bt->wake_expected = false; mutex_unlock(&bt->lock); } @@ -1587,6 +1665,7 @@ static int bt7932_arm_wakeup(struct bt7932 *bt) ret = bt->fault ? -EIO : 0; if (!ret) { bt->wake_armed = true; + bt->wake_expected = true; /* A failed wake handshake would fault the transport. */ bt->tx_blocked = true; } @@ -1611,7 +1690,6 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) mutex_lock(&bt->lock); bt->wake_armed = false; bt->tx_blocked = false; - bt->wake_tracking = false; bt->pci_saved = false; mutex_unlock(&bt->lock); wake_up_all(&bt->wake_wait); @@ -1621,6 +1699,28 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) } } +/* + * After a sleep set up for Bluetooth wake, the controller delivers what woke + * the host once the transport runs again. Take it while HCI is still + * suspended, so that the core records the wake reason too, and log it. + * Ends wake tracking. + */ +static void bt7932_take_wake_reason(struct bt7932 *bt) +{ + /* The interrupt thread leaves the rings alone until the restore ends. */ + bt7932_service(bt); + wait_event_timeout(bt->wake_wait, + !READ_ONCE(bt->wake_expected) || READ_ONCE(bt->fault), + msecs_to_jiffies(BT7932_WAKE_REASON_MS)); + mutex_lock(&bt->lock); + if (bt->wake_expected && !bt->fault) + dev_dbg(&bt->pdev->dev, "Bluetooth did not wake the host\n"); + mutex_unlock(&bt->lock); + bt7932_track_wake(bt, false); + /* The core takes a wake reason only until hci_resume_dev(). */ + flush_work(&bt->hdev->rx_work); +} + /* * With wakeup enabled, the transport stays up through suspend-to-idle * instead of being quiesced: @@ -1646,7 +1746,7 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) * up its wake scans. The quiesce (sleep control 3, notification 3) is the * host sleep that macOS enters at S2R. The controller keeps scanning and * wakes the system through the SMC. bt7932_resume() ends the host sleep - * with control 0, as macOS does. + * with control 0, as macOS does, and then takes the wake reason. */ static int bt7932_suspend(struct device *dev) { @@ -1708,6 +1808,10 @@ static int bt7932_suspend(struct device *dev) dev_warn(dev, "no Bluetooth wake for this sleep\n"); return 0; } + mutex_lock(&bt->lock); + bt->wake_quiesced = true; + bt->wake_expected = true; + mutex_unlock(&bt->lock); dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } @@ -1760,8 +1864,10 @@ static int bt7932_resume_noirq(struct device *dev) static int bt7932_resume(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + bool wake_quiesced = bt->wake_quiesced; int ret; + bt->wake_quiesced = false; /* A deep sleep retrains the link. With the chip gone, nothing here may touch it. */ if (!pci_device_is_present(bt->pdev)) { bt7932_track_wake(bt, false); @@ -1774,6 +1880,7 @@ static int bt7932_resume(struct device *dev) if (bt->wake_armed) { bt7932_disarm_wakeup(bt); + bt7932_take_wake_reason(bt); if (READ_ONCE(bt->fault)) return -EIO; return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; @@ -1792,8 +1899,8 @@ static int bt7932_resume(struct device *dev) return ret; } } - /* Reports now are no longer wake events. */ - bt7932_track_wake(bt, false); + if (wake_quiesced) + bt7932_take_wake_reason(bt); dev_dbg(dev, "transport restored after system sleep\n"); return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } diff --git a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c index 61e93ce1f31db..8a57f9f30ca1a 100644 --- a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c +++ b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c @@ -65,11 +65,57 @@ static void bt7932_wake_event_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, bt7932_wake_event(error, sizeof(error)), BT7932_WAKE_ERROR); } +/* The wake log names the event and the remote address it carries. */ +static void bt7932_wake_name_bredr_test(struct kunit *test) +{ + static const u8 request[] = { HCI_EV_CONN_REQUEST, 10, + 1, 2, 3, 4, 5, 6, 0x40, 0x25, 0x00, 0x01 }; + static const u8 complete[] = { HCI_EV_CONN_COMPLETE, 11, 0x00, 0x01, 0x00, + 1, 2, 3, 4, 5, 6, 0x01, 0x00 }; + static const u8 sync_short[] = { HCI_EV_SYNC_CONN_COMPLETE, 1, 0x00 }; + static const u8 error[] = { HCI_EV_HARDWARE_ERROR, 1, 0x00 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(request, sizeof(request), &addr), + "connection request"); + KUNIT_EXPECT_PTR_EQ(test, addr, &request[2]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(complete, sizeof(complete), &addr), + "connection complete"); + KUNIT_EXPECT_PTR_EQ(test, addr, &complete[5]); + bt7932_wake_name(sync_short, sizeof(sync_short), &addr); + KUNIT_EXPECT_NULL(test, addr); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(error, sizeof(error), &addr), "hardware error"); + KUNIT_EXPECT_NULL(test, addr); +} + +static void bt7932_wake_name_le_test(struct kunit *test) +{ + /* Subevent, one report, event type, address type, address, data length, RSSI. */ + static const u8 adv[] = { HCI_EV_LE_META, 12, HCI_EV_LE_ADVERTISING_REPORT, 1, + 0x00, 0x01, 1, 2, 3, 4, 5, 6, 0x00, 0xc0 }; + /* The extended report's event type takes two bytes. */ + static const u8 ext[] = { HCI_EV_LE_META, 11, HCI_EV_LE_EXT_ADV_REPORT, 1, + 0x13, 0x00, 0x01, 1, 2, 3, 4, 5, 6 }; + static const u8 adv_short[] = { HCI_EV_LE_META, 2, HCI_EV_LE_ADVERTISING_REPORT, 1 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(adv, sizeof(adv), &addr), + "LE advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &adv[6]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(ext, sizeof(ext), &addr), + "LE extended advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &ext[7]); + bt7932_wake_name(adv_short, sizeof(adv_short), &addr); + KUNIT_EXPECT_NULL(test, addr); +} + static struct kunit_case bt7932_wake_test_cases[] = { KUNIT_CASE(bt7932_wake_event_bredr_test), KUNIT_CASE(bt7932_wake_event_le_test), KUNIT_CASE(bt7932_wake_event_replies_test), KUNIT_CASE(bt7932_wake_event_error_test), + KUNIT_CASE(bt7932_wake_name_bredr_test), + KUNIT_CASE(bt7932_wake_name_le_test), {} }; From c80e6eb2829962b79d26c0ccf5f25db2fd679eeb Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:05:43 -0500 Subject: [PATCH 46/73] usb: core: hcd: don't release root hub PHYs twice after a failed resume A system suspend of a root hub powers its PHYs off, and exits them when the controller can't wake the system. hcd_bus_resume() takes them back before the bus resume, and releases them again if that fails. When the HCD is then removed, usb_remove_hcd() releases them a third time, and the phy core takes the power count below zero. The J700's DWC3 glue keeps xHCI away from a core that came back from a deep sleep without its registers, so the bus resume fails with -ESHUTDOWN and the glue restarts the host. The USB3 PHY's count then ends one short. The restarted host's own root hub suspend at the next sleep takes it to zero, and the PHY's power_off turns the SuperSpeed side off under the sleeping host, which refused that sleep ("USB changed for the sleep (mode 0, PIPE 0, host 0): refusing it", measured on a J700). Nothing logs the count itself, so that it is what the refusal hit is inferred from the code: the hcd_bus_resume() failure path and usb_remove_hcd() both release the PHYs, and phy_power_off() doesn't check for zero. Track with a flag whether a system suspend released the PHYs. Resume takes them back only then, a failed resume releases only what it took (an autoresume takes nothing), a suspend after a raced autoresume doesn't release them again, and removal releases only what is still held. Signed-off-by: Diljit Singh --- drivers/usb/core/hcd.c | 25 ++++++++++++++++++++----- include/linux/usb/hcd.h | 1 + 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 8e8a5f59b319b..0ea6a9a729bc5 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -2280,7 +2280,9 @@ int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg) usb_set_device_state(rhdev, USB_STATE_SUSPENDED); hcd->state = HC_STATE_SUSPENDED; - if (!PMSG_IS_AUTO(msg)) + /* A suspend after a raced autoresume finds them released already. */ + if (!PMSG_IS_AUTO(msg) && + !test_and_set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); @@ -2321,11 +2323,12 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) return 0; } - if (!PMSG_IS_AUTO(msg)) { + if (!PMSG_IS_AUTO(msg) && test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { status = usb_phy_roothub_resume(hcd->self.sysdev, hcd->phy_roothub); if (status) return status; + clear_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); } if (!hcd->driver->bus_resume) @@ -2368,7 +2371,11 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) } } else { hcd->state = old_state; - usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + /* Only release what this resume took: an autoresume took nothing. */ + if (!PMSG_IS_AUTO(msg)) { + usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); + } dev_dbg(&rhdev->dev, "bus %s fail, err %d\n", "resume", status); if (status != -ESHUTDOWN) @@ -3084,8 +3091,16 @@ void usb_remove_hcd(struct usb_hcd *hcd) usb_deregister_bus(&hcd->self); hcd_buffer_destroy(hcd); - usb_phy_roothub_power_off(hcd->phy_roothub); - usb_phy_roothub_exit(hcd->phy_roothub); + /* + * A root hub whose resume failed still has its PHYs released by the suspend, + * powered off and, without wakeup, exited too. Don't release them twice. + */ + if (!test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { + usb_phy_roothub_power_off(hcd->phy_roothub); + usb_phy_roothub_exit(hcd->phy_roothub); + } else if (device_may_wakeup(hcd->self.sysdev)) { + usb_phy_roothub_exit(hcd->phy_roothub); + } usb_put_invalidate_rhdev(hcd); hcd->flags = 0; diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 604b9dda94317..b8bf424370bc7 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -112,6 +112,7 @@ struct usb_hcd { #define HCD_FLAG_DEAD 6 /* controller has died? */ #define HCD_FLAG_INTF_AUTHORIZED 7 /* authorize interfaces? */ #define HCD_FLAG_DEFER_RH_REGISTER 8 /* Defer roothub registration */ +#define HCD_FLAG_PHY_SUSPENDED 9 /* root hub PHYs released by a suspend */ /* The flags can be tested using these macros; they are likely to * be slightly faster than test_bit(). From 047beb14f9263b940965f829e5c018fb2587edb8 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:05:44 -0500 Subject: [PATCH 47/73] usb: xhci-plat: let a resume quirk say the controller lost its state A platform's resume quirk may find that the controller lost power in the sleep and set its glue up again. xhci_resume() then has to start the controller over instead of restoring a state the controller no longer has. Take a positive return from the quirk for that, as the hibernation restore's power_lost does. The existing quirks (rcar, rzg3e, cdns3 and dwc3) return zero or a negative error. Signed-off-by: Diljit Singh --- drivers/usb/host/xhci-plat.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index dd1e3ef800e31..9fb820269d768 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -650,11 +650,12 @@ static int xhci_plat_resume_common(struct device *dev, bool power_lost) } } + /* A positive result: the controller lost its state, which the quirk set up again. */ ret = xhci_priv_resume_quirk(hcd); - if (ret) + if (ret < 0) goto disable_clks; - ret = xhci_resume(xhci, power_lost || priv->power_lost, false); + ret = xhci_resume(xhci, power_lost || priv->power_lost || ret > 0, false); if (ret) goto disable_clks; From ee5b0b1e5ab5a98cd23d886aeee39787829f05e7 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:22 -0500 Subject: [PATCH 48/73] usb: apple: stop the USB controller and port controllers for a deep sleep A deep sleep powers off the controller's domain, and the sleep engine needs ATC0_COMMON off, so keeping the core up as a syscore device can't work: the suspend is refused, and it would find a dead core if it weren't. xHCI can't resume across that either. dwc3-apple already knows how to stop everything and start again, for a cable disconnect. Do the same around a deep sleep: stop before the freezer runs and start once the system is back, when firmware for what's behind the hub can be found. USB devices come back as new devices, so a mounted USB disk doesn't survive this. The wakeup setting is turned off for the sleep, as it would keep the domain on, and a role that arrives during the sleep is applied afterwards. A controller is always stopped. macOS sends the port controllers SSPS with state 3 before a sleep. tipd does that where the tree asks for it with apple,sleep-ssps, and goes back to S0 after. A controller that doesn't take it only logs. The sleep controller ends the debug link with a cold reset of its port controller, which leaves that firmware's interrupt mask and power state at their defaults. tipd sets the controller up again when the link was cut, and rearms the link where the tree gives the board's key in apple,debug-lock-key, as a boot loader does: unlock, debug mode on, enable the debug VDM, debug mode off, lock. A failure only logs. The ATC PHY forgets what its DisplayPort side owned or was stopping once its domain has been off, since none of that survives. Signed-off-by: Diljit Singh --- drivers/phy/apple/atc.c | 10 +++ drivers/usb/dwc3/dwc3-apple-state-test.c | 9 +++ drivers/usb/dwc3/dwc3-apple-state.h | 19 +++++ drivers/usb/dwc3/dwc3-apple.c | 79 ++++++++++++++++++- drivers/usb/typec/tipd/core.c | 56 +++++++++++++ .../drivers/gpu/apple-dcp/dwc3/mocks.c | 3 + .../drivers/gpu/apple-dcp/lifecycle.py | 3 + .../drivers/gpu/apple-dcp/teardown_pm.py | 2 +- 8 files changed, 179 insertions(+), 2 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 6ed4bc2295d7b..d341c20a7c2f9 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4200,6 +4200,15 @@ static void atcphy_restore_after_pd_off(struct apple_atcphy *atcphy) atcphy->tunnel_clock_on = false; atcphy->tunnel_rate = 0; atcphy->tunnel_users = 0; + /* Nothing the DisplayPort side owned or was stopping survived the domain. */ + if (atcphy->hw->has_t8140_dp) { + atcphy->dp_owned_pmas = 0; + atcphy->dp_idle_pmas = 0; + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->aux_power_requested = false; + atcphy->mode_dirty = false; + } if (!atcphy->dp_only) { _atcphy_dwc3_reset_assert(atcphy); @@ -6557,6 +6566,7 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .has_usb2phy_reg = true, .has_t8140_dp = true, .optional_tunables = true, + .restore_after_pd_off = true, }; static const struct atcphy_hw atcphy_hw_t8142 = { diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index 7a04604ddd4a1..69abb0f32653f 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -38,10 +38,19 @@ static void dwc3_apple_non_role_test(struct kunit *test) } } +/* A role that arrives during a sleep decides what comes back after it. */ +static void dwc3_apple_role_during_sleep_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST), DWC3_APPLE_HOST); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE), DWC3_APPLE_DEVICE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE), DWC3_APPLE_NO_CABLE); +} + static struct kunit_case dwc3_apple_state_test_cases[] = { KUNIT_CASE(dwc3_apple_host_left_to_xhci_test), KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), KUNIT_CASE(dwc3_apple_non_role_test), + KUNIT_CASE(dwc3_apple_role_during_sleep_test), {} }; diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index 42c13d900428e..bb8d827fdec4c 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -9,6 +9,7 @@ #include #include +#include /* See the comment at the top of dwc3-apple.c for how these states are used. */ enum dwc3_apple_state { @@ -47,4 +48,22 @@ static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target } } +/** + * dwc3_apple_state_for_role() - state a role asks for while the core is stopped for sleep + * @role: the role asked for + * + * Return: the state to bring the controller up in after the sleep. + */ +static inline enum dwc3_apple_state dwc3_apple_state_for_role(enum usb_role role) +{ + switch (role) { + case USB_ROLE_HOST: + return DWC3_APPLE_HOST; + case USB_ROLE_DEVICE: + return DWC3_APPLE_DEVICE; + default: + return DWC3_APPLE_NO_CABLE; + } +} + #endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f16aa6f21014b..c7b49e48f302c 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dwc3-apple-state.h" @@ -91,6 +92,10 @@ * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device * may wake the system, as a halted xHC could not signal the wakeup. * + * A deep sleep takes the controller's power domain away, so nothing can be kept running across + * it. The glue stops everything before the freezer runs, the way a cable disconnect does, and + * brings it back once the system has resumed. See dwc3_apple_pm_notify(). + * * Runtime PM (T6040 and T8132, see dwc3_apple_core_probe()): the glue holds a reference from * the reset deassert to the end of the shutdown, so with no cable the controller can * runtime-suspend and its power domain drop, its DARTs (which save and restore their state) and, @@ -157,6 +162,11 @@ struct dwc3_apple { bool rpm_held; bool pre_core_rpm; bool dt_dis_u3_susphy; + struct notifier_block pm_nb; + /* Stopped for a deep sleep: the state to come back to, and the wakeup setting. */ + bool sleep_stopped; + bool sleep_wakeup; + enum dwc3_apple_state sleep_state; void __iomem *drd_cfg; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; @@ -766,6 +776,12 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role ro if (appledwc->fixed_hub) return 0; + /* Stopped for a sleep: this is what comes back after it. */ + if (appledwc->sleep_stopped) { + appledwc->sleep_state = dwc3_apple_state_for_role(role); + return 0; + } + /* * Stay off instead of starting device mode (see dwc3_apple_host_only()). Reporting success * keeps the port controller from retrying: get() then reports USB_ROLE_NONE. @@ -820,7 +836,7 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) guard(mutex)(&appledwc->lock); - switch (appledwc->state) { + switch (appledwc->sleep_stopped ? appledwc->sleep_state : appledwc->state) { case DWC3_APPLE_HOST: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: @@ -850,6 +866,59 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } +/* + * Runs before the freezer, when the sleep's real target isn't known yet, so this goes by the + * state "mem" means now. Writing "freeze" with deep selected costs a USB restart and nothing else. + * A stop that fails refuses the sleep. PM_POST_SUSPEND follows an abandoned sleep as well. + */ +static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct dwc3_apple *appledwc = container_of(nb, struct dwc3_apple, pm_nb); + struct device *dev = appledwc->dev; + int ret; + + guard(mutex)(&appledwc->lock); + + switch (action) { + case PM_SUSPEND_PREPARE: + if (pm_suspend_default_s2idle() || appledwc->shutting_down) + break; + appledwc->sleep_state = appledwc->state; + ret = dwc3_apple_exit(appledwc); + if (ret) + return notifier_from_errno(ret); + /* A wakeup path would keep the domain on, and it's going away. */ + appledwc->sleep_wakeup = device_may_wakeup(dev); + device_set_wakeup_enable(dev, false); + appledwc->sleep_stopped = true; + break; + case PM_POST_SUSPEND: + if (!appledwc->sleep_stopped) + break; + appledwc->sleep_stopped = false; + device_set_wakeup_enable(dev, appledwc->sleep_wakeup); + if (appledwc->shutting_down || + (appledwc->sleep_state != DWC3_APPLE_HOST && + appledwc->sleep_state != DWC3_APPLE_DEVICE)) + break; + ret = dwc3_apple_init(appledwc, appledwc->sleep_state); + if (ret) + dev_err(dev, "Failed to start after sleep, err=%d\n", ret); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +static void dwc3_apple_pm_notifier_remove(void *data) +{ + struct dwc3_apple *appledwc = data; + + unregister_pm_notifier(&appledwc->pm_nb); +} + static bool dwc3_apple_has_fixed_hub(struct device *dev) { struct device_node *hub __free(device_node) = @@ -940,6 +1009,13 @@ static int dwc3_apple_probe(struct platform_device *pdev) * details. */ appledwc->state = DWC3_APPLE_PROBE_PENDING; + appledwc->pm_nb.notifier_call = dwc3_apple_pm_notify; + ret = register_pm_notifier(&appledwc->pm_nb); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dwc3_apple_pm_notifier_remove, appledwc); + if (ret) + return ret; /* * Until the first cable there is no core to keep powered: let the controller * runtime-suspend, and its DARTs with it, so that its power domain can drop. @@ -989,6 +1065,7 @@ static void dwc3_apple_remove(struct platform_device *pdev) guard(mutex)(&appledwc->lock); + appledwc->shutting_down = true; usb_role_switch_unregister(appledwc->role_sw); /* diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 74c820dd0be24..05812642c84e7 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -1577,6 +1579,51 @@ static void cd321x_prepare(struct tps6598x *tps) /* Gate scheduling first; neither worker may be drained under its mutex. */ cancel_delayed_work_sync(&cd321x->resume_work); cancel_delayed_work_sync(&cd321x->update_work); + + /* As macOS does a second or so before the commit, measured on the same controller. */ + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) { + int ret; + + guard(mutex)(&tps->lock); + ret = tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S3); + if (ret) + dev_warn(tps->dev, "didn't take system power state S3: %d\n", ret); + else + dev_dbg(tps->dev, "system power state S3\n"); + } +} + +/* + * A sleep cut this port's debug link. Rearm it the way a boot loader does: unlock with + * the board's key, debug mode on, enable the debug VDM, debug mode off, lock. + */ +static void cd321x_rearm_debug_link(struct tps6598x *tps) +{ + static const u8 vdm[] = { 0x06, 0x46, 0x82, 0x01 }; + static const u8 no_key[4]; + const char *key; + u8 on = 1, off = 0, out = 0; + int ret; + + if (device_property_read_string(tps->dev, "apple,debug-lock-key", &key) || + strlen(key) != sizeof(no_key) || !apple_sleep_debug_link_was_cut(tps->dev)) + return; + + ret = tps6598x_exec_cmd(tps, "LOCK", sizeof(no_key), (const u8 *)key, 1, &out); + if (!ret && !(out & 0xf)) + ret = tps6598x_exec_cmd(tps, "DBMa", 1, &on, 1, &out); + if (!ret && !(out & 0xf)) + ret = tps6598x_exec_cmd(tps, "DVEn", sizeof(vdm), vdm, 1, &out); + if (!ret && (out & 0xf)) + ret = -EIO; + if (!ret) + ret = tps6598x_exec_cmd(tps, "DBMa", 1, &off, 0, NULL); + if (!ret) + ret = tps6598x_exec_cmd(tps, "LOCK", sizeof(no_key), no_key, 0, NULL); + if (ret) + dev_warn(tps->dev, "debug link not rearmed: %d\n", ret); + else + dev_dbg(tps->dev, "debug link rearmed\n"); } static void cd321x_complete(struct tps6598x *tps) @@ -1584,6 +1631,15 @@ static void cd321x_complete(struct tps6598x *tps) struct cd321x *cd321x = container_of(tps, struct cd321x, tps); guard(mutex)(&tps->lock); + /* + * The cut was a cold reset of this controller's firmware: its interrupt mask and power + * state are back at their defaults, so set them up again whether or not an event comes. + */ + if (apple_sleep_debug_link_cut_peek(tps->dev)) + cd321x->pm.setup_pending = true; + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + cd321x_rearm_debug_link(tps); if (cd321x_pm_resume(&cd321x->pm)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 806ec38dd7fd4..a53a29f32999f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -85,6 +85,9 @@ struct dwc3_apple { void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending, shutting_down; + /* Stopped for a deep sleep, and the state to bring back. */ + bool sleep_stopped; + enum dwc3_apple_state sleep_state; bool rpm_held, host_only; /* T8132 only: runtime PM before the first core probe; never set here. */ bool pre_core_rpm; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index f42f8251fee56..2db0e2b87e8f9 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1068,6 +1068,9 @@ def fn(n, name): for name in ('atcphy_mode_configuration', 'atcphy_hw', 'atcphy_dp_link_rate_configuration'): decls.append(block(files['atc'], files['atc'].index('struct ' + name + ' {')) + ';') decls.append(block(files['state'], files['state'].index('enum dwc3_apple_state {')) + ';') + role = 'static inline enum dwc3_apple_state dwc3_apple_state_for_role(' + if role in files['state']: + decls.append(block(files['state'], files['state'].index(role))) start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') end = files['atc'].index('\n};', start) + 3 decls.append(files['atc'][start:end]) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index b293087d4917c..c208cd9587224 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -95,7 +95,7 @@ def unit_for(source): struct dwc3 { bool sys_wakeup, dis_u3_susphy_quirk; struct platform_device *xhci; }; enum { DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_HOST, DWC3_APPLE_DEVICE }; /* pre_core_rpm is T8132 only: runtime PM before the first core probe; never set here. */ -struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy; }; +struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy, shutting_down; }; static struct dwc3_apple apple_context; #define to_dwc3_apple(d) container_of(d,struct dwc3_apple,dwc) static bool inside_host_remove; From 3317baeffb29dbc52a01d70f030818591f7e7af6 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:22 -0500 Subject: [PATCH 49/73] phy: apple: atc: take the USB controller through hibernation for a sleep Before a deep sleep the controller stopped as for a cable disconnect, and every USB device was gone after the wake. macOS's host goes to Sleep, not Off, and the hub and its devices keep their slots: - xHCI halts and saves its state. - The controller's power control takes the core into hibernation: the D3 request, wait for VVDD_USB_OFF, clear it, arm the two wake events, wait for SLEEP. - The core's reset is held without the clamp, which only its Off path sets. - The eUSB2 PHY goes to its sleep state, reference clock gated and APB in reset. - ATC0_USB and ATC0_COMMON go off. ATC0_USB_AON stays on. - At the wake: the eUSB2 PHY out of its sleep state, D0 requested, the reset released, wait for D0_ON, clear the events. xHCI restores its state and the root port resumes. Do that from the PHY's noirq callbacks. The PHY already drives this controller's reset in the PIPE handler, owns the USB2 PHY, and suspends after the controller as its supplier, so xHCI has saved its state by then. ATC0_COMMON takes the common block's setup with it, so the block is set up again in the mode it had, USB2 or USB3, without the controller's reset or a USB2 PHY power cycle, before the controller's D0; a USB3 setup that fails there falls back to USB2. The secondary eUSB2 bank, next to the controller and on a rail that isn't known, gets its defaults only after D0. The sleep controller is told to keep ATC0_USB_AON for that sleep. The SuperSpeed lanes stay as they are, as macOS leaves them: when its T8140 host goes to Sleep, setUSB3Mode locks the TX-detect/RX state in the power control and the PIPE interface in the PIPE handler, two handshakes whose timeouts it only logs, and puts the USB3 PHY in its sleep power level (25G83, read statically). Here USB3 sleeps in place like USB2: at noirq, with xHCI halted, the PIPE is parked on the dummy backend just before the hibernation, and a lock the halted controller doesn't acknowledge is logged and the PIPE parked unlocked, as for a stopped controller. The commit takes the lanes down, and the wake sets them up in USB3 again with the PIPE still parked. The controller's glue routes the PIPE to them when it sets the core up, before xHCI starts, so no Type-C mode change and no SuperSpeed port power cycle follow. On macOS too, the raise from Sleep to the saved USB3 mode is the TX-detect/RX lock handshake and the USB3 PHY's power level (setUSB3Mode, read statically). The controller's glue asks for all this with apple_atcphy_sleep_prepare() before the freezer; apple_atcphy_sleep_result() says how the wake went, and apple_atcphy_sleep_finish() checks the SuperSpeed side against the mux afterwards. A mux update runs then only if the mux changed during the sleep, the core kept its registers, or the setup at the wake failed. DisplayPort on the port leaves the controller stopping for the sleep as before. A mode change in between refuses the sleep. A hibernation that doesn't complete refuses that sleep, and only such failures of the PHY's own make the controller stop for the later sleeps: a host its glue set up again or restarted after a sleep hibernates again at the next one. The block's always-on side can lose its power in the sleep anyway: its power control then reads all zeros at the wake and never raises D0_ON. The wake doesn't wait for it then, and redoes what the reset's release and the probe do for the rest of the block: the core's clamp released and the USB2 PHY's host role set. Both are no-ops for a block that kept its state. A mux update decides it has work to do before it takes the USB locks, and an update holds them for the whole of its change. One that waited behind another for the same mode took the SuperSpeed port down and up again and ran its own retrain. The port controller's revalidation after a wake did that behind the glue's SuperSpeed restore, and held a quick sleep's freezer (measured on a J700). An update now looks again once it holds the locks, and leaves the port alone if the other one finished the change. A KUnit case covers it. A stopped T8140 controller no longer clocks the PIPE lock's acknowledgment, so a stopped controller's PIPE is parked without the lock, as on T8103 and T8122. Measured on a J700: the block loses its always-on power in every sleep so far, so the core comes back with its registers reset and its glue sets it up again. Taking USB3 to USB2 and back for the sleep, as this did first, cost 0.6 s of every sleep's entry, and waiting for a D0_ON that couldn't come cost 200 ms of every wake. Setting the block up in USB2 at the wake and bringing USB3 back with a whole mode change afterwards took 1.8 s after every wake, most of it two retrains waiting for a SuperSpeed link that wasn't there. With the mode set up again at the wake, the SuperSpeed side is already in its mode once the system is up. Signed-off-by: Diljit Singh --- drivers/phy/apple/atc-test.c | 35 ++ drivers/phy/apple/atc.c | 445 +++++++++++++++++- include/linux/phy/apple-atc.h | 52 ++ .../selftests/drivers/gpu/apple-dcp/common.py | 9 + .../drivers/gpu/apple-dcp/lifecycle.py | 2 +- .../selftests/drivers/gpu/apple-dcp/phy.py | 3 +- .../selftests/drivers/gpu/apple-dcp/pm.py | 2 + 7 files changed, 524 insertions(+), 24 deletions(-) create mode 100644 include/linux/phy/apple-atc.h diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 36e5afed0227f..09eec7f9d6185 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -564,6 +564,33 @@ static void atcphy_test_update_port_taken(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); } +/* The root hub, once another update that held it has brought the port up again */ +static int atcphy_test_pm_get_after_other(struct apple_atcphy *atcphy, + struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('g'); + guard(mutex)(&atcphy->lock); + atcphy->usb3_port_off = false; + return 0; +} + +/* An update that waited for another one doing the same change leaves the port alone */ +static void atcphy_test_update_after_other(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get_after_other); + steps->mode_port_on = true; + + /* The lanes and the PIPE are in place, the port still owed by the other update */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3_DP); +} + /* * Zeroed memory in place of the core, PIPE handler and USB2 PHY registers: * enough to park the PIPE and power the PHY off, with a PIPE lock request @@ -641,18 +668,25 @@ static struct apple_atcphy *atcphy_test_alloc_failed_handoff(struct kunit *test) return atcphy; } +static int atcphy_test_count_refused_lock(struct apple_atcphy *atcphy); + /* Once dwc3 has stopped, its USB3 PHY power-off parks the PIPE without one */ static void atcphy_test_stopped_power_off_common(struct kunit *test, bool dp) { struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + unsigned int requests = 0; KUNIT_ASSERT_NOT_NULL(test, phy); phy_set_drvdata(phy, atcphy); if (!dp) atcphy->mode = APPLE_ATCPHY_MODE_USB3; + /* The stopped controller can't acknowledge a lock, so none is asked for. */ + test->priv = &requests; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, atcphy_test_count_refused_lock); KUNIT_EXPECT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); KUNIT_EXPECT_EQ(test, atcphy->mode, @@ -1368,6 +1402,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_retry), KUNIT_CASE(atcphy_test_update_unsupported), KUNIT_CASE(atcphy_test_update_port_taken), + KUNIT_CASE(atcphy_test_update_after_other), KUNIT_CASE(atcphy_test_set_port_on), KUNIT_CASE(atcphy_test_host_start), KUNIT_CASE(atcphy_test_stopped_power_off), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d341c20a7c2f9..1ea2d57ee8b8e 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -558,6 +560,24 @@ enum atcphy_lane_mode { #define USB2PHY_MISCTUNE_APBCLK_GATE_OFF BIT(29) #define USB2PHY_MISCTUNE_REFCLK_GATE_OFF BIT(30) +/* + * The power control of the USB controller the T8140 PHY serves ("pwrctl"). macOS's + * controller driver takes the core into hibernation with it at a system sleep and back + * at the wake, in the sequences below. + */ +#define AUSBC_PWR_CFG 0x00 +#define AUSBC_PWR_CFG_STATE_REQ GENMASK(5, 4) +#define AUSBC_PWR_STATUS 0x04 +#define AUSBC_PWR_STATUS_SLEEP BIT(20) +#define AUSBC_PWR_EVENTS 0x08 +#define AUSBC_PWR_ENABLE 0x10 +#define AUSBC_PWR_CLEAR 0x14 +#define AUSBC_PWR_EVENT_MASK GENMASK(3, 0) +#define AUSBC_PWR_EVENT_D0_ON BIT(0) +#define AUSBC_PWR_EVENT_VDD_OFF BIT(1) +#define AUSBC_PWR_EVENT_WAKE GENMASK(3, 2) +#define AUSBC_PWR_TIMEOUT_US 200000 + enum atcphy_generation { ATCPHY_GENERATION_T8103, ATCPHY_GENERATION_T8122, @@ -858,6 +878,16 @@ struct apple_atcphy { bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; + /* + * The controller sleeps in place for this system sleep, the SuperSpeed side is checked + * against the mux after it, and the controller is in hibernation now. + */ + bool sleep_in_place; + bool sleep_ss_down; + bool hibernated; + int sleep_ret; + /* Going through hibernation failed on this boot: the controller stops from now on. */ + bool sleep_no_keep; bool usb3_port_off; struct delayed_work usb3_port_work; unsigned int usb3_port_retries; @@ -898,6 +928,7 @@ struct apple_atcphy { void __iomem *usb2phy_reg; void __iomem *pipehandler; void __iomem *lpdptx; + void __iomem *pwrctl; } regs; struct { @@ -2662,9 +2693,11 @@ static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) /* * The T8140 keeps per-device defaults for its secondary eUSB2 register * bank in the ADT. The power-off path asserts the PHY resets, so apply - * them after every power-on. + * them after every power-on, but not with the controller's core in + * hibernation: see atcphy_resume_noirq(). */ - apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + if (!atcphy->hibernated) + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); } static int atcphy_power_on(struct apple_atcphy *atcphy) @@ -4158,6 +4191,26 @@ static void atcphy_rpm_release(void *data) atcphy->rpm_held = false; } +/* What a power-down of the common block takes: the DisplayPort side and a tunnel clock. */ +static void atcphy_forget_common_state(struct apple_atcphy *atcphy) +{ + atcphy->dp_t8122.aux = false; + atcphy->dp_t8122.pll = false; + atcphy->dp_t8122.pairs = 0; + atcphy->dp_t8122.rate = 0; + atcphy->tunnel_clock_on = false; + atcphy->tunnel_rate = 0; + atcphy->tunnel_users = 0; + if (atcphy->hw->has_t8140_dp) { + atcphy->dp_owned_pmas = 0; + atcphy->dp_idle_pmas = 0; + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->aux_power_requested = false; + atcphy->mode_dirty = false; + } +} + /* * A T8122 PHY loses its register state whenever its power domain is switched * off. In system sleep that happens to a domain that no running device holds: @@ -4193,22 +4246,7 @@ static void atcphy_restore_after_pd_off(struct apple_atcphy *atcphy) return; WRITE_ONCE(atcphy->pd_was_off, false); - atcphy->dp_t8122.aux = false; - atcphy->dp_t8122.pll = false; - atcphy->dp_t8122.pairs = 0; - atcphy->dp_t8122.rate = 0; - atcphy->tunnel_clock_on = false; - atcphy->tunnel_rate = 0; - atcphy->tunnel_users = 0; - /* Nothing the DisplayPort side owned or was stopping survived the domain. */ - if (atcphy->hw->has_t8140_dp) { - atcphy->dp_owned_pmas = 0; - atcphy->dp_idle_pmas = 0; - atcphy->dp_usb_context_valid = 0; - atcphy->dp_usb_context_dirty = 0; - atcphy->aux_power_requested = false; - atcphy->mode_dirty = false; - } + atcphy_forget_common_state(atcphy); if (!atcphy->dp_only) { _atcphy_dwc3_reset_assert(atcphy); @@ -5689,16 +5727,24 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return ret; } -static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +/* Whether an update to @target_mode has nothing left to do, recording the request if so */ +static bool atcphy_mux_done(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + guard(mutex)(&atcphy->lock); + if (!atcphy_mux_is_noop(atcphy, target_mode)) + return false; + atcphy->typec_mode = target_mode; + return true; +} + +static int atcphy_mux_update_to(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { struct atcphy_usb3_pm pm = {}; - enum atcphy_mode target_mode; bool in_mode, port_on, usb_err = true; int ret; scoped_guard(mutex, &atcphy->lock) { atcphy_restore_after_pd_off(atcphy); - target_mode = atcphy_typec_target_mode(atcphy, state); /* * Only a lane or PIPE change, or a SuperSpeed port still owed * its power, needs the USB3 root hub below. Taking it means @@ -5723,6 +5769,13 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux if (ret) goto out; + /* + * Another update holds the root hub for the whole of its change. One that finished + * while this one waited may have left nothing to do: don't cycle the port again. + */ + if (atcphy_mux_done(atcphy, target_mode)) + goto put; + /* * The order in which dwc3 is taken down and brought up around a PHY * change (see dwc3-apple.c), for the SuperSpeed side only: the @@ -5774,6 +5827,11 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux return ret; } +static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +{ + return atcphy_mux_update_to(atcphy, atcphy_typec_target_mode(atcphy, state)); +} + static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); @@ -6180,6 +6238,16 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph atcphy->res.usb2phy_reg = res; } + /* Optional: without it the controller stops for a system sleep. */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwrctl"); + if (res) { + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map pwrctl regs"); + atcphy->regs.pwrctl = addr; + } + return 0; } @@ -6443,6 +6511,335 @@ static int atcphy_probe(struct platform_device *pdev) return 0; } +/* + * A system sleep with the controller kept, as macOS does it on the J700: its host goes to + * Sleep, not Off. xHCI has halted and saved its state by the time this PHY suspends, as + * the controller is its consumer. The PHY then takes the controller's core into + * hibernation through its power control, holds the core's reset without the clamp, and + * puts the eUSB2 PHY in its sleep state: reference clock gated, APB in reset, the link + * left as it is. The sleep's commit lowers ATC0_USB and ATC0_COMMON and keeps + * ATC0_USB_AON, which holds the power control, the PIPE handler and the USB2 PHY. + * + * At the wake ATC0_COMMON is back with the common block at its reset values. It's set up + * in the mode it had again, the controller is brought to D0, and xHCI restores its own state. + * + * USB2 and USB3 qualify, without DisplayPort, whose link the common block's power-down + * would end. With USB3 the halted controller's PIPE is parked on the dummy backend just + * before the hibernation, as macOS locks and parks it when its host goes to Sleep + * (setUSB3Mode, static), and the lanes are left as they are: the commit takes them down. + * At the wake the lanes come back in USB3 and the PIPE is routed to them when the controller's + * core is set up, before xHCI starts, with no Type-C mode change and no SuperSpeed port power + * cycle. macOS's raise from Sleep to its saved USB3 mode is also just a handshake and the USB3 + * PHY's power level (static, setUSB3Mode). macOS also stops its PHY's fabric traffic before + * the clock gate; that block isn't known here, and a halted controller has none. + */ +static bool atcphy_can_sleep_in_place(const struct apple_atcphy *atcphy) +{ + enum atcphy_pipehandler_state pipe = atcphy_pipe_state_for(atcphy, atcphy->mode); + + return atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->regs.pwrctl && + !atcphy->sleep_no_keep && atcphy->host_active && + (atcphy->mode == APPLE_ATCPHY_MODE_USB2 || atcphy->mode == APPLE_ATCPHY_MODE_USB3) && + !atcphy->mode_dirty && !atcphy->pipe_pending && + (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY || + atcphy->pipe_state == pipe) && !atcphy->usb_off_pending && + !atcphy->dp_owned_pmas && !atcphy->aux_power_requested && !atcphy->tunnel_users; +} + +/* + * The halted controller's PIPE goes to the dummy backend for the hibernation. A lock it + * doesn't acknowledge is only logged, as macOS's sleep only logs it too; the PIPE is then + * parked unlocked, as for a stopped controller. + */ +static int atcphy_sleep_park_pipe(struct apple_atcphy *atcphy) +{ + bool dirty = atcphy->mode_dirty; + int ret; + + ret = atcphy_park_pipehandler(atcphy, false); + if (!ret) + return 0; + + dev_warn(atcphy->dev, "PIPE lock not acknowledged for the sleep (%d): parking unlocked\n", + ret); + atcphy->mode_dirty = dirty; + return atcphy_park_pipehandler(atcphy, true); +} + +static void atcphy_usb2_sleep_enter(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); +} + +static void atcphy_usb2_sleep_exit(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); +} + +/* + * macOS's raise from Sleep: D0 asked for, the core's reset released, D0_ON awaited. Returns + * -ENODEV only if the core may be unpowered, when touching its registers would be an SError. + * With @went_off the core's rail is known to have gone off, so only D0_ON will do. Otherwise + * a core no longer reporting SLEEP has left hibernation, and the event alone is missing. + */ +static int atcphy_controller_d0(struct apple_atcphy *atcphy, bool went_off) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 val, status; + int ret; + + /* All zeros: the block lost power in the sleep and was reset, so no D0_ON will come. */ + status = readl(pc + AUSBC_PWR_STATUS); + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, 0); + clear32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + /* As the reset's release does: a block that lost its state may have the clamp on. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + if (!went_off && !status) { + dev_dbg(atcphy->dev, "USB controller lost its power state in the sleep\n"); + return 0; + } + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, val, val & AUSBC_PWR_EVENT_D0_ON, 100, + AUSBC_PWR_TIMEOUT_US); + status = readl(pc + AUSBC_PWR_STATUS); + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, + AUSBC_PWR_EVENT_D0_ON | AUSBC_PWR_EVENT_WAKE); + if (!ret) + return 0; + + dev_err(atcphy->dev, "USB controller gave no D0_ON: events %#x, status %#x\n", val, status); + if (!went_off && !(status & AUSBC_PWR_STATUS_SLEEP)) + return 0; + + return -ENODEV; +} + +/* + * A failure refuses this sleep, and the controller stops for the later ones, so that it + * can't hold every other driver's sleep up. sleep_ret says what state the core is in. + */ +static int atcphy_controller_hibernate(struct apple_atcphy *atcphy) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 events, status = 0; + bool went_off; + int ret; + + set32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, events, events & AUSBC_PWR_EVENT_VDD_OFF, + 100, AUSBC_PWR_TIMEOUT_US); + went_off = !ret; + if (!ret) { + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_VDD_OFF); + /* Armed as macOS arms them. That they drive wake source 70 is inferred. */ + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_WAKE); + ret = readl_poll_timeout(pc + AUSBC_PWR_STATUS, status, + status & AUSBC_PWR_STATUS_SLEEP, 100, + AUSBC_PWR_TIMEOUT_US); + } + if (ret) { + dev_err(atcphy->dev, "USB controller did not hibernate: events %#x, status %#x\n", + events, status); + atcphy->sleep_no_keep = true; + atcphy->sleep_ret = atcphy_controller_d0(atcphy, went_off); + return ret; + } + + /* The core's reset, without the clamp that only macOS's Off path sets. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + atcphy_usb2_sleep_enter(atcphy); + dev_dbg(atcphy->dev, "USB controller in hibernation for the sleep\n"); + + return 0; +} + +/* + * The common block went off with ATC0_COMMON. Set it up again in the mode it had, USB2 or + * USB3, without the controller's reset or a USB2 PHY power cycle: atcphy_configure() powers + * the USB2 PHY on over the running one, which rewrites only what it holds, so the hub's link + * stays. The host role goes in again as at probe, for a USB2 PHY that lost its registers + * too. The PIPE stays parked until the controller's core is set up and routes it. + */ +static int atcphy_common_restore(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode = atcphy->mode; + int ret; + + atcphy_forget_common_state(atcphy); + ret = atcphy_power_off(atcphy); + if (ret) + dev_warn(atcphy->dev, "PHY did not power down after the sleep: %d\n", ret); + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy_setup_pipehandler(atcphy); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + + ret = atcphy_configure(atcphy, mode); + if (!ret || mode == APPLE_ATCPHY_MODE_USB2) + return ret; + /* USB2 keeps the hub; the mux update after the wake then tries the mode again. */ + dev_warn(atcphy->dev, "mode %d not set up again after the sleep (%d): USB2 for now\n", + mode, ret); + return atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB2); +} + +static int atcphy_suspend_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + int ret; + + guard(mutex)(&atcphy->lock); + /* Suspend-to-idle keeps every domain on: nothing to do. */ + if (!atcphy->sleep_in_place || !pm_suspend_via_firmware()) + return 0; + /* A mode change since the prepare: the controller would lose its PIPE. */ + if (!atcphy_can_sleep_in_place(atcphy)) { + dev_err(dev, "USB changed for the sleep (mode %d, PIPE %d, host %d): refusing it\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + atcphy->sleep_ret = -EBUSY; + return -EBUSY; + } + ret = atcphy_sleep_park_pipe(atcphy); + if (ret) { + dev_err(dev, "PIPE not parked for the sleep: %d, refusing it\n", ret); + atcphy->sleep_ret = -EBUSY; + return -EBUSY; + } + + ret = atcphy_controller_hibernate(atcphy); + if (ret) + return ret; + atcphy->hibernated = true; + apple_sleep_usb_set_asleep(true); + + return 0; +} + +static int atcphy_resume_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + int ret = 0; + + guard(mutex)(&atcphy->lock); + if (!atcphy->hibernated) + return 0; + apple_sleep_usb_set_asleep(false); + + atcphy_usb2_sleep_exit(atcphy); + /* Only a sleep that reached the commit lowered ATC0_COMMON. */ + if (pm_resume_via_firmware()) { + ret = atcphy_common_restore(atcphy); + if (ret) + dev_err(dev, "PHY not set up again after the sleep: %d\n", ret); + } + atcphy->sleep_ret = atcphy_controller_d0(atcphy, false) ?: ret; + atcphy->hibernated = false; + if (!atcphy->sleep_ret) { + dev_dbg(dev, "USB controller back in D0 after the sleep, PHY in mode %d\n", + atcphy->mode); + /* + * The APB reset may have taken the secondary bank's defaults. That bank is next + * to the core and macOS's PHY exit doesn't touch it, so it may be under the core's + * rail, and an access to it with the rail off would be an SError: only now. + */ + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + } + + return 0; +} + +static struct apple_atcphy *atcphy_from_usb2_phy(struct phy *phy) +{ + if (!phy || phy->ops != &apple_atc_usb2_phy_ops) + return NULL; + return phy_get_drvdata(phy); +} + +/* + * After a sleep in place the PHY goes back to what the mux last asked for. The wake set the + * block up in its mode again and the controller's core routed the PIPE, so a mux update is + * left only if the mux changed during the sleep or that setup failed. + */ +static void atcphy_sleep_ss_up(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode; + int ret; + + scoped_guard(mutex, &atcphy->lock) { + if (!atcphy->sleep_ss_down) + return; + atcphy->sleep_ss_down = false; + mode = atcphy->typec_mode; + atcphy_restore_after_pd_off(atcphy); + if (atcphy_mux_is_noop(atcphy, mode)) { + dev_dbg(atcphy->dev, "SuperSpeed side already in mode %d after the sleep\n", + mode); + return; + } + } + ret = atcphy_mux_update_to(atcphy, mode); + dev_dbg(atcphy->dev, "SuperSpeed side back after the sleep, mode %d: %d\n", mode, ret); +} + +/* No lane or port change here: anything the sleep needs is done at noirq, in a few ms. */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + + guard(mutex)(&atcphy->lock); + if (atcphy_can_sleep_in_place(atcphy)) { + atcphy->sleep_in_place = true; + atcphy->sleep_ss_down = true; + atcphy->sleep_ret = 0; + return 0; + } + if (atcphy->regs.pwrctl && atcphy->fixed_usb2) + dev_dbg(atcphy->dev, + "USB controller stops for the sleep: mode %d, PIPE %d, host %d\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + + return -EBUSY; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_prepare); + +int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + + return atcphy->sleep_in_place ? atcphy->sleep_ret : -ENODATA; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_result); + +/* + * A controller that was restarted after the sleep is set up as at boot and may hibernate + * again. Only the PHY's own failures stop it for the later sleeps; a sleep refused for a + * USB change doesn't. + */ +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return; + scoped_guard(mutex, &atcphy->lock) { + if (!kept && atcphy->sleep_ret && atcphy->sleep_ret != -EBUSY) + atcphy->sleep_no_keep = true; + atcphy->sleep_in_place = false; + atcphy->sleep_ret = 0; + } + atcphy_sleep_ss_up(atcphy); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish); + static int atcphy_suspend(struct device *dev) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); @@ -6519,7 +6916,10 @@ static void atcphy_shutdown(struct platform_device *pdev) dev_err(atcphy->dev, "USB3 common shutdown failed: %d\n", ret); } -static DEFINE_SIMPLE_DEV_PM_OPS(atcphy_pm_ops, atcphy_suspend, atcphy_resume); +static const struct dev_pm_ops atcphy_pm_ops = { + SYSTEM_SLEEP_PM_OPS(atcphy_suspend, atcphy_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(atcphy_suspend_noirq, atcphy_resume_noirq) +}; static const struct atcphy_hw atcphy_hw_t8103 = { .gen = ATCPHY_GENERATION_T8103, @@ -6566,6 +6966,7 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .has_usb2phy_reg = true, .has_t8140_dp = true, .optional_tunables = true, + .park_pipe_unlocked = true, .restore_after_pd_off = true, }; diff --git a/include/linux/phy/apple-atc.h b/include/linux/phy/apple-atc.h new file mode 100644 index 0000000000000..180e97e6e8eaf --- /dev/null +++ b/include/linux/phy/apple-atc.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What the Apple Type-C PHY offers the USB controller it serves, beyond the + * generic PHY operations. + */ +#ifndef _LINUX_PHY_APPLE_ATC_H_ +#define _LINUX_PHY_APPLE_ATC_H_ + +#include +#include + +struct phy; + +#if IS_REACHABLE(CONFIG_PHY_APPLE_ATC) +/* + * For a firmware-driven system sleep, before any device suspends and without the + * controller's locks held. 0 means the PHY takes the controller through hibernation after + * xHCI has saved its state and brings it back before xHCI resumes, so the host can stay + * up; with USB3 its PIPE is parked for it. Otherwise the controller has to be stopped for + * the sleep. + */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy); + +/* + * After the sleep: 0 if the controller is in D0. -ENODEV if it may be unpowered, so that its + * registers must not be touched. Any other error: it's in D0, but the PHY isn't set up for it. + */ +int apple_atcphy_sleep_result(struct phy *usb2_phy); + +/* + * Ends a sleep that prepare took on, with the SuperSpeed side back as the mux wants it. + * Without @kept the host was restarted; only a failure of the PHY's own then makes the + * controller stop for later sleeps. + */ +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept); +#else +static inline int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ +} +#endif + +#endif diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index c15ae40d30b8d..877d1b416f764 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -141,6 +141,15 @@ def atc_restore_helpers(source): return atc_function(source, 'atcphy_setup_pipehandler') +def atc_common_state_helpers(source): + """What atcphy_restore_after_pd_off() calls to drop the common block's DisplayPort and + tunnel state after a power-down. Older sources do that inline: ''. + """ + if 'atcphy_forget_common_state(' not in atc_function(source, 'atcphy_restore_after_pd_off'): + return '' + return atc_function(source, 'atcphy_forget_common_state') + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 2db0e2b87e8f9..2b3c4f0338e43 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1090,7 +1090,7 @@ def fn(n, name): 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', - 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_restore_after_pd_off', + 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_forget_common_state', 'atcphy_restore_after_pd_off', 'atcphy_usb2_init', 'atcphy_usb3_power_off', 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index a56a3a3ccfbff..40d3d5d17fc8a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -4,7 +4,7 @@ from pathlib import Path import re from common import (block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE, - atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, + atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, atc_common_state_helpers, atc_cio_helpers) def parking(root): @@ -181,6 +181,7 @@ def function(source, name): parts.append(function(atc, name)) parts.append(dp) parts.append(atc_restore_helpers(atc)) + parts.append(atc_common_state_helpers(atc)) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_set_orientation', diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 8cae88d722d88..9c59dda578c79 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -428,6 +428,8 @@ def ownership(root, usb_unreachable=False): helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n' helpers+=''.join(atc_function(s,name) for name in ('atcphy_park_retry_delay','atcphy_park_retry','atcphy_park_due','atcphy_mux_change')) + # The mux callback's body, split out so the wake can set the mode it had. + helpers+=''.join(atc_function(s,name) for name in ('atcphy_mux_done','atcphy_mux_update_to')) helpers+=ns['function'](s,'atcphy_mux_update')+'\n' # Lane fixtures provide the target mapping and locked body; use the real callback here. # The mux callback powers an idle-power-off PHY first; this PHY is not one. From d1c3c0256bb73aee8c0a35b48624b0113dd7919c Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:23 -0500 Subject: [PATCH 50/73] usb: dwc3: apple: keep the host through a deep sleep macOS keeps its host through a sleep: the hub isn't reset, xHCI's state comes back in place and the devices keep their slots. For a host behind a fixed hub whose PHY can take the core through hibernation, don't stop it for a deep sleep. xHCI suspends and resumes as on any platform, saving and restoring its state, and the PHY hibernates the core in between, see apple_atcphy_sleep_prepare(). The wakeup setting stays: the glue is a syscore device while its core runs, so genpd keeps away from its domain either way, and the sleep's commit does the lowering. - macOS's T8140 controller reset sets GCTL's hibernation enable and a power-down scale of 0xd. Without the enable the core accepts a D3 request but saves and restores nothing. Set both after every core init on T8140, since the soft reset undoes them. - After the core's D0 macOS polls the PIPE PHY's ready bit, bit 30 of +0x2c in the DRD config block, for 100 ms, and only logs a timeout. Map that block and do the same ahead of xHCI's restore. - The GUID scratch register is marked before the sleep and read back after. A glue hook in xHCI's resume quirk checks it and the PHY's result before xHCI's first access. A core in D0 that lost its registers is set up again in place, the way the bring-up does after its reset's release: dwc3_core_init(), dropping the PHY references it takes a second time, the CIO setup, the tunables, the hibernation setup and the host role, then the USB3 PHY's host mode as the role switch sets it, which routes the PIPE to a PHY that came back in USB3 before xHCI starts. The hook then tells xHCI the controller lost its state, and xHCI starts it over: the root hubs and their devices reset-resume and keep their drivers and input nodes, and the next sleep hibernates again. If that setup fails the host is restarted as after a stop, and the devices enumerate again. A core that may be unpowered is left alone: USB stays off until a reboot and later sleeps are refused. A sleep refused before the hibernation left the core as it was, and xHCI resumes it as usual. - A fixed-hub host started after a sleep, by the stop path or that restart, starts with no role, as at probe, so dwc3_core_soft_reset() runs as well: on a J700 the hub didn't come back after a restart without it, and did when the driver was bound again. After every sleep of a fixed-hub host the glue checks 5 s later that the hub is on its root port, the hub node's reg. If it isn't, the host is stopped, the hub's link left down for half a second and the host started again, once. - The PHY's SuperSpeed check after a wake, and any mux update it needs, runs from a work item rather than from the PM notifier, so the resume doesn't wait for it; the next sleep's PM_SUSPEND_PREPARE waits for it to end. - Before the freeze, a host that sleeps in place records which of the fixed hub's ports have a device at or below them with wakeup enabled, the setting usbcore goes by when it arms remote wakeup for a system sleep, for the port controllers' USB wake. The hub's own setting doesn't count, as it's on the board. The record is cleared after the sleep, and stays clear for a host that stops. A setting or a device that changes between the notifier and the freeze isn't seen. Everywhere else, and for a PHY that can't hibernate the core, the glue stops the controller as before. Measured on a J700: the core loses its registers in every sleep so far, and after each wake the hub and a USB keyboard on it reset-resume instead of enumerating again, and the SuperSpeed side is in its mode again when the glue checks it. Signed-off-by: Diljit Singh --- drivers/usb/dwc3/core.h | 3 + drivers/usb/dwc3/dwc3-apple.c | 432 +++++++++++++++++- drivers/usb/dwc3/host.c | 12 + .../drivers/gpu/apple-dcp/dwc3/mocks.c | 5 + 4 files changed, 440 insertions(+), 12 deletions(-) diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 30b5c5fc4d071..7821ccb4686e5 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1002,10 +1002,13 @@ struct dwc3_scratchpad_array { * need to be passed on to glue layer * @pre_set_role: Notify glue of role switch notifications * @pre_run_stop: Notify run stop enable/disable information to glue + * @pre_host_resume: Before xHCI touches the core at a system resume; an error keeps it away, + * and a positive value says the core lost its state and was set up again */ struct dwc3_glue_ops { void (*pre_set_role)(struct dwc3 *dwc, enum usb_role role); void (*pre_run_stop)(struct dwc3 *dwc, bool is_on); + int (*pre_host_resume)(struct dwc3 *dwc); }; /** diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index c7b49e48f302c..cd3357378a00b 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -9,17 +9,23 @@ */ #include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include #include +#include +#include #include #include "dwc3-apple-state.h" @@ -92,9 +98,13 @@ * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device * may wake the system, as a halted xHC could not signal the wakeup. * - * A deep sleep takes the controller's power domain away, so nothing can be kept running across - * it. The glue stops everything before the freezer runs, the way a cable disconnect does, and - * brings it back once the system has resumed. See dwc3_apple_pm_notify(). + * A deep sleep takes the controller's power domain away. Behind a fixed hub on T8140 the host + * sleeps in place, as macOS's does: xHCI suspends and resumes as usual, and the PHY takes the + * core through hibernation in between, so the hub and its devices stay. A core that comes + * back without its registers is set up again in place and xHCI starts over, so the devices + * reset-resume. Anywhere else, or when the PHY can't, the glue stops everything before the + * freezer runs, the way a cable disconnect does, and brings it back once the system has + * resumed. See dwc3_apple_pm_notify(). * * Runtime PM (T6040 and T8132, see dwc3_apple_core_probe()): the glue holds a reference from * the reset deassert to the end of the shutdown, so with no cable the controller can @@ -147,6 +157,8 @@ struct dwc3_apple { struct phy *usb3_phy; struct usb_role_switch *role_sw; bool fixed_hub; + /* The root port the fixed hub sits on, its node's reg. */ + u32 hub_port; bool host_only; struct mutex lock; @@ -163,6 +175,17 @@ struct dwc3_apple { bool pre_core_rpm; bool dt_dis_u3_susphy; struct notifier_block pm_nb; + /* Kept through a deep sleep, with the PHY's hibernation, and the GUID it had. */ + bool sleep_in_place; + u32 sleep_guid; + /* The core lost its registers in that sleep and was set up again in place. */ + bool sleep_reinit; + /* Looks for the fixed hub once a host restarted after a sleep has had time to find it. */ + struct delayed_work hub_check; + struct work_struct sleep_finish; + bool sleep_finish_kept; + /* The core didn't come back from a sleep, and nothing may touch it until a reboot. */ + bool sleep_dead; /* Stopped for a deep sleep: the state to come back to, and the wakeup setting. */ bool sleep_stopped; bool sleep_wakeup; @@ -218,6 +241,27 @@ static inline void dwc3_apple_mask(struct dwc3_apple *appledwc, u32 offset, u32 dwc3_apple_writel(appledwc, offset, reg); } +/* + * macOS's T8140 controller reset sets GCTL's hibernation enable and a power-down scale + * of 0xd. Without the enable the core's power management accepts a + * D0/D3 request but saves and restores nothing, so the PHY's hibernation for a sleep + * needs it. + */ +#define DWC3_APPLE_T8140_PWRDNSCALE 0xd + +static void dwc3_apple_setup_hibernation(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 reg; + + if (!of_device_is_compatible(appledwc->dev->of_node, "apple,t8140-dwc3")) + return; + reg = dwc3_readl(dwc, DWC3_GCTL); + reg &= ~DWC3_GCTL_PWRDNSCALE_MASK; + reg |= DWC3_GCTL_PWRDNSCALE(DWC3_APPLE_T8140_PWRDNSCALE) | DWC3_GCTL_GBLHIBERNATIONEN; + dwc3_writel(dwc, DWC3_GCTL, reg); +} + static void dwc3_apple_setup_cio(struct dwc3_apple *appledwc) { dwc3_apple_writel(appledwc, APPLE_DWC3_CIO_LFPS_OFFSET, APPLE_DWC3_CIO_LFPS_OFFSET_VALUE); @@ -334,6 +378,96 @@ static void dwc3_apple_set_ptrcap(struct dwc3_apple *appledwc, u32 mode) dwc3_set_prtcap(&appledwc->dwc, mode, false); } +/* Written to the core's GUID scratch register before a sleep in place, read back after it. */ +#define DWC3_APPLE_SLEEP_GUID 0x534c5045 + +/* macOS waits up to 100 ms for this after the core's D0, and only logs a timeout. */ +#define DWC3_APPLE_CFG_STATUS 0x2c +#define DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY BIT(30) +#define DWC3_APPLE_PIPE_PHY_READY_US 100000 + +/* + * The core is powered but its registers are at their reset values. Set it up again the way + * dwc3_apple_core_start() and dwc3_apple_set_role() do, in place: the PHYs and the xHCI + * device stay, so xHCI only has to start over and its devices reset-resume. Runs in the + * resume, while nothing else touches the core. + */ +static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + int i, ret; + + ret = dwc3_core_init(dwc); + if (ret) + return ret; + /* dwc3_core_init() took PHY references the core holds already. */ + for (i = 0; i < dwc->num_usb3_ports; i++) { + phy_power_off(dwc->usb3_generic_phy[i]); + phy_exit(dwc->usb3_generic_phy[i]); + } + for (i = 0; i < dwc->num_usb2_ports; i++) { + phy_power_off(dwc->usb2_generic_phy[i]); + phy_exit(dwc->usb2_generic_phy[i]); + } + + dwc3_apple_setup_cio(appledwc); + dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); + dwc3_apple_set_ptrcap(appledwc, DWC3_GCTL_PRTCAP_HOST); + dwc->dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; + dwc3_enable_susphy(dwc, true); + /* A PHY that came back in a USB3 mode routes the PIPE to it now, before xHCI starts. */ + ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_HOST); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed after the sleep, USB2 only, err=%d\n", + ret); + + return 0; +} + +/* + * Runs before xHCI touches the core after a sleep in place. A core that may be unpowered must + * be left alone, as an access could be an SError: dwc3_apple_sleep_kept() deals with it once + * the system is back. A core that came back without its registers is set up again, and xHCI + * is told it lost its state (a positive return). A sleep refused before the hibernation left + * the core as it was. + */ +static int dwc3_apple_pre_host_resume(struct dwc3 *dwc) +{ + struct dwc3_apple *appledwc = to_dwc3_apple(dwc); + u32 val; + int ret; + + if (!READ_ONCE(appledwc->sleep_in_place)) + return 0; + + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + if (ret == -EBUSY) + ret = 0; + if (!ret && appledwc->drd_cfg && + readl_poll_timeout(appledwc->drd_cfg + DWC3_APPLE_CFG_STATUS, val, + val & DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY, 100, + DWC3_APPLE_PIPE_PHY_READY_US)) + dev_warn(appledwc->dev, "PIPE PHY not ready after the sleep: %#x\n", val); + if (!ret && dwc3_readl(dwc, DWC3_GUID) != DWC3_APPLE_SLEEP_GUID) { + ret = dwc3_apple_core_reinit(appledwc); + if (!ret) { + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core lost its registers in the sleep: set up again, xHCI starts over\n"); + return 1; + } + } + if (ret) + dev_err(appledwc->dev, "not resuming xHCI, the core didn't come back: %d\n", ret); + + return ret; +} + +static const struct dwc3_glue_ops dwc3_apple_glue_ops = { + .pre_host_resume = dwc3_apple_pre_host_resume, +}; + static bool dwc3_apple_host_before_core_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { @@ -351,6 +485,7 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc, enum dwc3_apple_st WARN_ON_ONCE(appledwc->state != DWC3_APPLE_PROBE_PENDING); appledwc->dwc.dev = appledwc->dev; + appledwc->dwc.glue_ops = &dwc3_apple_glue_ops; probe_data.dwc = &appledwc->dwc; probe_data.res = appledwc->mmio_resource; probe_data.ignore_clocks_and_resets = true; @@ -595,6 +730,7 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st dwc3_apple_host_before_core_init(appledwc, state)) dwc3_apple_setup_cio(appledwc); dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); appledwc->core_active = true; appledwc->usb2_power_off_pending = BIT(appledwc->dwc.num_usb2_ports) - 1; appledwc->usb3_power_off_pending = BIT(appledwc->dwc.num_usb3_ports) - 1; @@ -739,6 +875,152 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st return ret; } +static int dwc3_apple_exit(struct dwc3_apple *appledwc); + +/* A hub that hasn't enumerated by then didn't come back. */ +#define DWC3_APPLE_HUB_CHECK_MS 5000 + +/* Any way a fixed-hub host came out of a sleep, its hub must be there a little later. */ +static void dwc3_apple_check_hub_later(struct dwc3_apple *appledwc) +{ + if (appledwc->fixed_hub) + mod_delayed_work(system_freezable_wq, &appledwc->hub_check, + msecs_to_jiffies(DWC3_APPLE_HUB_CHECK_MS)); +} + +/* + * Bring a controller up again after a sleep that stopped it or lost it. A fixed-hub host + * starts as at probe, where the core has no role yet, so dwc3_core_soft_reset() runs too: + * rebinding the driver brought the J700's hub back where a restart without it didn't. + */ +static int dwc3_apple_start_after_sleep(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +{ + int ret; + + lockdep_assert_held(&appledwc->lock); + if (appledwc->fixed_hub) + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, state); + if (ret) { + dev_err(appledwc->dev, "Failed to start after sleep, err=%d\n", ret); + return ret; + } + dwc3_apple_check_hub_later(appledwc); + return 0; +} + +/* Whether the fixed hub is on its USB2 root port. Takes the xHCI device's lock. */ +static bool dwc3_apple_hub_present(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root; + struct usb_hcd *hcd; + bool present = true; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return true; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + present = usb_hub_find_child(root, appledwc->hub_port); + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return present; +} + +/* Whether @udev or a device below it can wake the system. Takes the hubs' locks. */ +static bool dwc3_apple_can_wake(struct usb_device *udev) +{ + struct usb_device *child; + bool wake = device_may_wakeup(&udev->dev); + int port1; + + if (wake || !udev->maxchild) + return wake; + + usb_lock_device(udev); + usb_hub_for_each_child(udev, port1, child) { + wake = dwc3_apple_can_wake(child); + if (wake) + break; + } + usb_unlock_device(udev); + + return wake; +} + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The hub's own setting doesn't count. Takes the xHCI device's lock and the hubs'. + */ +static unsigned long dwc3_apple_hub_wake_ports(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root, *hub, *child; + unsigned long ports = 0; + struct usb_hcd *hcd; + int port1; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return 0; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + hub = usb_hub_find_child(root, appledwc->hub_port); + if (hub) { + usb_lock_device(hub); + usb_hub_for_each_child(hub, port1, child) { + if (port1 < BITS_PER_LONG && dwc3_apple_can_wake(child)) + ports |= BIT(port1); + } + usb_unlock_device(hub); + } + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return ports; +} + +/* + * Once after a sleep: if the hub isn't there, stop the host and start it again, with the + * hub's link down for a moment first, as unbinding and binding the driver does. This also + * catches an xHCI that didn't come back after the core was set up again. + */ +static void dwc3_apple_hub_check(struct work_struct *work) +{ + struct dwc3_apple *appledwc = container_of(to_delayed_work(work), struct dwc3_apple, + hub_check); + int ret; + + guard(mutex)(&appledwc->lock); + if (appledwc->shutting_down || appledwc->state != DWC3_APPLE_HOST || + dwc3_apple_hub_present(appledwc)) + return; + + dev_warn(appledwc->dev, "hub not back after the sleep: starting the host again\n"); + ret = dwc3_apple_exit(appledwc); + if (ret) { + dev_err(appledwc->dev, "Failed to stop the host again, err=%d\n", ret); + return; + } + msleep(500); + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); + if (ret) + dev_err(appledwc->dev, "Failed to start the host again, err=%d\n", ret); +} + static int dwc3_apple_exit(struct dwc3_apple *appledwc) { lockdep_assert_held(&appledwc->lock); @@ -866,23 +1148,88 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } +/* + * After a sleep in place: the PHY says whether the core came back to D0, and the GUID register + * whether it kept its registers. A core set up again at the resume kept its host and devices, + * which reset-resumed. Otherwise restart the host as after a stop, and the devices enumerate + * again. With a core that may be unpowered even that would touch it, so USB stays off, and + * later sleeps are refused, as xHCI's suspend would touch it too. Returns whether the host + * was kept. + */ +static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 guid = 0; + int ret; + + appledwc->sleep_in_place = false; + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + if (ret == -ENODEV) { + appledwc->sleep_dead = true; + dev_err(appledwc->dev, "the core didn't come back from the sleep: USB off until a reboot\n"); + return false; + } + if (appledwc->sleep_reinit) { + appledwc->sleep_reinit = false; + dev_dbg(appledwc->dev, "host kept through the sleep, set up again\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + if (!ret || ret == -EBUSY) { + guid = dwc3_readl(dwc, DWC3_GUID); + if (guid == DWC3_APPLE_SLEEP_GUID) { + dwc3_writel(dwc, DWC3_GUID, appledwc->sleep_guid); + dev_dbg(appledwc->dev, "host kept through the sleep\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + ret = -EIO; + } + + dev_warn(appledwc->dev, "host lost its state in the sleep (%d, GUID %#x): restarting it\n", + ret, guid); + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after sleep, err=%d\n", ret); + else + dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + + return false; +} + /* * Runs before the freezer, when the sleep's real target isn't known yet, so this goes by the * state "mem" means now. Writing "freeze" with deep selected costs a USB restart and nothing else. * A stop that fails refuses the sleep. PM_POST_SUSPEND follows an abandoned sleep as well. */ -static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +static int dwc3_apple_pm_notify_locked(struct dwc3_apple *appledwc, unsigned long action, + bool in_place, bool *finish, bool *kept) { - struct dwc3_apple *appledwc = container_of(nb, struct dwc3_apple, pm_nb); struct device *dev = appledwc->dev; int ret; - guard(mutex)(&appledwc->lock); + lockdep_assert_held(&appledwc->lock); switch (action) { case PM_SUSPEND_PREPARE: - if (pm_suspend_default_s2idle() || appledwc->shutting_down) + if (pm_suspend_default_s2idle() || appledwc->shutting_down) { + *finish = in_place; + *kept = true; break; + } + if (appledwc->sleep_dead) { + dev_err(dev, "refusing the sleep: the core is gone until a reboot\n"); + return notifier_from_errno(-EIO); + } + /* xHCI suspends and resumes as usual, and the wakeup setting stays. */ + if (in_place) { + appledwc->sleep_guid = dwc3_readl(&appledwc->dwc, DWC3_GUID); + dwc3_writel(&appledwc->dwc, DWC3_GUID, DWC3_APPLE_SLEEP_GUID); + appledwc->sleep_in_place = true; + /* For the port controllers' USB wake. Changes after this go unseen. */ + apple_sleep_usb_set_wake_ports(dwc3_apple_hub_wake_ports(appledwc)); + break; + } appledwc->sleep_state = appledwc->state; ret = dwc3_apple_exit(appledwc); if (ret) @@ -893,6 +1240,12 @@ static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, appledwc->sleep_stopped = true; break; case PM_POST_SUSPEND: + if (appledwc->sleep_in_place) { + apple_sleep_usb_set_wake_ports(0); + *kept = dwc3_apple_sleep_kept(appledwc); + *finish = true; + break; + } if (!appledwc->sleep_stopped) break; appledwc->sleep_stopped = false; @@ -901,9 +1254,7 @@ static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, (appledwc->sleep_state != DWC3_APPLE_HOST && appledwc->sleep_state != DWC3_APPLE_DEVICE)) break; - ret = dwc3_apple_init(appledwc, appledwc->sleep_state); - if (ret) - dev_err(dev, "Failed to start after sleep, err=%d\n", ret); + dwc3_apple_start_after_sleep(appledwc, appledwc->sleep_state); break; default: break; @@ -912,6 +1263,52 @@ static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, return NOTIFY_DONE; } +/* + * The PHY's sleep prepare and finish may take the SuperSpeed port down and up, under the + * controller's device lock, so they run outside the glue's lock. + */ +static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct dwc3_apple *appledwc = container_of(nb, struct dwc3_apple, pm_nb); + bool in_place = false, finish = false, kept = false; + int ret; + + /* + * The last wake's SuperSpeed restore ends before this sleep looks at the PHY. The hub + * check takes the glue's lock, and has nothing to look at until the sleep is over. + */ + if (action == PM_SUSPEND_PREPARE) { + flush_work(&appledwc->sleep_finish); + cancel_delayed_work_sync(&appledwc->hub_check); + } + if (action == PM_SUSPEND_PREPARE && !pm_suspend_default_s2idle()) { + scoped_guard(mutex, &appledwc->lock) + in_place = appledwc->fixed_hub && appledwc->state == DWC3_APPLE_HOST && + !appledwc->shutting_down && !appledwc->sleep_dead; + in_place = in_place && !apple_atcphy_sleep_prepare(appledwc->usb2_phy); + } + + scoped_guard(mutex, &appledwc->lock) + ret = dwc3_apple_pm_notify_locked(appledwc, action, in_place, &finish, &kept); + + /* After a wake, user space doesn't wait for SuperSpeed: it comes back in the background. */ + if (finish && action == PM_POST_SUSPEND) { + WRITE_ONCE(appledwc->sleep_finish_kept, kept); + queue_work(system_unbound_wq, &appledwc->sleep_finish); + } else if (finish) { + apple_atcphy_sleep_finish(appledwc->usb2_phy, kept); + } + + return ret; +} + +static void dwc3_apple_sleep_finish(struct work_struct *work) +{ + struct dwc3_apple *appledwc = container_of(work, struct dwc3_apple, sleep_finish); + + apple_atcphy_sleep_finish(appledwc->usb2_phy, READ_ONCE(appledwc->sleep_finish_kept)); +} + static void dwc3_apple_pm_notifier_remove(void *data) { struct dwc3_apple *appledwc = data; @@ -919,11 +1316,16 @@ static void dwc3_apple_pm_notifier_remove(void *data) unregister_pm_notifier(&appledwc->pm_nb); } -static bool dwc3_apple_has_fixed_hub(struct device *dev) +/* A fixed hub's node gives the root port it sits on in its reg, 1 if it doesn't. */ +static bool dwc3_apple_has_fixed_hub(struct device *dev, u32 *port) { struct device_node *hub __free(device_node) = of_get_available_child_by_name(dev->of_node, "hub"); + *port = 1; + if (hub) + of_property_read_u32(hub, "reg", port); + return hub; } @@ -938,7 +1340,7 @@ static int dwc3_apple_probe(struct platform_device *pdev) return -ENOMEM; appledwc->dev = &pdev->dev; - appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev); + appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev, &appledwc->hub_port); appledwc->host_only = dwc3_apple_host_only(dev); appledwc->dt_dis_u3_susphy = device_property_read_bool(dev, "snps,dis_u3_susphy_quirk"); /* The PHY driver and the DARTs follow runtime PM on these SoCs. */ @@ -1009,6 +1411,12 @@ static int dwc3_apple_probe(struct platform_device *pdev) * details. */ appledwc->state = DWC3_APPLE_PROBE_PENDING; + ret = devm_delayed_work_autocancel(dev, &appledwc->hub_check, dwc3_apple_hub_check); + if (ret) + return ret; + ret = devm_work_autocancel(dev, &appledwc->sleep_finish, dwc3_apple_sleep_finish); + if (ret) + return ret; appledwc->pm_nb.notifier_call = dwc3_apple_pm_notify; ret = register_pm_notifier(&appledwc->pm_nb); if (ret) diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 89e1c0c492dbe..652942d3f6f4f 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -79,8 +79,20 @@ static void dwc3_xhci_plat_start(struct usb_hcd *hcd) dwc3_enable_susphy(dwc, true); } +static int dwc3_xhci_plat_resume_quirk(struct usb_hcd *hcd) +{ + struct platform_device *pdev = to_platform_device(hcd->self.controller); + struct dwc3 *dwc = dev_get_drvdata(pdev->dev.parent); + + if (!usb_hcd_is_primary_hcd(hcd) || !dwc->glue_ops || !dwc->glue_ops->pre_host_resume) + return 0; + + return dwc->glue_ops->pre_host_resume(dwc); +} + static const struct xhci_plat_priv dwc3_xhci_plat_quirk = { .plat_start = dwc3_xhci_plat_start, + .resume_quirk = dwc3_xhci_plat_resume_quirk, }; static void dwc3_host_fill_xhci_irq_res(struct dwc3 *dwc, diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index a53a29f32999f..ff5a386f88c53 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -331,6 +331,11 @@ static void dwc3_apple_setup_cio(struct dwc3_apple *a) (void)a; setup_cio++; } +/* The hibernation block's register setup; teardown order doesn't depend on it. */ +static void __attribute__((unused)) dwc3_apple_setup_hibernation(struct dwc3_apple *a) +{ + (void)a; +} static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, enum phy_mode *m) { From abaee2664d805df8ee584b775238869cc254efe8 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:23 -0500 Subject: [PATCH 51/73] usb: typec: tipd: tell the root port's controller about a USB sleep macOS's root port, when it goes to sleep with the host in Sleep, sends its port controller the 4CC USBw with a mode by the speed on the port: 0x09 for full and high speed. At the wake it sends 0x80 and then 0x00 before the port resumes. The controller then watches the USB2 lines for a remote wakeup. macOS (25G83, read statically) does this only for a root port with Usb2ExternalRemoteWake, set from a dock-remote-wake property on the port, a usb-repeater value of 'mcaw' or the usb-dock-remote-wake boot argument. The J700's ADT has none of them (J713's root ports have dock-remote-wake), so macOS likely sends no USBw there; here it is what lets a USB keyboard on a connector wake a J700. Do the same on the controller a tree marks with apple,usb-remote-wake, for a firmware-driven sleep only, and only when the fixed hub's port wired to this connector, found through the connector's graph, has a device at or below it with wakeup enabled, as the USB host's glue recorded before the freeze. Armed without one, the controller reported a USB2 plug at the sleep's entry and the sleep came straight back: on an empty port, and on one left with a passive adapter, a partner with nothing on USB (measured on a J700). A device on the other connector doesn't count: going by the empty-port and adapter cases, the controller watches only its own connector's lines. A device plugged in during the sleep still wakes the system through the SMC's plug event. The arm comes right before the controller's SPMI sleep, the hand back right after its wake, and the controller out of S3 before that, at every wake. It registers before xHCI, so xHCI has suspended by the first and hasn't resumed by the second. A failure only logs. A controller the sleep's debug link cut reset is set up again after the wake, as asked; that's no longer reported as a warning, which stays for one that reset by itself. Measured on a J700: with nothing on the ports, USB wake isn't armed and the sleeps commit; with a USB keyboard plugged into a connector, a key press wakes the system through the SMC. Signed-off-by: Diljit Singh --- drivers/usb/typec/tipd/core.c | 90 ++++++++++++++++++++++++++++++- drivers/usb/typec/tipd/spmi.c | 7 ++- drivers/usb/typec/tipd/tps6598x.h | 5 ++ 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 05812642c84e7..a5d7b5b03fd8a 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1635,8 +1635,10 @@ static void cd321x_complete(struct tps6598x *tps) * The cut was a cold reset of this controller's firmware: its interrupt mask and power * state are back at their defaults, so set them up again whether or not an event comes. */ - if (apple_sleep_debug_link_cut_peek(tps->dev)) + if (apple_sleep_debug_link_cut_peek(tps->dev)) { cd321x->pm.setup_pending = true; + cd321x->reset_by_cut = true; + } if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); cd321x_rearm_debug_link(tps); @@ -1709,8 +1711,11 @@ static int cd321x_check_setup(struct cd321x *cd321x, u64 event) dev_err(tps->dev, "controller setup recovery exhausted: %d\n", ret); return ret; } - if (ret) + if (ret && cd321x->reset_by_cut) + dev_dbg(tps->dev, "set up again after the debug link's cut\n"); + else if (ret) dev_warn(tps->dev, "restored controller power state and interrupt mask\n"); + cd321x->reset_by_cut = false; /* Only a fresh snapshot may be applied after restoring the controller. */ cd321x->state_valid = false; cd321x->pm.phase = CD321X_PM_REVALIDATE; @@ -3181,6 +3186,87 @@ void tipd_remove(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_remove); +/* + * macOS tells the port controller of a USB root port with dock-remote-wake when that port + * sleeps, and the controller then watches the USB2 lines for a remote wakeup (static; what its + * firmware does with it isn't known). The J700's root port has none, but measured on a J700, + * a USB keyboard on a connector wakes it this way. The mode goes by the speed on the port: 0x09 + * for full and high speed. At the wake the controller hands the lines back before the port + * resumes. Only a firmware-driven sleep needs this, and a failure only logs. + */ +#define TIPD_USBW_HIGH_SPEED 0x09 +#define TIPD_USBW_HAND_BACK 0x80 +#define TIPD_USBW_OFF 0x00 + +static void tipd_usbw(struct tps6598x *tps, u8 mode) +{ + int ret; + + guard(mutex)(&tps->lock); + ret = tps6598x_exec_cmd(tps, "USBw", sizeof(mode), &mode, 0, NULL); + if (ret) + dev_warn(tps->dev, "USBw %#x failed: %d\n", mode, ret); + else + dev_dbg(tps->dev, "USBw %#x\n", mode); +} + +/* The fixed hub's port that the connector's USB2 pair goes to, from the graph, or 0. */ +static u32 tipd_hub_port(struct tps6598x *tps) +{ + struct fwnode_handle *con __free(fwnode_handle) = + device_get_named_child_node(tps->dev, "connector"); + struct fwnode_handle *ep __free(fwnode_handle) = + fwnode_graph_get_endpoint_by_id(con, 0, 0, 0); + struct fwnode_handle *port __free(fwnode_handle) = fwnode_graph_get_remote_port(ep); + u32 reg = 0; + + fwnode_property_read_u32(port, "reg", ®); + return reg; +} + +void tipd_usb_wake(struct tps6598x *tps, bool asleep) +{ + bool armed; + + if (!pm_suspend_via_firmware() || + !device_property_read_bool(tps->dev, "apple,usb-remote-wake")) + return; + if (asleep) { + u32 port = tipd_hub_port(tps); + bool wake = apple_sleep_usb_port_can_wake(port); + + /* + * Only for a device on this connector that can wake. Armed without one, the + * controller reported a USB2 plug at the sleep's entry and the sleep came straight + * back, on an empty port and on one left with a passive adapter (measured on a + * J700). A device plugged in during the sleep still wakes through the SMC's plug + * event. + */ + if (wake) + dev_dbg(tps->dev, "a device on hub port %u can wake: arming USB wake\n", + port); + else + dev_dbg(tps->dev, "nothing on hub port %u can wake: USB wake not armed\n", + port); + scoped_guard(mutex, &tps->lock) + tps->usbw_armed = wake; + if (wake) + tipd_usbw(tps, TIPD_USBW_HIGH_SPEED); + return; + } + /* Out of S3 first: .complete would only come once the port has resumed. */ + scoped_guard(mutex, &tps->lock) { + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + armed = tps->usbw_armed; + tps->usbw_armed = false; + } + if (!armed) + return; + tipd_usbw(tps, TIPD_USBW_HAND_BACK); + tipd_usbw(tps, TIPD_USBW_OFF); +} +EXPORT_SYMBOL_GPL(tipd_usb_wake); + int tipd_suspend(struct tps6598x *tps) { if (tps->wakeup) { diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 32c3778871013..45ea737223ed7 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -839,8 +839,11 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) } else { err = sn201202x_wake(sn); } - if (!err) + if (!err) { + /* Registered before xHCI, so this is before the root port resumes. */ + tipd_usb_wake(tps, false); err = tipd_resume(tps); + } /* The resume revalidation acknowledges events latched while asleep. */ if (tps->irq) enable_irq(tps->irq); @@ -890,6 +893,8 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) else dev_warn(dev, "port state unreadable, no wakeup for an attach\n"); } + /* Registered before xHCI, so its root port has suspended by now. */ + tipd_usb_wake(tps, true); reinit_completion(&sn->sleep_completion); err = spmi_command_sleep(sn->sdev); if (err) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c39feab87c20f..f0dfd0a3b1b77 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -395,6 +395,8 @@ struct tps6598x { struct tipd_debugfs *debugfs; struct typec_altmode *partner_altmode_dp; + /* USB remote wake armed on this port for the sleep (USBw). */ + bool usbw_armed; }; /* TPS_REG_USB4_STATUS */ @@ -466,6 +468,8 @@ struct cd321x { atomic_t ready_event; atomic_t restore_event; bool display_route_active; + /* The sleep's debug link cut reset this controller: its setup is expected. */ + bool reset_by_cut; bool dp_sink_reported; bool dp_irq_hpd; @@ -522,6 +526,7 @@ struct cd321x_sleep_state { }; int cd321x_sleep_events(struct tps6598x *tps, struct cd321x_sleep_state *st); +void tipd_usb_wake(struct tps6598x *tps, bool asleep); int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, size_t in_len, const u8 *in_data, size_t out_len, u8 *out_data, From cdf4563bc5d57cdaaaebf3de10e97f1281b58362 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:24 -0500 Subject: [PATCH 52/73] media: apple: isp: stop and restart T8140 firmware across a deep sleep On T8140 iBoot loads the ISP firmware, and the driver starts it once and keeps it resident. A sleep that powers the SoC down takes the coprocessor with it, so the firmware has to be stopped first, and the camera stayed dead until the next boot. The firmware rebuilds its own data from a pristine copy early in each boot, so it can start again from its memory as iBoot handed it over. Stop it as macOS does for a cold start, and start it again: - The stop: the stop key request, opcode 0x1d on the DEBUG channel with macOS's 1 s timeout, which gives a word of the firmware's memory and the value it writes there once stopped; CISP_CMD_STOP; GPR0 set and CISP_CMD_POWER_DOWN posted, answered in GPR0; then a wait for the key and for the coprocessor's WFI before its domains go off. macOS carries on if the key isn't there or a wait times out, and so does this. On the IO channel 0x1d is another command, and neither it nor the STOP after it was answered, which cost every sleep 6 s. - The interrupt: the mailbox's enable and route words zeroed, then the usual GPR1 write and a 10 us wait. - The start: the coprocessor's reset vector, which a power-off can clear, is noted at the first start and written back, with bit 0 set, where it reads lost; one latched elsewhere refuses the start. After a restart CONFIG_GET has to report the channels found at probe, or the firmware is stopped again. For a system sleep the firmware is stopped before it, and starts again in process context: its handshake in the PM resume path held up the whole resume. A stream that ran across the sleep restarts from a work item on system_freezable_wq, which runs once tasks thaw, so a stream no longer refuses the sleep; without one the firmware starts at the next stream. A stream turned off before the work runs leaves it nothing to restart, and remove cancels it. Hibernation still leaves the camera down, since the image resumes in a boot where that memory isn't the driver's. Stopped firmware that can start again is off, not dead, so a capture stop only sends its channel commands to running firmware. T8140's cold stop sends STOP as a command with its reply, as macOS does, through its own isp_cmd_stop_cold(); isp_cmd_stop() stays as it was for the other SoCs, which keep their stop and start. Signed-off-by: Diljit Singh --- .../media/platform/apple/isp/isp-cam-test.c | 56 ++++++ drivers/media/platform/apple/isp/isp-cam.c | 2 +- drivers/media/platform/apple/isp/isp-cmd.c | 55 +++++- drivers/media/platform/apple/isp/isp-cmd.h | 18 ++ drivers/media/platform/apple/isp/isp-drv.c | 75 ++++++- drivers/media/platform/apple/isp/isp-drv.h | 19 +- drivers/media/platform/apple/isp/isp-fw.c | 183 ++++++++++++++++-- drivers/media/platform/apple/isp/isp-v4l2.c | 2 + 8 files changed, 380 insertions(+), 30 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 191d8ada0d265..30fe53484b0a5 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -545,6 +545,61 @@ static void isp_test_command_ack_failures(struct kunit *test) KUNIT_EXPECT_EQ(test, state.calls, 0); } +/* The firmware's answer to the stop key request, as macOS reads it */ +static int isp_test_stop_key_reply(struct apple_isp *isp, + struct isp_channel *chan, + unsigned long timeout) +{ + struct kunit *test = kunit_get_current_test(); + unsigned int *calls = test->priv; + u8 *cmd = isp->cmd_virt; + + (*calls)++; + KUNIT_EXPECT_PTR_EQ(test, chan, isp->chan_dg); + KUNIT_EXPECT_EQ(test, chan->req.arg1, 0x20); + KUNIT_EXPECT_EQ(test, chan->req.arg2, 0x20); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(cmd + 4), 0x1d); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(cmd + 8), 1); + KUNIT_EXPECT_TRUE(test, timeout); + put_unaligned_le64(0x10001e00010ULL, cmd + 0x10); + put_unaligned_le64(0x8042006, cmd + 0x18); + chan->rsp = (struct isp_message){ .arg0 = chan->req.arg0 | ISP_IPC_FLAG_ACK, + .arg1 = chan->req.arg2 }; + return 0; +} + +static void isp_test_stop_key_on_debug(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct cmd_stop_key key; + unsigned int calls = 0; + + isp->cmd_virt = kunit_kzalloc(test, 0x400, GFP_KERNEL); + isp->chan_io = kunit_kzalloc(test, sizeof(*isp->chan_io), GFP_KERNEL); + isp->chan_dg = kunit_kzalloc(test, sizeof(*isp->chan_dg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_io); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_dg); + isp->chan_io->name = "IO"; + isp->chan_dg->name = "DEBUG"; + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_COMMAND; + test->priv = &calls; + kunit_deactivate_static_stub(test, isp_cmd_send); + kunit_activate_static_stub(test, ipc_chan_send, isp_test_stop_key_reply); + + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), 0); + KUNIT_EXPECT_EQ(test, calls, 1); + KUNIT_EXPECT_EQ(test, key.addr, 0x10001e00010ULL); + KUNIT_EXPECT_EQ(test, key.key, 0x8042006ULL); + + /* No DEBUG command ring: nothing is sent, on IO either. */ + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_REPLY; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + isp->chan_dg = NULL; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + KUNIT_EXPECT_EQ(test, calls, 1); +} + static void isp_test_call_crop(struct kunit *test) { struct isp_preset source = { @@ -661,6 +716,7 @@ static struct kunit_case isp_cam_test_cases[] = { KUNIT_CASE(isp_test_setfile_failure_stops_setup), KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failures), + KUNIT_CASE(isp_test_stop_key_on_debug), KUNIT_CASE(isp_test_sensor_geometry), KUNIT_CASE(isp_test_sensor_config_list), KUNIT_CASE(isp_test_sensor_config_failure), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 4b096aa29dddf..0f8ae7eb2f554 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -1102,7 +1102,7 @@ int apple_isp_start_capture(struct apple_isp *isp) void apple_isp_stop_capture(struct apple_isp *isp) { - if (isp->ipc_failed || isp->fw_state == ISP_FW_DEAD) + if (isp->ipc_failed || isp->fw_state != ISP_FW_RUNNING) return; isp_cmd_ch_stop(isp, 0); // TODO channel mask isp_cmd_ch_buffer_return(isp, isp->current_ch); diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 75b8704ccedcc..a1a5557111ec0 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -16,26 +16,26 @@ #define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) #define CISP_TIMEOUT msecs_to_jiffies(3000) +/* what macOS allows a DEBUG command */ +#define CISP_DEBUG_TIMEOUT msecs_to_jiffies(1000) #define CISP_SEND_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) #define CISP_SEND_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) #define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) #define CISP_POST_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, 0)) #define CISP_POST_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), 0)) -int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +/* Send through the command area on a command channel. */ +static int isp_cmd_send_chan(struct apple_isp *isp, struct isp_channel *chan, + void *args, u32 insize, u32 outsize, + unsigned long timeout) { - struct isp_channel *chan; struct isp_message *req; int err; -#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) - KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); -#endif /* Before the command area is overwritten */ err = isp_ipc_resync(isp); if (err) return err; - chan = isp->chan_io; req = &chan->req; req->arg0 = isp->cmd_iova; @@ -69,6 +69,15 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int return err; } +int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); +#endif + return isp_cmd_send_chan(isp, isp->chan_io, args, insize, outsize, + timeout); +} + static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, u32 outsize) { @@ -99,6 +108,16 @@ int isp_cmd_stop(struct apple_isp *isp, u32 mode) return CISP_SEND_IN(isp, args); } +/* T8140's cold stop, which macOS sends as a command with its reply. */ +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + int isp_cmd_power_down(struct apple_isp *isp) { struct cmd_power_down args = { @@ -133,6 +152,30 @@ int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) return CISP_SEND_INOUT(isp, args); } +/* + * macOS asks for the key on the DEBUG channel. On IO, 0x1d is another + * command, and the firmware answers neither it nor the STOP after it. + */ +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args) +{ + struct isp_channel *chan = isp->chan_dg; + int err; + + if (!chan || chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + return -ENODEV; + + memset(args, 0, sizeof(*args)); + args->opcode = CISP_OPCODE(CISP_DEBUG_CMD_STOP_KEY_GET); + args->enable = 1; + err = isp_cmd_send_chan(isp, chan, args, sizeof(*args), sizeof(*args), + CISP_DEBUG_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, sizeof(*args)); + return 0; +} + int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) { args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 9e2678bd1610c..ab98f3447d6f7 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -101,6 +101,9 @@ #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD 0xc11d #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH 0xc120 +/* Opcodes of the DEBUG channel, which has its own numbering */ +#define CISP_DEBUG_CMD_STOP_KEY_GET 0x001d + #define CISP_POOL_TYPE_META 0x0 #define CISP_POOL_TYPE_RENDERED 0x1 #define CISP_POOL_TYPE_FD 0x2 @@ -159,6 +162,19 @@ struct cmd_print_enable { } __packed; static_assert(sizeof(struct cmd_print_enable) == 0xc); +/* + * CISP_DEBUG_CMD_STOP_KEY_GET: the firmware answers with a word of its + * memory and the value it writes there once it has stopped. + */ +struct cmd_stop_key { + u64 opcode; + u32 enable; + u32 pad_c; + u64 addr; + u64 key; +} __packed; +static_assert(sizeof(struct cmd_stop_key) == 0x20); + struct cmd_trace_enable { u64 opcode; u32 enable; @@ -263,10 +279,12 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout); int isp_cmd_start(struct apple_isp *isp, u32 mode); int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode); int isp_cmd_power_down(struct apple_isp *isp); int isp_cmd_suspend(struct apple_isp *isp); int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args); int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 627a9bdb9e94d..53048806566cf 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "isp-cam.h" @@ -524,6 +525,17 @@ isp_check_firmware_version(struct apple_isp *isp) return compat; } +/* After a system sleep, see apple_isp_resume(). */ +static void apple_isp_resume_work(struct work_struct *work) +{ + struct apple_isp *isp = container_of(work, struct apple_isp, resume_work); + + mutex_lock(&isp->video_lock); + isp->fw_sleep_stopped = false; + mutex_unlock(&isp->video_lock); + apple_isp_video_resume(isp); +} + static int apple_isp_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -646,6 +658,7 @@ static int apple_isp_probe(struct platform_device *pdev) } mutex_init(&isp->video_lock); + INIT_WORK(&isp->resume_work, apple_isp_resume_work); spin_lock_init(&isp->buf_lock); init_waitqueue_head(&isp->wait); INIT_LIST_HEAD(&isp->gc); @@ -722,6 +735,7 @@ static void apple_isp_remove(struct platform_device *pdev) { struct apple_isp *isp = platform_get_drvdata(pdev); + cancel_work_sync(&isp->resume_work); apple_isp_remove_video(isp); /* Stop resident firmware before freeing what it may still use. */ apple_isp_firmware_halt(isp); @@ -953,6 +967,7 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .fw_restart = true, .meta_exposure = true, }; @@ -1168,9 +1183,43 @@ static __maybe_unused int apple_isp_runtime_resume(struct device *dev) return 0; } +/* + * Stop resident firmware. For a system sleep it starts again on resume where + * it can; otherwise, or for hibernation, the camera is gone until the next + * boot. + */ +static void apple_isp_stop_resident(struct apple_isp *isp, bool sleep) +{ + bool restart = sleep && isp->hw->fw_restart; + + if (!isp->hw->resident_fw) + return; + + mutex_lock(&isp->video_lock); + if (isp->fw_state == ISP_FW_RUNNING) { + if (restart) + dev_dbg(isp->dev, "stopping the firmware for system sleep\n"); + else + dev_warn(isp->dev, "stopping the firmware for %s, the camera is unavailable until the next boot\n", + sleep ? "system sleep" : "hibernation"); + isp->fw_sleep_stopped = restart; + } + apple_isp_firmware_halt(isp); + /* Hibernation resumes in another boot, where that memory isn't ours. */ + if (!restart) + isp->fw_state = ISP_FW_DEAD; + mutex_unlock(&isp->video_lock); +} + static __maybe_unused int apple_isp_suspend(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + bool power_lost = isp->hw->resident_fw && pm_suspend_via_firmware(); + + /* A stream that couldn't restart on the other side keeps the system up. */ + if (power_lost && !isp->hw->fw_restart && + test_bit(ISP_STATE_STREAMING, &isp->state)) + return -EBUSY; /* We must restore V4L2 context on system resume. If we were streaming * before, we (essentially) stop streaming and start streaming again. @@ -1178,9 +1227,13 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) apple_isp_video_suspend(isp); /* - * Resident firmware stays up, idle, across suspend-to-idle and - * suspend-to-RAM: its domains stay on, see apple_isp_attach_genpd(). + * Resident firmware stays up, idle, across suspend-to-idle: its domains + * stay on, see apple_isp_attach_genpd(). A sleep that powers the SoC down + * takes it anyway, so stop it first. */ + if (power_lost) + apple_isp_stop_resident(isp, true); + return 0; } @@ -1196,13 +1249,7 @@ static __maybe_unused int apple_isp_freeze(struct device *dev) * that started it. Stop it cleanly instead; if it ran, the camera is * then unavailable until the next system boot. */ - if (isp->hw->resident_fw) { - mutex_lock(&isp->video_lock); - if (isp->fw_state == ISP_FW_RUNNING) - dev_warn(dev, "stopping the firmware for hibernation, the camera is unavailable until the next boot\n"); - apple_isp_firmware_halt(isp); - mutex_unlock(&isp->video_lock); - } + apple_isp_stop_resident(isp, false); return 0; } @@ -1211,6 +1258,16 @@ static __maybe_unused int apple_isp_resume(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + /* + * Firmware stopped for the sleep starts again in process context: its + * handshake here held up the whole resume. A stream that ran across the + * sleep restarts from a freezable work item, once tasks thaw; without one + * the firmware starts at the next stream. + */ + if (isp->fw_sleep_stopped) { + queue_work(system_freezable_wq, &isp->resume_work); + return 0; + } apple_isp_video_resume(isp); return 0; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 9056c3d7af143..fe60a6ee8a1ee 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -238,12 +239,15 @@ struct apple_isp_hw { bool fw_restore; /* - * The firmware can be started only once per system boot, so once - * started it is kept running while the driver is bound. It starts at - * probe, or at the first stream with deferred_start. + * The firmware isn't started for each stream: once started it is kept + * running while the driver is bound. It starts at probe, or at the + * first stream with deferred_start. */ bool resident_fw; + /* Resident firmware that starts again after a stop, from its own memory */ + bool fw_restart; + /* * Start the firmware, and detect the camera, at the first stream * instead of at probe, so the ISP stays off until then. The firmware @@ -390,6 +394,10 @@ struct apple_isp { u64 heap_base; u64 heap_top; u64 data_top; /* 0 unless the bootloader loads the data */ + /* the coprocessor's reset vector at the first start */ + u64 entry; + /* the coprocessor has run since probe */ + bool started; } fw; /* changes under video_lock once the video device is registered */ enum isp_fw_state fw_state; @@ -398,6 +406,11 @@ struct apple_isp { /* configurations the firmware has, once a deferred start detected them */ u32 num_fw_presets; + /* stopped for system sleep, to start again on resume */ + bool fw_sleep_stopped; + /* restarts a stream that ran across that sleep, see apple_isp_resume() */ + struct work_struct resume_work; + struct isp_surf *ipc_surf; struct isp_surf *extra_surf; struct isp_surf *data_surf; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 28727ef24fbb6..ac4e930ac3929 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -7,7 +7,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -29,6 +32,12 @@ #define ISP_COPROC_IN_WFI 0x3 +/* the waits for firmware that has been told to power down */ +#define ISP_STOP_KEY_TRIES 20 +#define ISP_STOP_KEY_MS 10 +#define ISP_STOP_WFI_POLL_US (10 * USEC_PER_MSEC) +#define ISP_STOP_WFI_US (190 * USEC_PER_MSEC) + static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) { return readl(isp->coproc + reg); @@ -254,9 +263,11 @@ static void isp_disable_irq(struct apple_isp *isp) if (isp->hw->mbox_irq_route) { isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, 0x0); isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, 0x0); + isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, 0x0); } free_irq(isp->irq, isp); isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ + fsleep(10); } static int isp_enable_irq(struct apple_isp *isp) @@ -414,6 +425,39 @@ static void isp_restore_fw_memory(struct apple_isp *isp) clear_highpage(tail->pages[i]); } +/* + * The coprocessor starts at its reset vector, which the bootloader sets and + * which bit 0 latches. Note it at the first start; where a power-off cleared + * it, set it again. + */ +static int isp_coproc_entry(struct apple_isp *isp) +{ + struct isp_firmware *fw = &isp->fw; + u64 rvbar = readq(isp->coproc + ISP_COPROC_RVBAR); + + if (!fw->started) { + fw->entry = rvbar & ~1ULL; + dev_dbg(isp->dev, "firmware reset vector %#llx\n", rvbar); + return 0; + } + + if (!fw->entry || rvbar == (fw->entry | 1)) + return 0; + + if (rvbar & 1) { + isp_err(isp, "reset vector %#llx latched, not the firmware's %#llx\n", + rvbar, fw->entry); + return -EIO; + } + + writeq(fw->entry | 1, isp->coproc + ISP_COPROC_RVBAR); + dev_dbg(isp->dev, "reset vector was %#llx, set to %#llx\n", rvbar, + readq(isp->coproc + ISP_COPROC_RVBAR)); + + return (readq(isp->coproc + ISP_COPROC_RVBAR) & ~1ULL) == fw->entry ? + 0 : -EIO; +} + static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { /* Only a coprocessor that was started has to settle and quiesce */ @@ -476,6 +520,12 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) if (err < 0) goto shutdown; + if (isp->hw->fw_restart) { + err = isp_coproc_entry(isp); + if (err < 0) + goto shutdown; + } + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); isp_gpio_write32(isp, ISP_GPIO_1, 0x0); isp_gpio_write32(isp, ISP_GPIO_2, 0x0); @@ -489,6 +539,7 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); isp_coproc_write32(isp, isp_coproc_control(isp), 0x10); + isp->fw.started = true; /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { @@ -886,10 +937,92 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) return 0; } +/* Wait for the word of its own memory the firmware sets once it has stopped. */ +static void isp_wait_stop_key(struct apple_isp *isp, u64 addr, u32 key) +{ + u64 iova = isp_fw_iova(isp, addr); + phys_addr_t phys = 0; + u32 *word = NULL; + + if (iova < isp_fw_iova(isp, isp->fw.heap_top)) + phys = iommu_iova_to_phys(isp->domain, iova); + if (phys) + word = memremap(phys, sizeof(*word), MEMREMAP_WC); + if (!word) { + dev_warn(isp->dev, "stop key at 0x%llx isn't in firmware memory\n", + addr); + return; + } + + for (int i = 0; i < ISP_STOP_KEY_TRIES && READ_ONCE(*word) != key; i++) + msleep(ISP_STOP_KEY_MS); + + if (READ_ONCE(*word) != key) + dev_warn(isp->dev, "stop key at 0x%llx reads 0x%x, not 0x%x\n", + addr, READ_ONCE(*word), key); + + memunmap(word); +} + +/* + * Stop firmware that starts cold next time. CISP_CMD_SUSPEND is for firmware + * that resumes: it isn't followed by a cold start. + */ +static int isp_stop_firmware_cold(struct apple_isp *isp) +{ + struct cmd_stop_key stop_key = {}; + ktime_t start = ktime_get(); + u32 status; + int err; + + /* Without the key the rest of the stop still works. */ + if (isp_cmd_stop_key_get(isp, &stop_key)) + stop_key.addr = 0; + + err = isp_cmd_stop_cold(isp, 0); + if (err) { + isp_err(isp, "isp_cmd_stop_cold() failed\n"); + return err; + } + + /* The firmware answers the power-down in this word, not on the ring. */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + err = isp_cmd_power_down(isp); + if (err) { + isp_err(isp, "isp_cmd_power_down() failed\n"); + return err; + } + + if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + if (stop_key.addr) + isp_wait_stop_key(isp, stop_key.addr, stop_key.key); + + /* Its domains go off next, so its CPU should be idle by now. */ + if (readl_poll_timeout(isp->coproc + ISP_COPROC_STATUS_T8140, status, + status & ISP_COPROC_IN_WFI, ISP_STOP_WFI_POLL_US, + ISP_STOP_WFI_US)) + dev_warn(isp->dev, "coprocessor not in WFI (status: 0x%x)\n", + status); + else + dev_dbg(isp->dev, "firmware stopped in %lld ms (key at 0x%llx, status 0x%x)\n", + ktime_ms_delta(ktime_get(), start), stop_key.addr, + status); + + return 0; +} + static int isp_stop_command_processor(struct apple_isp *isp) { int res, ret; + if (isp->hw->fw_restart) + return isp_stop_firmware_cold(isp); + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); /* @@ -1083,26 +1216,48 @@ static void isp_firmware_shutdown(struct apple_isp *isp) /* * Stop the coprocessor before releasing any memory the firmware - * uses, also when it did not acknowledge the suspend request. + * uses, also when it did not acknowledge the stop request. */ if (isp_stop_command_processor(isp)) - dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); + dev_warn(isp->dev, "firmware did not stop, stopping the coprocessor\n"); isp_disable_irq(isp); isp_firmware_stop(isp); } +/* + * After a restart, the firmware must find the camera found at probe. The + * query also shows that it takes commands. + */ +static int isp_check_restarted(struct apple_isp *isp) +{ + struct cmd_config_get args = {}; + int err; + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + if (min_t(u32, args.num_channels, ISP_MAX_CHANNELS) != isp->num_channels) { + isp_err(isp, "firmware found %u channels after a restart, %d before\n", + args.num_channels, isp->num_channels); + return -ENODEV; + } + + return 0; +} + /* * Resident firmware (hw->resident_fw) is loaded by the bootloader and - * does not start again once it has been stopped: after CISP_CMD_SUSPEND, - * a coprocessor reset and a power cycle of its domains it never completes - * the first handshake. It is booted once, at probe or with - * deferred_start at the first stream, and runs until the - * driver is unbound, probe fails or the system hibernates, also across - * suspend; after it has stopped the device stays unusable until the next - * system boot. + * changes its own memory as it runs. It is booted at probe, or with + * deferred_start at the first stream, and runs until the driver is unbound, + * probe fails, the system hibernates or a sleep powers the SoC down; + * suspend-to-idle keeps it running. With hw->fw_restart it starts again, and + * rebuilds that memory itself; other stopped firmware stays down until the + * next system boot. */ int apple_isp_firmware_boot(struct apple_isp *isp) { + bool restart = isp->hw->resident_fw && isp->fw.started; int err; switch (isp->fw_state) { @@ -1129,6 +1284,11 @@ int apple_isp_firmware_boot(struct apple_isp *isp) } err = isp_firmware_boot(isp); + if (!err && restart) { + err = isp_check_restarted(isp); + if (err) + isp_firmware_shutdown(isp); + } if (err) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); pm_runtime_put_sync(isp->dev); @@ -1161,7 +1321,7 @@ void apple_isp_firmware_shutdown(struct apple_isp *isp) isp_free_released_surfaces(isp); } -/* Stop the firmware; resident firmware cannot be started again. */ +/* Stop the firmware; resident firmware without hw->fw_restart stays down. */ void apple_isp_firmware_halt(struct apple_isp *isp) { /* Only a capture services the watchdog, but be sure. */ @@ -1173,5 +1333,6 @@ void apple_isp_firmware_halt(struct apple_isp *isp) isp_firmware_shutdown(isp); pm_runtime_put_sync(isp->dev); isp->ipc_stale = NULL; - isp->fw_state = isp->hw->resident_fw ? ISP_FW_DEAD : ISP_FW_OFF; + isp->fw_state = isp->hw->resident_fw && !isp->hw->fw_restart ? + ISP_FW_DEAD : ISP_FW_OFF; } diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 81964ab1916c9..f6df02e6fe8df 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -611,6 +611,8 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) { struct apple_isp *isp = vb2_get_drv_priv(q); + /* A restart still due from a sleep has nothing left to restart. */ + clear_bit(ISP_STATE_SLEEPING, &isp->state); apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); v4l2_ctrl_grab(isp->exposure_priority, false); From c922a676a197ce0a948b258211d95eaa151a058d Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:26 -0500 Subject: [PATCH 53/73] drm/apple: stop and restart the T8140 display processors for a deep sleep A deep sleep powers both display coprocessors off, and the sleep engine waits on DISP_CPU with no timeout, so the firmware has to be stopped through its driver and its domains lowered before the commit. Each step below is what macOS does, measured under aura: - The internal DCP's stop: set_power_state(0) in band after A414(0), A478 x2, A476(1), A478, A412; D572(0) from the firmware turns the display DART's streams off; A500; RTKit idle with the secure RTBuddy port answered between AP and IOP idle; the idle and stopped status bits; the mailbox off and the run bit cleared twice; the DCP DART's streams off with its GAPF slice released. A refused idle is retried once after draining what's queued. If it's refused again the firmware keeps running with the display off, and DPMS on lights it there; the run bit is never cleared under running firmware. - The external DCP's stop: A500, then about 4 ms later AP and IOP idle and the run bit, through the shared RTKit stop. An attached display refuses the sleep, since nothing brings it back yet. - The secure RTBuddy port. The DCP's firmware takes IOP idle only with its secure port answered, so the driver answers its HELLO and endpoint map at bind and again after a warm restart, between the main port's IOP report and its AP on. Its interrupts aren't in the DT, so the rest is polled: every jiffy for a second after its start or a message, otherwise every 100 ms, as a CPU woken every jiffy never reaches deep idle. - OSLOG counters. A DCP restarted warm holds each log counter until the last is echoed and refuses its next AP idle with one outstanding, so both DCPs have them echoed, as macOS does. - Display on after a sleep: the domains back up in macOS's order, the DART protected again, the warm restart, then the named calls and set_power_state(1) out of band. D572(1) turns the scanout DART back on. If that fails the panel stays dark until reboot and the DCP is treated as crashed. DPMS on the internal panel now does the same power cycle as macOS's display sleep: the backlight to 0, the firmware stopped and the display domains off, and the reverse on wake. Not during a system sleep, which stops the firmware itself. While the display is off its DART's registers aren't touched. apple-drm doesn't refuse a suspend whose display firmware is already stopped, a commit that leaves a crashed DCP's CRTC off passes, and DP audio refuses a sleep with a stream open. No mode fits a crashed DCP, so after a refused sleep that left one stopped the resume restores its CRTC disabled and userspace sets it again on the hotplug, rather than the commit failing with -22 and every output staying dark. A DPMS power-on that fails after the warm restart has raised the display domains and set the run bit leaves the firmware running while the driver takes it as stopped; the next sleep would skip the stop, find a bound display coprocessor running, and be refused, as every later one. On a J700 a power-on cut short as a sleep began did this at the 27th sleep in a row. So the driver tracks whether it raised the domains since it last lowered them, and clears the run bit of a DCP taken as stopped that has them up and running before the sleep, whose warm restart then brings it back. Only T8140 with H17P firmware takes these paths. Signed-off-by: Diljit Singh --- drivers/gpu/drm/apple/apple_drv.c | 30 +- drivers/gpu/drm/apple/audio.c | 19 +- drivers/gpu/drm/apple/dcp-internal.h | 29 + drivers/gpu/drm/apple/dcp.c | 855 +++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 6 +- drivers/gpu/drm/apple/iomfb.h | 2 + drivers/gpu/drm/apple/iomfb_h17p.c | 18 +- drivers/gpu/drm/apple/iomfb_template.c | 115 +++ drivers/gpu/drm/apple/iomfb_template.h | 4 + .../drivers/gpu/apple-dcp/lifecycle.py | 2 + 11 files changed, 1071 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 763ba82ad29ac..0fcc436c796df 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -64,6 +64,8 @@ struct apple_drm_private { * the screen, the DRM device stays unregistered and its DCPs running. */ bool aborted; + /* This suspend found every firmware already stopped. */ + bool s2r_skipped; }; /* Component readiness is later than platform probe/bind. */ @@ -1691,12 +1693,34 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); #ifdef CONFIG_PM_SLEEP +/* + * Left stopped by a suspend that never reached the commit. Nothing can commit to + * them, and a refused sleep has to be able to try again. + */ +static bool apple_drm_firmware_stopped(struct apple_drm_private *apple) +{ + struct drm_crtc *crtc; + bool stopped = false; + + drm_for_each_crtc(crtc, &apple->drm) { + if (t8132_ext_crtc(crtc) || + !dcp_s2r_firmware_stopped(to_apple_crtc(crtc)->dcp)) + return false; + stopped = true; + } + + return stopped; +} + static int apple_platform_suspend(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple && !apple->aborted) - return drm_mode_config_helper_suspend(&apple->drm); + if (apple && !apple->aborted) { + apple->s2r_skipped = apple_drm_firmware_stopped(apple); + if (!apple->s2r_skipped) + return drm_mode_config_helper_suspend(&apple->drm); + } return 0; } @@ -1723,7 +1747,7 @@ static int apple_platform_resume(struct device *dev) struct apple_drm_private *apple = dev_get_drvdata(dev); u32 dropped = 0; - if (apple && !apple->aborted) { + if (apple && !apple->aborted && !apple->s2r_skipped) { if (apple->drm.mode_config.suspend_state) dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); /* Have userspace set the mode of a CRTC resumed disabled again. */ diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 44c228deb2a89..92b37fbcdf788 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1574,7 +1575,23 @@ static __maybe_unused int dcpaud_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(dcpaud_pm_ops, dcpaud_suspend, dcpaud_resume, NULL); +/* The SIO channel and the sink's link both end in a sleep that reaches the hardware. */ +static __maybe_unused int dcpaud_system_suspend(struct device *dev) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + + if (pm_suspend_via_firmware() && dcpaud->substream && dcpaud->substream->ref_count) { + dev_err(dev, "a stream is open, refusing the sleep\n"); + return -EBUSY; + } + + return pm_runtime_force_suspend(dev); +} + +static const struct dev_pm_ops dcpaud_pm_ops = { + SYSTEM_SLEEP_PM_OPS(dcpaud_system_suspend, pm_runtime_force_resume) + RUNTIME_PM_OPS(dcpaud_suspend, dcpaud_resume, NULL) +}; static const struct of_device_id dcpaud_of_match[] = { { .compatible = "apple,dpaudio" }, diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5b9f50b60d300..aedf7ae756caf 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -361,6 +361,8 @@ const struct drm_edid *dcp_panel_edid_snapshot(struct apple_dcp *dcp, unsigned int count, u64 *generation); bool dcp_panel_metadata_current(struct apple_dcp *dcp, u64 generation); +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on); + struct apple_dcp { /* Generation-specific wire/transaction state, sharing the common KMS owner. */ struct iomfb_m4 *v26_6; @@ -626,6 +628,33 @@ struct apple_dcp { /* Messages from the coprocessor are held in the mailbox for sleep. */ bool rx_held; + /* T8140 suspend-to-RAM stops the firmware, see dcp_s2r_suspend(). */ + bool s2r_active; + bool s2r_stopped; + /* DPMS off stopped the firmware and lowered the domains; DPMS on brings them back. */ + bool dpms_powered_off; + /* Refused its idle with the display off: the firmware still runs, the display is dark. */ + bool s2r_refused_off; + /* The firmware's secure mailbox, where it has one. */ + struct apple_rtkit *secure_rtk; + bool secure_answered; + bool secure_polling; + /* Polled every jiffy until then, see dcp_secure_poll(). */ + unsigned long secure_fast_until; + int secure_map_ret; + s64 secure_map_us; + struct delayed_work secure_poll_wq; + /* Stopped and not restarted, which outlives a suspend that gets refused. */ + bool s2r_fw_down; + /* This driver raised the display domains, and nothing has lowered them since. */ + bool s2r_domains_up; + bool s2r_power_on; + bool s2r_power_oob; + u8 s2r_step; + struct completion s2r_dart_done; + struct device *s2r_disp_dart; + struct device *s2r_dcp_dart; + /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f56c4423ea247..943ba91991c68 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -18,13 +18,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -144,6 +147,135 @@ static void dcp_release_context(void *data) kfree(dcp); } +#define DCP_SECURE_EP 0x30 +#define DCP_SECURE_TIMEOUT_MS 200 + +/* + * The firmware's secure mailbox. No other software answers it under Linux, so the + * driver does, as macOS's secure side does: 3 is answered with 2, and 4 needs nothing. + * What these messages mean isn't known. + */ +static void dcp_secure_recv(void *cookie, u8 ep, u64 msg) +{ + struct apple_dcp *dcp = cookie; + int ret; + + dev_dbg(dcp->dev, "secure port: ep %#x msg %#llx\n", ep, msg); + if (ep != DCP_SECURE_EP || (msg >> 60) != 3) + return; + + ret = apple_rtkit_send_message(dcp->secure_rtk, ep, 2ULL << 60, NULL, false); + if (ret) + dev_warn(dcp->dev, "secure port: no reply sent: %d\n", ret); +} + +static const struct apple_rtkit_ops dcp_secure_rtkit_ops = { + .recv_message = dcp_secure_recv, +}; + +/* + * The port's interrupts aren't in the DT. Its handshakes, idle and restart poll it themselves; + * this answers the rest. Every jiffy for a second after its start or a message, as more tend to + * follow, otherwise every 100 ms: woken every jiffy, an idle CPU never reaches deep idle. + */ +#define DCP_SECURE_POLL_FAST_MS 1000 +#define DCP_SECURE_POLL_SLOW_MS 100 + +static void dcp_secure_poll(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + secure_poll_wq); + bool fast = time_before(jiffies, dcp->secure_fast_until); + + if (!READ_ONCE(dcp->secure_polling)) + return; + + if (apple_rtkit_poll(dcp->secure_rtk) > 0) { + if (!fast) + dev_info_once(dcp->dev, "secure port: a message while idle\n"); + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + fast = true; + } + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, + fast ? 1 : msecs_to_jiffies(DCP_SECURE_POLL_SLOW_MS)); +} + +static void dcp_secure_poll_start(struct apple_dcp *dcp) +{ + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + WRITE_ONCE(dcp->secure_polling, true); + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, 0); +} + +static void dcp_secure_poll_stop(struct apple_dcp *dcp) +{ + WRITE_ONCE(dcp->secure_polling, false); + cancel_delayed_work_sync(&dcp->secure_poll_wq); +} + +/* After the firmware's HELLO on its main mailbox. */ +static void dcp_secure_port_start(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->secure_rtk) + return; + + ret = apple_rtkit_port_handshake(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) { + dev_warn(dcp->dev, "secure port handshake not finished: %d\n", ret); + return; + } + dcp->secure_answered = true; + dcp_secure_poll_start(dcp); + dev_info(dcp->dev, "secure port up\n"); +} + +/* Between the main mailbox's AP idle and its IOP idle. */ +static void dcp_secure_port_idle(void *data) +{ + struct apple_dcp *dcp = data; + int ret; + + if (!dcp->secure_rtk || !dcp->secure_answered) + return; + + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_idle(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP idle not acknowledged: %d\n", ret); +} + +static void dcp_release_secure_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + dcp_secure_poll_stop(dcp); + apple_rtkit_free(dcp->secure_rtk); + dcp->secure_rtk = NULL; +} + +/* Nothing is read from the port until its handshake is wanted. */ +static int dcp_secure_port_init(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + + if (of_property_match_string(dev->of_node, "mbox-names", "secure") < 0) + return 0; + + dcp->secure_rtk = apple_rtkit_init(dev, dcp, "secure", 0, &dcp_secure_rtkit_ops); + if (IS_ERR(dcp->secure_rtk)) { + int ret = PTR_ERR(dcp->secure_rtk); + + dcp->secure_rtk = NULL; + return dev_err_probe(dev, ret, "Failed to initialize the secure port\n"); + } + + INIT_DELAYED_WORK(&dcp->secure_poll_wq, dcp_secure_poll); + + return devm_add_action_or_reset(dev, dcp_release_secure_rtkit, dcp); +} + static void dcp_release_rtkit(void *data) { struct apple_dcp *dcp = data; @@ -1517,11 +1649,16 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) return iomfb_v14_7_atomic_check(dcp, crtc, commit); - if (dcp->crashed) - return -EINVAL; - crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + /* + * A crashed DCP can't be lit, but a commit that leaves its CRTC off has to pass: + * the system suspend's disable-all includes every CRTC, and a stopped external DCP + * would refuse every sleep after the first. + */ + if (dcp->crashed) + return crtc_state && crtc_state->enable ? -EINVAL : 0; + /* External H17P does not program adaptive sync. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && @@ -3809,6 +3946,7 @@ int dcp_start(struct platform_device *pdev) /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ init_completion(&dcp->start_done); + init_completion(&dcp->s2r_dart_done); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) return iomfb_v26_6_start(dcp); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) @@ -4112,6 +4250,9 @@ static bool dcp_h17g_panel(struct apple_dcp *dcp) dcp_has_panel(dcp); } +static void dcp_dpms_power_off(struct apple_dcp *dcp); +static void dcp_dpms_power_on(struct apple_dcp *dcp); + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { /* The 14.7 panel has its own known-level blank and restore hooks. */ @@ -4181,6 +4322,7 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { + dcp_dpms_power_on(dcp); if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { @@ -4327,11 +4469,15 @@ void dcp_poweroff(struct platform_device *pdev) cancel_delayed_work(&dcp->swap_watchdog_wq); - /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + /* + * Internal H17P DPMS is a brightness present, never a pipe stop. On T8140 the + * display then powers off as well, after the backlight. + */ if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + dcp_dpms_power_off(dcp); return; } @@ -5418,6 +5564,13 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; + /* After a warm restart its log counters need echoing, or it refuses its next idle. */ + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcp") || + of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) + apple_rtkit_set_oslog_echo(dcp->rtk, true); + ret = dcp_secure_port_init(dcp); if (ret) goto err_piodma; @@ -5436,6 +5589,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } + dcp_secure_port_start(dcp); enable_work(&dcp->dimensions_wq); schedule_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); @@ -6666,6 +6820,691 @@ static void dcp_platform_complete(struct device *dev) mutex_unlock(&dcp->hpd_mutex); } +/* + * T8140 suspend-to-RAM: the sleep engine waits for the DCP's domain to be off, so + * the firmware has to stop first. Measured from macOS, in order: + * + * 1. A414(0), A478 x2, A476(1), A478, A412, A473(0). + * 2. D572(0) from the firmware: the display DART's streams go off. + * 3. A500, unnamed, then D107s and D575(0) from the firmware. + * 4. RTKit idle, then the ASC's run bit off. + * 5. The DCP DART's streams off. That's the coprocessor's own shutdown, apart + * from step 2. The sleep driver then lowers the domains. + * + * macOS also restarts the firmware 893 ms before the commit, calls A478, A473(0), + * A115, A500, and stops it again. Why isn't known, and that's not done here. + * + * The firmware takes IOP idle only with its secure mailbox answered, see + * dcp_s2r_answer_secure_port(). + */ +#define DCP_S2R_DISP_STREAMS (BIT(0) | BIT(3) | BIT(4) | BIT(16)) +#define DCP_S2R_DCP_STREAMS (BIT(23) | BIT(24)) +#define DCP_S2R_D572_TIMEOUT_MS 1000 /* macOS: about 170 ms after A473(0) */ + +static struct device *dcp_s2r_find_dart(struct device_node *np) +{ + struct device_node *dart; + struct platform_device *pdev; + + dart = of_parse_phandle(np, "iommus", 0); + if (!dart) + return NULL; + pdev = of_find_device_by_node(dart); + of_node_put(dart); + + return pdev ? &pdev->dev : NULL; +} + +static void dcp_s2r_put_darts(struct apple_dcp *dcp) +{ + if (dcp->s2r_dcp_dart) + put_device(dcp->s2r_dcp_dart); + if (dcp->s2r_disp_dart) + put_device(dcp->s2r_disp_dart); + dcp->s2r_dcp_dart = NULL; + dcp->s2r_disp_dart = NULL; +} + +static void dcp_s2r_find_darts(struct apple_dcp *dcp) +{ + struct device_node *piodma; + + /* A suspend that failed after an earlier lookup still holds those. */ + dcp_s2r_put_darts(dcp); + dcp->s2r_dcp_dart = dcp_s2r_find_dart(dcp->dev->of_node); + piodma = of_get_child_by_name(dcp->dev->of_node, "piodma"); + if (piodma) { + dcp->s2r_disp_dart = dcp_s2r_find_dart(piodma); + of_node_put(piodma); + } +} + +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on) +{ + if (!READ_ONCE(dcp->s2r_active)) + return; + + if (dcp->s2r_disp_dart) + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, on, false); + complete(&dcp->s2r_dart_done); +} + +static bool dcp_s2r_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcp"); +} + +static void dcp_s2r_cpu_stop(struct apple_dcp *dcp) +{ + /* macOS writes it twice. */ + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); +} + +/* Up from bind, unless its handshake failed then. A bound firmware needs it to take IOP idle. */ +static void dcp_s2r_answer_secure_port(struct apple_dcp *dcp) +{ + if (!dcp->secure_answered) + dcp_secure_port_start(dcp); +} + +/* + * macOS's order up to RTKit idle. On success the firmware is idle and its run bit + * is still set. + */ +static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) +{ + ktime_t start = ktime_get(); + bool display_off = false; + s64 power_ms; + bool idle; + int ret; + + reinit_completion(&dcp->s2r_dart_done); + dcp_s2r_find_darts(dcp); + WRITE_ONCE(dcp->s2r_active, true); + + /* In band, 3 of 3 on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, false, true, false); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(0): %d\n", ret); + goto err; + } + power_ms = ktime_ms_delta(ktime_get(), start); + + /* Without D572 the display isn't known to be off, and nothing below can undo this. */ + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) { + dev_err(dcp->dev, "no D572 %lld ms after set_power_state(0)\n", + ktime_ms_delta(ktime_get(), start)); + ret = -ETIMEDOUT; + goto err; + } + dev_dbg(dcp->dev, "display off: set_power_state(0) %lld ms, D572 %lld ms\n", power_ms, + ktime_ms_delta(ktime_get(), start)); + display_off = true; + + /* D572 itself is still being answered, and a busy channel can't be sent on. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before A500\n"); + + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + goto err; + } + dev_dbg(dcp->dev, "A500 replied after %lld ms\n", ktime_ms_delta(ktime_get(), start)); + + /* The firmware's callbacks after A500 have to be answered first. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before idle\n"); + + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); + if (ret) { + /* Answer what it still has queued, its log counters among them, and ask again. */ + dev_warn(dcp->dev, "firmware refused idle after %lld ms: %d, control %#x status %#x: retrying once\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + apple_rtkit_poll(dcp->rtk); + msleep(30); + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); + } + if (ret) { + dev_err(dcp->dev, "firmware did not go idle after %lld ms: %d, control %#x status %#x\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + goto err; + } + + /* Both set once the firmware's last message has been read; macOS waits for them. */ + if (readl_poll_timeout(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS, *status, + (*status & (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) == + (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED), 100, 5000)) + dev_warn(dcp->dev, "idle status not set: %#x\n", *status); + dev_dbg(dcp->dev, "firmware idle after %lld ms, status %#x\n", + ktime_ms_delta(ktime_get(), start), *status); + + WRITE_ONCE(dcp->s2r_active, false); + + return 0; + +err: + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); + /* + * Refused with the display off: leave the firmware running, never stop it under its + * own feet. DPMS on lights the display again on the running firmware. + */ + if (display_off) { + dcp->s2r_refused_off = true; + dev_warn(dcp->dev, "display left off with the firmware running\n"); + return ret; + } + /* The display may be off and the firmware half idle. Nothing here brings it back. */ + WRITE_ONCE(dcp->crashed, true); + + return ret; +} + +/* Its power state can only be lowered once this has finished. */ +static int dcp_s2r_stop_streams(struct apple_dcp *dcp) +{ + if (!dcp->s2r_dcp_dart) + return -ENODEV; + + return apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, false, true); +} + +static int dcp_s2r_stop_firmware(struct apple_dcp *dcp) +{ + u32 status; + int ret; + + ret = dcp_s2r_quiesce(dcp, &status); + if (ret) + return ret; + + apple_rtkit_mailbox_off(dcp->rtk); + dcp_s2r_cpu_stop(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + return dcp_s2r_stop_streams(dcp); +} + +/* + * macOS stops this firmware with one call the size of A500, then about 4 ms later AP + * then IOP idle and the run bit off twice. Nothing has shown it needing a secure + * port. Its panel path is already off with no display attached. Nothing brings it + * back yet, so an attached display refuses the sleep. + */ +static bool dcp_s2r_ext_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); +} + +static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) +{ + int ret; + + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + ret = dcp_s2r_stop_streams(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + device_set_wakeup_path(dcp->dev); + return 0; + } + + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) { + dev_err(dcp->dev, "a display is attached, refusing sleep\n"); + return -EBUSY; + } + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dcp_enable_typec_work(dcp); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + WRITE_ONCE(dcp->crashed, true); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + usleep_range(4000, 5000); + + ret = apple_rtkit_sleep_stop(dcp->rtk, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (ret) { + dev_err(dcp->dev, "firmware did not go idle: %d, control %#x status %#x\n", ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + WRITE_ONCE(dcp->crashed, true); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + ret = dcp_s2r_stop_streams(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + + device_set_wakeup_path(dcp->dev); + + return 0; +} + +/* + * Taken as stopped, but a start since raised its domains and failed after the run bit, + * as one a signal ended. Left running, it refuses every sleep; stopped, it comes back + * from the sleep's power cycle like any other. + */ +static void dcp_s2r_stop_failed_start(struct apple_dcp *dcp) +{ + if (!dcp->s2r_domains_up || + !(readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL) & + APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return; + dev_warn(dcp->dev, "a failed start left the firmware running: stopping it for the sleep\n"); + dcp_s2r_cpu_stop(dcp); +} + +/* A failure before the firmware has stopped leaves it running, and refuses cleanly. */ +static int dcp_s2r_suspend(struct apple_dcp *dcp) +{ + int ret; + + /* Still stopped from a suspend that never reached the commit. */ + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + ret = dcp_s2r_stop_streams(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + device_set_wakeup_path(dcp->dev); + return 0; + } + + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + dcp_s2r_answer_secure_port(dcp); + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* Presentation stays paused if this fails: see dcp_s2r_quiesce(). */ + ret = dcp_s2r_stop_firmware(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static void dcp_s2r_ps_lower(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & BIT(28)) { + val &= ~(BIT(28) | GENMASK(9, 8)); + writel(val, reg); + } + writel((val & ~GENMASK(3, 0)) | GENMASK(9, 8), reg); +} + +/* macOS's power-off, measured 3 of 3. The firmware is stopped and its DARTs are off. */ +static int dcp_s2r_domains_off(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + dcp->s2r_domains_up = false; + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + writel(0, pmc); + val = readl(ps + 0x10000); + if (val & BIT(28)) + writel(val & ~(BIT(28) | GENMASK(9, 8)), ps + 0x10000); + writel(3, pwrgate + 0xc8c); + writel(0, pwrgate + 0xcd0); + dcp_s2r_ps_lower(ps + 0x10000); + dcp_s2r_ps_lower(ps + 0x318); + dcp_s2r_ps_lower(ps + 0x280); + + ret = readl_poll_timeout(ps + 0x10000, val, !FIELD_GET(GENMASK(7, 4), val), 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power off: %#x\n", val); + dev_dbg(dcp->dev, "display domains off: CPU %#x FE %#x SYS %#x\n", readl(ps + 0x10000), + readl(ps + 0x318), readl(ps + 0x280)); +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * Exact mirror of the writes in macOS's power-off, measured 3 of 3. DISP_SYS and + * DISP_FE first, then DISP_CPU with AUTO_ENABLE last. + */ +static int dcp_s2r_domains_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + val = readl(ps + 0x280) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x280); + writel(val | 0xf | BIT(28), ps + 0x280); + val = readl(ps + 0x318) & ~(GENMASK(9, 8) | GENMASK(3, 0)); + writel(val | 0xf, ps + 0x318); + val = readl(ps + 0x10000) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x10000); + writel(0, pwrgate + 0xc8c); + writel(1, pwrgate + 0xcd0); + writel(val | 0xf | BIT(28), ps + 0x10000); + writel(1, pmc); + + ret = readl_poll_timeout(ps + 0x10000, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power on: %#x\n", val); + else + dcp->s2r_domains_up = true; +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* Between the main mailbox's IOP report and its AP on, measured 2 of 2. */ +static void dcp_secure_port_map(void *data) +{ + struct apple_dcp *dcp = data; + ktime_t start = ktime_get(); + + dcp->secure_map_ret = apple_rtkit_port_map(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + dcp->secure_map_us = ktime_us_delta(ktime_get(), start); +} + +static int dcp_s2r_warm_restart(struct apple_dcp *dcp) +{ + int ret; + + ret = dcp_s2r_domains_on(dcp); + if (ret) + return ret; + + /* After the domains, whose writes it would reject, the DART protects them again. */ + if (dcp->s2r_dcp_dart) + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + + if (dcp->secure_rtk) { + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_restart(dcp->secure_rtk); + if (ret) + return ret; + } + + ret = apple_rtkit_warm_restart(dcp->rtk); + if (ret) + return ret; + + writel(APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + ret = apple_rtkit_boot_with(dcp->rtk, dcp->secure_rtk ? dcp_secure_port_map : NULL, dcp); + if (ret) + return ret; + + if (dcp->secure_rtk) { + ret = dcp->secure_map_ret ?: + apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port handshake after restart: %d\n", ret); + else + dev_dbg(dcp->dev, "secure port up after restart, map took %lld us\n", + dcp->secure_map_us); + dcp_secure_poll_start(dcp); + } + + return 0; +} + +/* + * Display on after a sleep, and after DPMS off: the warm restart, then the named calls + * macOS sends first, A414(0), A478 x2, A476(1), A478, A412, then A473(1). + * + * Not sent: A439, which macOS overlaps with A473(1), A438 around swaps, A461 per frame + * and A352. They stay unnamed, and A439 and A438 hold macOS kernel pointers. Linux + * already lights the panel without A438 at boot. + */ +static int dcp_s2r_display_on(struct apple_dcp *dcp) +{ + int ret; + + ret = dcp_s2r_warm_restart(dcp); + if (ret) { + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + return ret; + } + + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + + /* Out of band; either channel answered on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(1): %d\n", ret); + goto out; + } + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); +out: + WRITE_ONCE(dcp->s2r_active, false); + + return ret; +} + +/* + * If the display doesn't come back, the panel stays dark until reboot, and the driver + * must not talk to the stopped firmware, so treat it as a crashed DCP. Nothing else in + * the system depends on it. + */ +static int dcp_s2r_resume(struct apple_dcp *dcp) +{ + int ret = -EAGAIN; + + dcp->s2r_stopped = false; + /* A real sleep took the domains' power. */ + if (pm_resume_via_firmware()) + dcp->s2r_domains_up = false; + /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ + if (dcp->dpms_powered_off) { + dcp_s2r_put_darts(dcp); + return 0; + } + /* After a suspend that never reached the commit, the domains' state is unknown. */ + if (pm_resume_via_firmware() && dcp_s2r_capable(dcp)) + ret = dcp_s2r_display_on(dcp); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); + } else { + /* Back from a full restart, even after a failed start had marked it crashed. */ + WRITE_ONCE(dcp->crashed, false); + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + } + + return 0; +} + +/* + * DPMS on the internal T8140 panel as macOS does a display sleep: the backlight to 0, + * then the firmware stopped and the display domains off, and the reverse on wake. Not + * during a system sleep: that path stops the firmware itself. + */ +static bool dcp_dpms_power_cycles(struct apple_dcp *dcp) +{ + return dcp_s2r_capable(dcp) && pm_suspend_target_state == PM_SUSPEND_ON; +} + +static void dcp_dpms_power_off(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (!dcp_dpms_power_cycles(dcp) || READ_ONCE(dcp->s2r_fw_down)) + return; + + ret = dcp_s2r_suspend(dcp); + if (ret && !READ_ONCE(dcp->s2r_fw_down)) { + dev_warn(dcp->dev, "DPMS: firmware stop refused, backlight only: %d\n", ret); + return; + } + dcp->s2r_stopped = false; + if (!ret) + ret = dcp_s2r_domains_off(dcp); + dcp_s2r_put_darts(dcp); + dcp->dpms_powered_off = true; + dev_dbg(dcp->dev, "DPMS: display powered off in %lld ms: %d\n", + ktime_ms_delta(ktime_get(), start), ret); +} + +/* After a refused idle the firmware still runs with the display off: light it again there. */ +static void dcp_s2r_light_running(struct apple_dcp *dcp) +{ + int ret; + + dcp_s2r_find_darts(dcp); + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (!ret && !wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "display not back on the running firmware: %d\n", ret); + return; + } + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "display back on the running firmware\n"); +} + +static void dcp_dpms_power_on(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (dcp->s2r_refused_off) { + dcp->s2r_refused_off = false; + dcp_s2r_light_running(dcp); + return; + } + if (!dcp->dpms_powered_off) + return; + dcp->dpms_powered_off = false; + if (!READ_ONCE(dcp->s2r_fw_down)) + return; + + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_display_on(dcp); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "DPMS: display power-on failed, it stays off: %d\n", ret); + return; + } + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "DPMS: display powered on in %lld ms\n", + ktime_ms_delta(ktime_get(), start)); +} + +bool dcp_s2r_firmware_stopped(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return READ_ONCE(dcp->s2r_fw_down); +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -6674,6 +7513,11 @@ static int dcp_platform_suspend(struct device *dev) if (dcp_takeover_was_aborted(dcp)) return -EBUSY; + if (pm_suspend_via_firmware() && dcp_s2r_capable(dcp)) + return dcp_s2r_suspend(dcp); + if (pm_suspend_via_firmware() && dcp_s2r_ext_capable(dcp)) + return dcp_s2r_ext_suspend(dcp); + /* * T8132: setPowerState powers the display down; the coprocessor keeps * running and its power domains stay on, as below. @@ -6764,6 +7608,9 @@ static int dcp_platform_resume(struct device *dev) if (dcp_takeover_was_aborted(dcp)) return 0; + if (dcp->s2r_stopped) + return dcp_s2r_resume(dcp); + if (iomfb_is_m4(dcp)) return iomfb_m4_resume(dcp); if (READ_ONCE(dcp->ext_dead)) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index d65367c8af13f..c09516b77693a 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -119,6 +119,7 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp); int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); +bool dcp_s2r_firmware_stopped(struct platform_device *pdev); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a504dca5a6bea..fdba30ad1299e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -299,7 +299,7 @@ static bool iomfb_uses_queue(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } -static bool iomfb_channels_idle(struct apple_dcp *dcp) +bool iomfb_channels_idle(struct apple_dcp *dcp) { return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && @@ -2873,6 +2873,10 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct platform_device *pdev = apple_crtc->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + /* Can't be lit: a resume then restores its CRTC disabled, not failing every output. */ + if (READ_ONCE(dcp->crashed)) + return false; + if (iomfb_is_m4(dcp)) { struct drm_display_mode fixed; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 5e5795ba3c379..d36d186df54d4 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -190,6 +190,7 @@ enum dcpep_method { dcpep_pipe_query_478, dcpep_pipe_query_474, dcpep_pipe_set_476, + dcpep_unnamed_500, iomfbep_set_brightness_correction, dcpep_num_methods }; @@ -210,6 +211,7 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_channels_idle(struct apple_dcp *dcp); bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, u32 out_len); /* Desktop 26.6: does the record hold what the tag's handler reads and writes? */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index ad8307f602ea7..59a276462b5a8 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -241,6 +241,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A500", dcpep_unnamed_500), /* (0x4, 0) */ IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ IOMFB_METHOD("A428", iomfbep_set_brightness_correction), /* (0x4, 0x4) 0x10000 */ IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ @@ -312,6 +313,21 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +/* + * D572: the firmware controls the display DART's power stage with this. The + * argument is 0 when it's done with the display and 1 when it wants it back. + */ +static bool trampoline_dart_power_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_packet_header *header = in - sizeof(*header); + + if (header->in_len >= sizeof(u32)) + dcp_s2r_dart_power(dcp, get_unaligned_le32(in)); + + return trampoline_zero(dcp, tag, out, in); +} + static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -691,7 +707,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [563] = trampoline_true, [565] = trampoline_true, [567] = trampoline_true, - [572] = trampoline_zero, /* powerUpDART */ + [572] = trampoline_dart_power_h17p, /* powerUpDART */ [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 82e5310f3fd6e..0a2e71334c910 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -504,6 +504,7 @@ DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_IN(dcp_unnamed_500, dcpep_unnamed_500, u32); #endif DCP_THUNK_INOUT(dcp_set_brightness_correction, iomfbep_set_brightness_correction, u32, u32); @@ -2294,6 +2295,120 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } } +#if DCP_FW_H17(DCP_FW_VER) +/* + * set_power_state around a firmware-driven suspend. What macOS sends before it in + * either direction: A414(0), A478 twice, A476(1), A478, A412. All are named in + * this firmware's table, and their arguments are measured from macOS. + * + * For display on, macOS also sends A439 at the same moment. Its name isn't known, + * it carries a macOS kernel pointer and its reply wasn't captured, so it isn't + * sent here. + */ +enum dcp_s2r_step { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A412, DCP_S2R_A473, +}; + +static const u8 dcp_s2r_seq[] = { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A478, + DCP_S2R_A412, DCP_S2R_A473, +}; + +static void dcp_s2r_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned int step = dcp->s2r_step++; + + switch (dcp_s2r_seq[step]) { + case DCP_S2R_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A478: + dcp_pipe_query_478(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A476: { + u32 val = 1; + + dcp_pipe_set_476(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A412: + dcp_is_main_display(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A473: { + /* Measured: the state, then 00 00 00 00 00 aa aa aa. */ + struct DCP_FW_NAME(dcp_set_power_state_req) req = { + .unklong = dcp->s2r_power_on, + .padding = { 0xaa, 0xaa, 0xaa }, + }; + + dcp_set_power_state(dcp, dcp->s2r_power_oob, &req, + dcp->s2r_power_on ? dcp_on_final : complete_set_powerstate, + cookie); + break; + } + } +} + +static void dcp_s2r_500_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); +} + +/* Unnamed. macOS sends it after D572 on every display off, with these bytes. */ +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + u32 val = 0xaaaa0000; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_unnamed_500(dcp, false, &val, dcp_s2r_500_done, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} + +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->s2r_power_on = on; + dcp->s2r_power_oob = oob; + dcp->s2r_step = named_calls ? 0 : ARRAY_SIZE(dcp_s2r_seq) - 1; + dcp_s2r_run(dcp, NULL, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} +#endif + static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { /* H17P does not confirm that scanout has stopped. */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 3bd2b0d2c214b..93e8895e9fb48 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -389,5 +389,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); +#if DCP_FW_H17(DCP_FW_VER) +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp); +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob); +#endif #endif diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 2b3c4f0338e43..7e66c3bb9b714 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -792,6 +792,8 @@ def poweroff_replug(root): { (void)d; (void)generation; return 0; } static void dcp_external_crtc_released(struct apple_dcp *d) { (void)d; } static bool dcp_uses_soft_dpms(struct apple_dcp *d) { return d->soft_dpms; } +/* The T8140 display power cycle has its own test on hardware; here it only has to be reached. */ +static void dcp_dpms_power_off(struct apple_dcp *d) { (void)d; } static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return !d->typec_output; } static void __attribute__((unused)) iomfb_blank_h17p(struct apple_dcp *d) { (void)d; assert(!"unexpected internal-panel blank in external-link test"); } From 7a8f48728a6aed35a243ed24766c150151993aa8 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:26 -0500 Subject: [PATCH 54/73] drm/apple: bring DCPEXT back after a sleep, and both DCPs after a refusal After a deep sleep DCPEXT's resume found nothing to do, so the external display processor stayed down until a reboot, with no firmware running. macOS brings it back the same way in every wake measured: DISPEXT0's SYS and FE on, then its CPU with PS_MIN 4 first, since the wake reset it, then on, pmc-dcp up, its DART's streams on again, and a warm restart, with no secure port and no display-on call. Do that from the resume after a real sleep, releasing the DART's GAPF slice first if the wake left it protected. Type-C work, stopped for the sleep, starts again once the firmware is back. A DCPEXT that a failed start left running is stopped before the next sleep, as the internal DCP is. pmc-dcp holds the number of powered DCPs on macOS: 0, 1, 2, 1, 0 across a wake. The driver wrote an absolute 1 or 0 for the internal DCP, which is right only while DCPEXT never powers up. Keep one count for both: a DCP counts from when this driver powers its domains up until it powers them off, or a sleep stops it, since the sleep controller lowers the domains itself. A firmware that has idled for a sleep doesn't come back with its power kept: after a refused sleep the DCPs woke on the first message and never read their inbox, and the display stayed dark until the next real sleep. The internal DCP now comes back as DPMS on brings it, from a power cycle: its domains off in macOS's order, then the warm restart and display on, measured 11 of 11. DCPEXT goes through DISPEXT0's own power-off, CPU first, which really gates its CPU, holds it off for a second, then starts as after a real sleep, through apple_rtkit_power_cycled_return(). Its display DART goes dark before the domains drop and is lit again once they're back, as the compositor can still map framebuffers into it. The sleep controller lowers DISPEXT0 before refusals of its own, and with it off the CPU and the display DART don't answer, so a cycle that finds DISPEXT0 off raises it first. A stop that fails once the CPU has stopped, at the DART's stream stop, fails the suspend, and the PM core doesn't call .resume for it: the firmware is started again there, as for a refused sleep, before the error goes back. On the internal DCP the refused path releases the GAPF slice first, as a failed flush leaves it protecting the CPU's power word, which faults when written. A crashed DCP no longer keeps the machine out of every deep sleep until reboot. As macOS recovers one, its CPU is stopped cold, its mailbox turned off and its DART's streams stopped, so the sleep controller can lower its domain, and the wake's power cycle starts it again; a good start clears the crash. DCPEXT skips its A500 and the display disconnect, and its Type-C work goes off. DPMS off, which has no power cycle behind it, still refuses a crashed DCP. An idle the internal DCP refused with the display already off is taken back: AP on to its main mailbox, as the GPU and the AOP send it after a refused idle, and its secure port's AP side brought back and polled again. The next sleep doesn't send set_power_state(0) a second time, whose D572 may never come, and goes straight to A500 and the idle; if that stops the firmware, the display stays dark after the wake, as after DPMS off, until DPMS on starts it, and if that sleep is refused its domains are lowered first, as DPMS off does. Signed-off-by: Diljit Singh --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 417 +++++++++++++++++++++++++-- 2 files changed, 387 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index aedf7ae756caf..efdb97489abd6 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -648,6 +648,8 @@ struct apple_dcp { bool s2r_fw_down; /* This driver raised the display domains, and nothing has lowered them since. */ bool s2r_domains_up; + /* Counted in pmc-dcp, the number of powered DCPs */ + bool pmc_up; bool s2r_power_on; bool s2r_power_oob; u8 s2r_step; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 943ba91991c68..8ae4bdb3be758 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -6840,6 +6840,7 @@ static void dcp_platform_complete(struct device *dev) #define DCP_S2R_DISP_STREAMS (BIT(0) | BIT(3) | BIT(4) | BIT(16)) #define DCP_S2R_DCP_STREAMS (BIT(23) | BIT(24)) #define DCP_S2R_D572_TIMEOUT_MS 1000 /* macOS: about 170 ms after A473(0) */ +#define DCP_PS_MIN GENMASK(19, 16) static struct device *dcp_s2r_find_dart(struct device_node *np) { @@ -6909,6 +6910,22 @@ static void dcp_s2r_answer_secure_port(struct apple_dcp *dcp) dcp_secure_port_start(dcp); } +/* After a refused idle: AP on to both mailboxes, as the GPU and the AOP send it. */ +static void dcp_s2r_idle_undo(struct apple_dcp *dcp) +{ + int ret; + + ret = apple_rtkit_ap_power(dcp->rtk, true); + if (ret) + dev_warn(dcp->dev, "AP on after the refused idle: %d\n", ret); + if (!dcp->secure_rtk || !dcp->secure_answered || READ_ONCE(dcp->secure_polling)) + return; + ret = apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP on after the refused idle: %d\n", ret); + dcp_secure_poll_start(dcp); +} + /* * macOS's order up to RTKit idle. On success the firmware is idle and its run bit * is still set. @@ -6916,7 +6933,9 @@ static void dcp_s2r_answer_secure_port(struct apple_dcp *dcp) static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) { ktime_t start = ktime_get(); - bool display_off = false; + /* Off since a refused idle: no second set_power_state(0), whose D572 may never come. */ + bool display_off = dcp->s2r_refused_off; + bool idle_sent = false; s64 power_ms; bool idle; int ret; @@ -6925,6 +6944,11 @@ static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) dcp_s2r_find_darts(dcp); WRITE_ONCE(dcp->s2r_active, true); + if (display_off) { + dev_info(dcp->dev, "display still off from a refused idle\n"); + goto display_is_off; + } + /* In band, 3 of 3 on macOS. */ ret = iomfb_s2r_power_h17p(dcp, false, true, false); if (ret) { @@ -6945,6 +6969,7 @@ static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) ktime_ms_delta(ktime_get(), start)); display_off = true; +display_is_off: /* D572 itself is still being answered, and a busy channel can't be sent on. */ usleep_range(5000, 6000); if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) @@ -6962,6 +6987,7 @@ static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) dev_warn(dcp->dev, "firmware calls still in flight before idle\n"); + idle_sent = true; ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); if (ret) { /* Answer what it still has queued, its log counters among them, and ask again. */ @@ -6992,6 +7018,7 @@ static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) ktime_ms_delta(ktime_get(), start), *status); WRITE_ONCE(dcp->s2r_active, false); + dcp->s2r_refused_off = false; return 0; @@ -7003,6 +7030,8 @@ static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) * own feet. DPMS on lights the display again on the running firmware. */ if (display_off) { + if (idle_sent) + dcp_s2r_idle_undo(dcp); dcp->s2r_refused_off = true; dev_warn(dcp->dev, "display left off with the firmware running\n"); return ret; @@ -7054,20 +7083,107 @@ static bool dcp_s2r_ext_capable(struct apple_dcp *dcp) of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); } +/* + * Taken as stopped, but a start since raised its domains and failed after the run bit, + * as one a signal ended. Left running, it refuses every sleep; stopped, it comes back + * from the sleep's power cycle like any other. + */ +static void dcp_s2r_stop_failed_start(struct apple_dcp *dcp) +{ + if (!dcp->s2r_domains_up || + !(readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL) & + APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return; + dev_warn(dcp->dev, "a failed start left the firmware running: stopping it for the sleep\n"); + dcp_s2r_cpu_stop(dcp); +} + +/* + * pmc-dcp counts the powered DCPs, as macOS keeps it: 0, 1, 2, 1, 0 across a wake. A DCP + * counts from when this driver powers it up until it powers it off or a sleep stops it. + */ +static DEFINE_MUTEX(dcp_pmc_lock); +static unsigned int dcp_pmc_count; + +static void dcp_pmc_set(struct apple_dcp *dcp, void __iomem *pmc, bool up) +{ + guard(mutex)(&dcp_pmc_lock); + if (up != dcp->pmc_up) { + dcp->pmc_up = up; + if (up) + dcp_pmc_count++; + else if (dcp_pmc_count) + dcp_pmc_count--; + } + writel(dcp_pmc_count, pmc); +} + +/* The sleep controller lowers the domains and pmc-dcp itself. */ +static void dcp_pmc_forget(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_pmc_lock); + if (dcp->pmc_up && dcp_pmc_count) + dcp_pmc_count--; + dcp->pmc_up = false; +} + +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp); + +/* + * Crashed firmware takes no stop sequence, and refusing would hold every later sleep off until + * reboot. Its CPU stops cold, as for a failed start, and the wake starts it again, as macOS + * recovers a crashed DCP. A system sleep only: DPMS off still refuses it. + */ +static bool dcp_s2r_stops_crashed(struct apple_dcp *dcp) +{ + return READ_ONCE(dcp->crashed) && !IS_ERR_OR_NULL(dcp->rtk) && + pm_suspend_target_state != PM_SUSPEND_ON; +} + +static int dcp_s2r_stop_crashed(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + dev_warn(dcp->dev, "firmware crashed: stopped cold for the sleep, started again after it\n"); + apple_rtkit_mailbox_off(dcp->rtk); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) { int ret; if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); dcp_s2r_find_darts(dcp); dcp->s2r_stopped = true; ret = dcp_s2r_stop_streams(dcp); if (ret) return ret == -ETIMEDOUT ? -EBUSY : ret; + dcp_pmc_forget(dcp); device_set_wakeup_path(dcp->dev); return 0; } + /* No A500 and no disconnect: the link just goes down. */ + if (dcp_s2r_stops_crashed(dcp)) { + dcp_disable_typec_work(dcp, false); + return dcp_s2r_stop_crashed(dcp); + } if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) return -EBUSY; @@ -7118,32 +7234,21 @@ static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); ret = dcp_s2r_stop_streams(dcp); - if (ret) + if (ret) { + dcp_s2r_restart_failed_stop(dcp); return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); device_set_wakeup_path(dcp->dev); return 0; } -/* - * Taken as stopped, but a start since raised its domains and failed after the run bit, - * as one a signal ended. Left running, it refuses every sleep; stopped, it comes back - * from the sleep's power cycle like any other. - */ -static void dcp_s2r_stop_failed_start(struct apple_dcp *dcp) -{ - if (!dcp->s2r_domains_up || - !(readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL) & - APPLE_DCP_COPROC_CPU_CONTROL_RUN)) - return; - dev_warn(dcp->dev, "a failed start left the firmware running: stopping it for the sleep\n"); - dcp_s2r_cpu_stop(dcp); -} - /* A failure before the firmware has stopped leaves it running, and refuses cleanly. */ static int dcp_s2r_suspend(struct apple_dcp *dcp) { + bool was_off; int ret; /* Still stopped from a suspend that never reached the commit. */ @@ -7154,10 +7259,13 @@ static int dcp_s2r_suspend(struct apple_dcp *dcp) ret = dcp_s2r_stop_streams(dcp); if (ret) return ret == -ETIMEDOUT ? -EBUSY : ret; + dcp_pmc_forget(dcp); device_set_wakeup_path(dcp->dev); return 0; } + if (dcp_s2r_stops_crashed(dcp)) + return dcp_s2r_stop_crashed(dcp); if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) return -EBUSY; @@ -7174,10 +7282,18 @@ static int dcp_s2r_suspend(struct apple_dcp *dcp) } /* Presentation stays paused if this fails: see dcp_s2r_quiesce(). */ + was_off = dcp->s2r_refused_off; ret = dcp_s2r_stop_firmware(dcp); - if (ret) + if (ret) { + if (READ_ONCE(dcp->s2r_fw_down)) + dcp_s2r_restart_failed_stop(dcp); return ret == -ETIMEDOUT ? -EBUSY : ret; + } + /* Dark since a refused idle, as after DPMS off: it stays so until DPMS on. */ + if (was_off) + dcp->dpms_powered_off = true; + dcp_pmc_forget(dcp); device_set_wakeup_path(dcp->dev); return 0; @@ -7213,7 +7329,7 @@ static int dcp_s2r_domains_off(struct apple_dcp *dcp) if (!ps || !pwrgate || !pmc) goto out; - writel(0, pmc); + dcp_pmc_set(dcp, pmc, false); val = readl(ps + 0x10000); if (val & BIT(28)) writel(val & ~(BIT(28) | GENMASK(9, 8)), ps + 0x10000); @@ -7270,7 +7386,7 @@ static int dcp_s2r_domains_on(struct apple_dcp *dcp) writel(0, pwrgate + 0xc8c); writel(1, pwrgate + 0xcd0); writel(val | 0xf | BIT(28), ps + 0x10000); - writel(1, pmc); + dcp_pmc_set(dcp, pmc, true); ret = readl_poll_timeout(ps + 0x10000, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, 100, 50000); @@ -7289,6 +7405,193 @@ static int dcp_s2r_domains_on(struct apple_dcp *dcp) return ret; } +/* + * DISPEXT0 up as macOS does it after a wake: SYS and FE on, then the CPU with PS_MIN 4 first, + * since the wake reset it, then on, and pmc-dcp up once it reads on. + */ +static int dcp_s2r_ext_domains_on(struct apple_dcp *dcp) +{ + static const u16 sys_fe[] = { 0x288, 0x320 }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + u32 val; + int i, ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + for (i = 0; i < ARRAY_SIZE(sys_fe); i++) { + val = readl(ps + sys_fe[i]) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + sys_fe[i]); + writel(val | 0xf | BIT(28), ps + sys_fe[i]); + } + val = readl(ps + 0x328) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28) | DCP_PS_MIN); + val |= FIELD_PREP(DCP_PS_MIN, 4); + writel(val, ps + 0x328); + writel(val | 0xf, ps + 0x328); + writel(val | 0xf | BIT(28), ps + 0x328); + + ret = readl_poll_timeout(ps + 0x328, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) { + dev_err(dcp->dev, "DISPEXT0_CPU did not power on: %#x\n", val); + } else { + dcp_pmc_set(dcp, pmc, true); + dcp->s2r_domains_up = true; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * DCPEXT back after a real sleep, the same way in every macOS wake measured: its domains, + * its DART's streams, and a warm restart, with no secure port and no display-on call. After + * its own power cycle the start is the same, with its display DART lit again. + */ +static int dcp_s2r_ext_start(struct apple_dcp *dcp, bool cycled) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (!ret) + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + /* After the domains, whose writes it would reject, the DART protects them again. */ + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + if (!cycled) + return apple_rtkit_sleep_return(dcp->rtk, control); + + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, false); + return apple_rtkit_power_cycled_return(dcp->rtk, control); +} + +/* DISPEXT0 off in its own order, CPU first. Only the CPU has to power-gate to start again. */ +static int dcp_s2r_ext_domains_off(struct apple_dcp *dcp) +{ + static const struct { + u16 offset; + u32 first, second; + bool member; + } steps[] = { + { 0x328, 0xf0400ff, 0xf0403f0 }, { 0x320, 0, 0x23f0, true }, { 0x288, 0, 0x23f0 }, + }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + int i, ret = -ENODEV; + u32 val; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + /* + * Its display DART goes off with DISPEXT0, and every framebuffer is mapped into it: a + * map during the hold would reach its registers. Dark until the domains are back. + */ + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, true); + dcp->s2r_domains_up = false; + dcp_pmc_set(dcp, pmc, false); + ret = 0; + for (i = 0; i < ARRAY_SIZE(steps); i++) { + int gated; + + if (steps[i].first) + writel(steps[i].first, ps + steps[i].offset); + writel(steps[i].second, ps + steps[i].offset); + gated = readl_poll_timeout(ps + steps[i].offset, val, + !(val & GENMASK(7, 4)) && !(val & BIT(11)) && + (steps[i].member || (val & BIT(8))), 100, 50000); + dev_dbg(dcp->dev, "DISPEXT0 power word %#x reads %#x: %d\n", steps[i].offset, val, + gated); + if (i == 0 && gated) + ret = gated; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +#define DCP_S2R_EXT_CYCLE_HOLD_MS 1000 + +static bool dcp_s2r_ext_cpu_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps; + bool on; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return false; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + of_node_put(np); + if (!ps) + return false; + on = FIELD_GET(GENMASK(7, 4), readl(ps + 0x328)); + iounmap(ps); + + return on; +} + +/* + * After a refused sleep DCPEXT has idled with its power kept, and an idled firmware doesn't + * wake again: power-cycle it through DISPEXT0's own sequence, then start it as after a sleep. + * The sleep driver lowers DISPEXT0 before refusals of its own, and with it off the CPU and the + * display DART don't answer, so it's raised first. + */ +static int dcp_s2r_ext_cycle(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (ret) + return ret; + if (!dcp_s2r_ext_cpu_on(dcp)) { + dev_info(dcp->dev, "DISPEXT0 lowered by the refused sleep: raised for the cycle\n"); + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + } + + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + ret = dcp_s2r_ext_domains_off(dcp); + if (!ret) { + msleep(DCP_S2R_EXT_CYCLE_HOLD_MS); + ret = dcp_s2r_ext_start(dcp, true); + } + + return ret; +} + /* Between the main mailbox's IOP report and its AP on, measured 2 of 2. */ static void dcp_secure_port_map(void *data) { @@ -7378,27 +7681,43 @@ static int dcp_s2r_display_on(struct apple_dcp *dcp) return ret; } +/* Power kept: lowered as DPMS off does. A failed flush left the GAPF slice faulting that. */ +static int dcp_s2r_refused_domains_off(struct apple_dcp *dcp) +{ + int ret = dcp->s2r_dcp_dart ? apple_dart_gapf_ready(dcp->s2r_dcp_dart) : -ENODEV; + + return ret ?: dcp_s2r_domains_off(dcp); +} + /* * If the display doesn't come back, the panel stays dark until reboot, and the driver * must not talk to the stopped firmware, so treat it as a crashed DCP. Nothing else in * the system depends on it. */ -static int dcp_s2r_resume(struct apple_dcp *dcp) +static int dcp_s2r_restart(struct apple_dcp *dcp, bool real) { int ret = -EAGAIN; - dcp->s2r_stopped = false; - /* A real sleep took the domains' power. */ - if (pm_resume_via_firmware()) - dcp->s2r_domains_up = false; - /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ - if (dcp->dpms_powered_off) { - dcp_s2r_put_darts(dcp); - return 0; + /* + * After a refused sleep the firmware has idled with its power kept, and an idled firmware + * doesn't wake again: the internal DCP comes back as DPMS on brings it, from a power + * cycle, and DCPEXT through its own. + */ + if (dcp_s2r_capable(dcp)) { + ret = real ? 0 : dcp_s2r_refused_domains_off(dcp); + if (!ret) + ret = dcp_s2r_display_on(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware restarted as for DPMS: %d\n", + ret); + } else if (dcp_s2r_ext_capable(dcp)) { + ret = real ? dcp_s2r_ext_start(dcp, false) : dcp_s2r_ext_cycle(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware %s after a power cycle: %d\n", + ret ? "not back" : "back", ret); + else if (ret) + dev_err(dcp->dev, "warm restart failed: %d\n", ret); } - /* After a suspend that never reached the commit, the domains' state is unknown. */ - if (pm_resume_via_firmware() && dcp_s2r_capable(dcp)) - ret = dcp_s2r_display_on(dcp); dcp_s2r_put_darts(dcp); if (ret) { WRITE_ONCE(dcp->crashed, true); @@ -7408,11 +7727,45 @@ static int dcp_s2r_resume(struct apple_dcp *dcp) WRITE_ONCE(dcp->crashed, false); WRITE_ONCE(dcp->s2r_fw_down, false); iomfb_pause_presentation(dcp, false); + if (dcp_s2r_ext_capable(dcp) && !READ_ONCE(dcp->typec_work_enabled)) + dcp_enable_typec_work(dcp); + } + + return ret; +} + +static int dcp_s2r_resume(struct apple_dcp *dcp) +{ + bool real = pm_resume_via_firmware(); + + dcp->s2r_stopped = false; + /* A real sleep took the domains' power. */ + if (real) + dcp->s2r_domains_up = false; + /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ + if (dcp->dpms_powered_off) { + /* Off after a refused idle, its power may be on: DPMS on needs it lowered. */ + if (!real) + dcp_s2r_refused_domains_off(dcp); + dcp_s2r_put_darts(dcp); + return 0; } + dcp_s2r_restart(dcp, real); return 0; } +/* + * Its own suspend failed with the CPU stopped, and the PM core won't call .resume: start it + * again as a refused sleep does, or the display stays dark until a later sleep gets through. + */ +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp) +{ + dev_warn(dcp->dev, "stop failed with the CPU stopped: starting the firmware again\n"); + dcp_s2r_restart(dcp, false); + dcp->s2r_stopped = false; +} + /* * DPMS on the internal T8140 panel as macOS does a display sleep: the backlight to 0, * then the firmware stopped and the display domains off, and the reverse on wake. Not From 0b0f605963f0d7119408bc8bf3c335bd643ea2fb Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 08:11:01 -0500 Subject: [PATCH 55/73] phy: apple: atc: set a DisplayPort mode up again after a deep sleep A T8140 PHY carrying DisplayPort doesn't sleep in place: its USB controller stops for the sleep instead. The sleep's commit then lowers ATC0_COMMON itself, behind the power domain's back, so the lanes, the tunables and the AUX channel are gone at the wake while the driver still tracks the DP mode the Type-C mux asked for. Nothing set them up again, so an external display that stayed attached through the sleep could never train its link. After a sleep that reached its commit, mark such a PHY's domain as having been off, so that atcphy_resume() brings the tracked mode back through atcphy_restore_after_pd_off(), before the display controller resumes and connects the link anew. A sleep in place keeps its own restore, and a refused sleep left the block powered. Signed-off-by: Diljit Singh --- drivers/phy/apple/atc.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 1ea2d57ee8b8e..11447189db988 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -6717,14 +6717,32 @@ static int atcphy_suspend_noirq(struct device *dev) return 0; } +/* + * A PHY in a DisplayPort mode doesn't sleep in place, and a sleep that reached its commit + * lowered ATC0_COMMON under it, behind its power domain's back: the lanes and the AUX + * channel went. Have atcphy_resume() set the tracked mode up again, before the display + * controller connects the link anew. + */ +static void atcphy_sleep_lost_dp(struct apple_atcphy *atcphy) +{ + if (!atcphy->hw->has_t8140_dp || !atcphy->hw->restore_after_pd_off || + !pm_resume_via_firmware() || !atcphy_modes[atcphy->mode].enable_dp_aux) + return; + WRITE_ONCE(atcphy->pd_was_off, true); + dev_dbg(atcphy->dev, "DisplayPort mode %d lost in the sleep: setting it up again\n", + atcphy->mode); +} + static int atcphy_resume_noirq(struct device *dev) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); int ret = 0; guard(mutex)(&atcphy->lock); - if (!atcphy->hibernated) + if (!atcphy->hibernated) { + atcphy_sleep_lost_dp(atcphy); return 0; + } apple_sleep_usb_set_asleep(false); atcphy_usb2_sleep_exit(atcphy); From df4485a51d2979264630d29e5c750137c24c6a3f Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 08:11:02 -0500 Subject: [PATCH 56/73] drm/apple: park a Type-C display route's crossbar for a deep sleep On T8140 the display crossbar sits in the ATC block that the sleep's commit powers off. The mux core still holds the route's selection after the wake and skips selecting it again, so a display kept attached would come back on a crossbar with its reset values. Add dcp_typec_route_park() and dcp_typec_route_unpark(). Parking takes a direct route's output down while it still has power, as releasing a display does, and unparking selects it again after the wake: the same teardown and setup an unplug before a sleep and a plug after it go through. The route, its owner and the connector stay as they are. A route released while parked has nothing held to give back, and one that could not be selected again is marked so. A crashed DCPEXT, which goes into the sleep without its A500, has its route parked the same way. Signed-off-by: Diljit Singh --- drivers/gpu/drm/apple/dcp-fabric.c | 54 ++++++++++++++++++- drivers/gpu/drm/apple/dcp-fabric.h | 4 ++ drivers/gpu/drm/apple/dcp.c | 3 +- .../drivers/gpu/apple-dcp/transaction.py | 2 +- 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 3dc3128b600ae..9f29f28b21f87 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -592,6 +592,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, scoped_guard(mutex, &dcp->tb_lock) { route->deactivate_xbar_done = false; route->deactivate_error = 0; + route->sleep_parked = false; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -639,7 +640,10 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return route->deactivate_error; if (!route->deactivate_xbar_done) { dcp_typec_route_phy_off(route); - if (!route->tunnel || route->xbar_up) + /* Parked for a sleep, its output is already down and not held. */ + if (route->sleep_parked) + ret = 0; + else if (!route->tunnel || route->xbar_up) ret = mux_control_deselect(route->active_xbar ?: route->xbar); else ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); @@ -653,6 +657,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) ret = 0; /* Deselect always unlocks: never consume that selection twice. */ route->deactivate_xbar_done = true; + route->sleep_parked = false; route->xbar_up = false; if (ret) { /* Idle is unproven; retain the owner and the real error. */ @@ -4289,6 +4294,53 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) return dcp_fabric_run_probe(!!routes, dcp_register_typec_routes_present, &ctx); } +/* + * A sleep that reaches its commit powers the crossbar off. Take the output of a direct + * route down while it still has power, as a released display's is, and select it again + * after the wake. The route stays, and with it the display on the port. + */ +void dcp_typec_route_park(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route || route->tunnel || !route->selected) + return; + guard(mutex)(&dcp->tb_lock); + if (route->deactivate_xbar_done || route->sleep_parked) + return; + /* Deselect always unlocks. */ + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + route->sleep_parked = true; + dev_dbg(dcp->dev, "display route parked for the sleep: crossbar output down: %d\n", ret); +} + +void dcp_typec_route_unpark(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route) + return; + guard(mutex)(&dcp->tb_lock); + if (!route->sleep_parked) + return; + route->sleep_parked = false; + ret = mux_control_try_select(route->active_xbar ?: route->xbar, route->mux_index); + if (ret) { + /* Nothing is held, so the route's release has nothing to give back. */ + route->deactivate_xbar_done = true; + dev_err(dcp->dev, "display route not back after the sleep: crossbar output: %d\n", + ret); + return; + } + dev_dbg(dcp->dev, "display route back after the sleep: crossbar output up\n"); +} + /* * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY * (see dcp_typec_route_deactivate()). Point it at the HDMI output again diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index f68b91e4d18f8..f46211ec3a8e1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -51,6 +51,8 @@ struct apple_dcp_typec_route { /* Lower mux release is consumed even when idle programming fails. */ bool deactivate_xbar_done; int deactivate_error; + /* Its crossbar output is down for a sleep, and not held. */ + bool sleep_parked; }; int dcp_typec_phy_on(struct apple_dcp *dcp); @@ -111,6 +113,8 @@ bool dcp_is_t8132_ext(const struct apple_dcp *dcp); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); +void dcp_typec_route_park(struct apple_dcp *dcp); +void dcp_typec_route_unpark(struct apple_dcp *dcp); void dcp_typec_routes_disable(struct apple_dcp *dcp); void dcp_typec_routes_disable_dev(struct device *dev); void dcp_typec_retrain_work(struct work_struct *work); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8ae4bdb3be758..e2ce0d0b6dca5 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -7179,9 +7179,10 @@ static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) return 0; } - /* No A500 and no disconnect: the link just goes down. */ + /* No A500 and no disconnect: the link and its crossbar output just go down. */ if (dcp_s2r_stops_crashed(dcp)) { dcp_disable_typec_work(dcp, false); + dcp_typec_route_park(dcp); return dcp_s2r_stop_crashed(dcp); } if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index b41f7e2f80689..8b500ac5527f8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -56,7 +56,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): actual=actual.replace('static struct apple_dcp_typec_port *sole_port(void) { return &port; }', 'static struct apple_dcp_typec_port *sole_port(void) { return &port; }\nstatic struct apple_dcp_typec_route * __attribute__((unused)) sole_route(void) { return &route; }',1) actual=actual.replace('bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;', 'struct typec_altmode *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;',1) - actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel;',1) + actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel,sleep_parked;',1) extra=r''' /* actual DCP leaves */ #define WRITE_ONCE(x,v) ((x)=(v)) From c057ccae43fec3a34818624f518d0f799e2eede7 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 08:11:02 -0500 Subject: [PATCH 57/73] drm/apple: take an attached external display through a deep sleep The external T8140 DCP refused every deep sleep with a display attached ("a display is attached, refusing sleep"), since nothing brought the display back after the wake. macOS releases the display at system sleep and replays the port's HPD at the wake. In the 25G83 kernel, read statically, IODPSwitch::handleSystemWillSleep() calls AppleATCDPNativePort::handleDPSwitchWillSleep(), which logs "releasing display..." and calls displayRelease() for a requested display; its handleDPSwitchWillWake() calls checkInitialHPDState(), which synthesizes the plug from the current HPD. The crossbar driver's strings suggest that the DCPEXT firmware answers a release with Deactivate, after which the crossbar connection goes down. Do the same. Linux already releases the display the way DPMS off does (pipe off, HPD low, DisplayRelease), and KDE turns the display off before the sleep anyway. Then park the route's crossbar output and stop the firmware as with no display. At the wake, after the firmware's warm restart, select the crossbar output again; the Type-C work connects the link as for a sleep with the CRTC off, and the CRTC is lit on it by DPMS on, or by the DRM resume when it was on. The Type-C route and the connector stay throughout, so the compositor keeps its output and its framebuffers. A display unplugged or replugged during the sleep goes through the port controller's revalidation after the wake, as any unplug or plug does. Measured on a J700: an external USB-C display attached through a sleep is released for it and lights again after the wake. Signed-off-by: Diljit Singh --- drivers/gpu/drm/apple/dcp.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e2ce0d0b6dca5..cf6b00041ae68 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -7074,8 +7074,12 @@ static int dcp_s2r_stop_firmware(struct apple_dcp *dcp) /* * macOS stops this firmware with one call the size of A500, then about 4 ms later AP * then IOP idle and the run bit off twice. Nothing has shown it needing a secure - * port. Its panel path is already off with no display attached. Nothing brings it - * back yet, so an attached display refuses the sleep. + * port. + * + * An attached display goes first, as macOS's DP port releases its display at system + * sleep: the pipe off, HPD low and the display released, as DPMS off does, then the + * crossbar output. The route and the connector stay, and after the wake the link + * connects again, as macOS replays the port's HPD. */ static bool dcp_s2r_ext_capable(struct apple_dcp *dcp) { @@ -7171,9 +7175,12 @@ static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) dcp_s2r_stop_failed_start(dcp); dcp_s2r_find_darts(dcp); dcp->s2r_stopped = true; + dcp_typec_route_park(dcp); ret = dcp_s2r_stop_streams(dcp); - if (ret) + if (ret) { + dcp_typec_route_unpark(dcp); return ret == -ETIMEDOUT ? -EBUSY : ret; + } dcp_pmc_forget(dcp); device_set_wakeup_path(dcp->dev); return 0; @@ -7188,10 +7195,8 @@ static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) return -EBUSY; - if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) { - dev_err(dcp->dev, "a display is attached, refusing sleep\n"); - return -EBUSY; - } + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) + dev_dbg(dcp->dev, "a display is attached: released for the sleep, connected again after it\n"); dcp_disable_typec_work(dcp, false); ret = dcp_external_disconnect(dcp, false); @@ -7210,10 +7215,14 @@ static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) return -EBUSY; } + /* The display's link went above: its crossbar output follows, as on macOS. */ + dcp_typec_route_park(dcp); + ret = iomfb_s2r_500_h17p(dcp); if (ret) { dev_err(dcp->dev, "no reply to A500: %d\n", ret); WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); return ret == -ETIMEDOUT ? -EBUSY : ret; } usleep_range(4000, 5000); @@ -7224,6 +7233,7 @@ static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); return ret == -ETIMEDOUT ? -EBUSY : ret; } @@ -7720,6 +7730,9 @@ static int dcp_s2r_restart(struct apple_dcp *dcp, bool real) dev_err(dcp->dev, "warm restart failed: %d\n", ret); } dcp_s2r_put_darts(dcp); + /* An attached display's crossbar output; its link reconnects from the Type-C work. */ + if (dcp_s2r_ext_capable(dcp)) + dcp_typec_route_unpark(dcp); if (ret) { WRITE_ONCE(dcp->crashed, true); dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); From 177cdc3952f2ee9cb712aa4cfec0d63b2b3f6fb4 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:09:38 -0500 Subject: [PATCH 58/73] drm/apple: keep the panel dark through a dark wake macOS keeps the display off in a dark wake. In the 25G83 kernel, read statically, the DCP family reads the root domain's AppleClamshellState and sends it to the DCP as A476 (set_dcp_clamshell_state), and the DisplayPort family has its own dark wake state. The panel is powered, as far as the code shows, only when WindowServer asks for it, in a full wake. In a dark wake the internal T8140 DCP now resumes as after DPMS off before the sleep: its firmware stays stopped, and the next CRTC enable brings it and the panel back through dcp_dpms_power_on(). The suspend that follows takes the firmware-down path that already exists. DCPEXT stays down the same way, its crossbar output parked and its Type-C work off, so a monitor on the USB-C port isn't driven up for a moment at every dark wake. The next resume that isn't dark restarts it as after any real sleep. If the next suspend fails before suspending it, ->complete() starts it, or a PM notifier does when the failure came before ->prepare(); the real sleep reset its CPU, so it starts as after a power cycle. apple-drm keeps its saved state and commits nothing, since user space stays frozen. Its next suspend leaves that state alone, so the real wake restores what was there before the first sleep. If that suspend fails, ->complete() restores the outputs, or a PM notifier does when the failure came before ->prepare() reached this device. Signed-off-by: Diljit Singh --- drivers/gpu/drm/apple/apple_drv.c | 77 ++++++++++++++++--- drivers/gpu/drm/apple/dcp-internal.h | 4 + drivers/gpu/drm/apple/dcp.c | 107 +++++++++++++++++++++++---- 3 files changed, 163 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 0fcc436c796df..d211e487893c5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -44,6 +44,7 @@ #include #include #include +#include #include "apple_boot_display.h" #include "apple_resume.h" @@ -66,8 +67,13 @@ struct apple_drm_private { bool aborted; /* This suspend found every firmware already stopped. */ bool s2r_skipped; + /* A dark wake left the outputs as the suspend did, for the next resume. */ + bool dark_held; + struct notifier_block pm_nb; }; +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data); + /* Component readiness is later than platform probe/bind. */ static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; @@ -1552,6 +1558,9 @@ static int apple_drm_init(struct device *dev) goto err_unbind; } + apple->pm_nb.notifier_call = apple_drm_pm_notify; + register_pm_notifier(&apple->pm_nb); + return 0; err_unlock: @@ -1575,6 +1584,8 @@ static void apple_drm_uninit(struct device *dev) if (apple->aborted) return; + unregister_pm_notifier(&apple->pm_nb); + /* Before anything of the DRM device is torn down. */ t8132_ext_drm_fini(&apple->drm); apple_drm_quiesce_connectors(&apple->drm); @@ -1716,6 +1727,10 @@ static int apple_platform_suspend(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + /* Still suspended from before a dark wake. */ + if (apple && apple->dark_held) + return 0; + if (apple && !apple->aborted) { apple->s2r_skipped = apple_drm_firmware_stopped(apple); if (!apple->s2r_skipped) @@ -1742,29 +1757,71 @@ static bool apple_drm_hold_resume_events(struct apple_drm_private *apple, u32 cr return held; } +static void apple_drm_resume_outputs(struct apple_drm_private *apple) +{ + u32 dropped = 0; + + apple->dark_held = false; + if (apple->drm.mode_config.suspend_state) + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped && apple_drm_hold_resume_events(apple, dropped)) + dropped = 0; + drm_mode_config_helper_resume(&apple->drm); + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - u32 dropped = 0; - if (apple && !apple->aborted && !apple->s2r_skipped) { - if (apple->drm.mode_config.suspend_state) - dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); - /* Have userspace set the mode of a CRTC resumed disabled again. */ - if (dropped && apple_drm_hold_resume_events(apple, dropped)) - dropped = 0; - drm_mode_config_helper_resume(&apple->drm); - if (dropped) - drm_kms_helper_hotplug_event(&apple->drm); + if (!apple || apple->aborted || apple->s2r_skipped) + return 0; + + /* A dark wake goes back to sleep with user space frozen: nothing is lit for it. */ + if (pm_dark_wake()) { + apple->dark_held = true; + dev_info(dev, "dark wake: outputs stay off\n"); + return 0; } + apple_drm_resume_outputs(apple); return 0; } +/* The suspend after a dark wake failed before it suspended this device. */ +static void apple_platform_complete(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple && apple->dark_held && !pm_dark_wake()) + apple_drm_resume_outputs(apple); +} + +/* It failed before ->prepare() reached this device, so ->complete() didn't run either. */ +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_drm_private *apple = container_of(nb, struct apple_drm_private, pm_nb); + + if (action == PM_POST_SUSPEND && apple->dark_held) { + dev_warn(apple->drm.dev, "dark wake over without a resume: outputs back now\n"); + apple_drm_resume_outputs(apple); + } + + return NOTIFY_DONE; +} + static const struct dev_pm_ops apple_platform_pm_ops = { .suspend = apple_platform_suspend, .resume = apple_platform_resume, + .complete = apple_platform_complete, }; +#else +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + return NOTIFY_DONE; +} #endif static struct platform_driver apple_platform_driver = { diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index efdb97489abd6..0149d854d37dc 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -646,6 +647,9 @@ struct apple_dcp { struct delayed_work secure_poll_wq; /* Stopped and not restarted, which outlives a suspend that gets refused. */ bool s2r_fw_down; + /* DCPEXT kept down through a dark wake, see dcp_s2r_dark_finish(). */ + bool s2r_dark_held; + struct notifier_block s2r_pm_nb; /* This driver raised the display domains, and nothing has lowered them since. */ bool s2r_domains_up; /* Counted in pmc-dcp, the number of powered DCPs */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index cf6b00041ae68..34e235dc52588 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5952,6 +5952,8 @@ static void dcp_t8132_ext_routes(struct apple_dcp *dcp) dcp->nr_typec_routes); } +static int dcp_s2r_pm_register(struct apple_dcp *dcp); + static int dcp_platform_probe_pipe(struct platform_device *pdev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); @@ -6128,6 +6130,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) } platform_set_drvdata(pdev, dcp); + ret = dcp_s2r_pm_register(dcp); + if (ret) + return ret; if (iomfb_is_m4(dcp)) { dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; @@ -6809,10 +6814,13 @@ static int dcp_platform_prepare(struct device *dev) return 0; } +static void dcp_s2r_dark_finish(struct apple_dcp *dcp); + static void dcp_platform_complete(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_s2r_dark_finish(dcp); if (!dcp->external) return; mutex_lock(&dcp->hpd_mutex); @@ -7700,6 +7708,27 @@ static int dcp_s2r_refused_domains_off(struct apple_dcp *dcp) return ret ?: dcp_s2r_domains_off(dcp); } +static void dcp_s2r_restart_done(struct apple_dcp *dcp, int ret) +{ + bool ext = dcp_s2r_ext_capable(dcp); + + dcp_s2r_put_darts(dcp); + /* An attached display's crossbar output; its link reconnects from the Type-C work. */ + if (ext) + dcp_typec_route_unpark(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); + return; + } + /* Back from a full restart, even after a failed start had marked it crashed. */ + WRITE_ONCE(dcp->crashed, false); + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + if (ext && !READ_ONCE(dcp->typec_work_enabled)) + dcp_enable_typec_work(dcp); +} + /* * If the display doesn't come back, the panel stays dark until reboot, and the driver * must not talk to the stopped firmware, so treat it as a crashed DCP. Nothing else in @@ -7729,21 +7758,7 @@ static int dcp_s2r_restart(struct apple_dcp *dcp, bool real) else if (ret) dev_err(dcp->dev, "warm restart failed: %d\n", ret); } - dcp_s2r_put_darts(dcp); - /* An attached display's crossbar output; its link reconnects from the Type-C work. */ - if (dcp_s2r_ext_capable(dcp)) - dcp_typec_route_unpark(dcp); - if (ret) { - WRITE_ONCE(dcp->crashed, true); - dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); - } else { - /* Back from a full restart, even after a failed start had marked it crashed. */ - WRITE_ONCE(dcp->crashed, false); - WRITE_ONCE(dcp->s2r_fw_down, false); - iomfb_pause_presentation(dcp, false); - if (dcp_s2r_ext_capable(dcp) && !READ_ONCE(dcp->typec_work_enabled)) - dcp_enable_typec_work(dcp); - } + dcp_s2r_restart_done(dcp, ret); return ret; } @@ -7756,6 +7771,18 @@ static int dcp_s2r_resume(struct apple_dcp *dcp) /* A real sleep took the domains' power. */ if (real) dcp->s2r_domains_up = false; + /* A dark wake sleeps again with the panel never lit, as if DPMS had turned it off. */ + if (pm_dark_wake() && dcp_s2r_capable(dcp) && !dcp->dpms_powered_off) { + dcp->dpms_powered_off = true; + dev_info(dcp->dev, "dark wake: firmware and panel stay off\n"); + } + /* DCPEXT too, its crossbar output parked: dcp_s2r_dark_finish() if no resume follows. */ + if (pm_dark_wake() && dcp_s2r_ext_capable(dcp)) { + dcp->s2r_dark_held = true; + dev_info(dcp->dev, "dark wake: firmware and output stay off\n"); + return 0; + } + dcp->s2r_dark_held = false; /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ if (dcp->dpms_powered_off) { /* Off after a refused idle, its power may be on: DPMS on needs it lowered. */ @@ -7769,6 +7796,56 @@ static int dcp_s2r_resume(struct apple_dcp *dcp) return 0; } +/* + * The suspend after a dark wake failed before suspending DCPEXT, or before preparing it. It's + * still down from the real sleep, whose wake reset its CPU: start it as after one. + */ +static void dcp_s2r_dark_finish(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->s2r_dark_held || pm_dark_wake() || IS_ERR_OR_NULL(dcp->rtk)) + return; + dcp->s2r_dark_held = false; + dcp->s2r_stopped = false; + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_ext_start(dcp, true); + dev_info(dcp->dev, "dark wake over without a resume: firmware %s: %d\n", + ret ? "not back" : "back", ret); + dcp_s2r_restart_done(dcp, ret); +} + +static int dcp_s2r_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_dcp *dcp = container_of(nb, struct apple_dcp, s2r_pm_nb); + + if (action == PM_POST_SUSPEND) + dcp_s2r_dark_finish(dcp); + + return NOTIFY_DONE; +} + +static void dcp_s2r_pm_unregister(void *data) +{ + struct apple_dcp *dcp = data; + + unregister_pm_notifier(&dcp->s2r_pm_nb); +} + +static int dcp_s2r_pm_register(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp_s2r_ext_capable(dcp)) + return 0; + dcp->s2r_pm_nb.notifier_call = dcp_s2r_pm_notify; + ret = register_pm_notifier(&dcp->s2r_pm_nb); + if (ret) + return ret; + + return devm_add_action_or_reset(dcp->dev, dcp_s2r_pm_unregister, dcp); +} + /* * Its own suspend failed with the CPU stopped, and the PM core won't call .resume: start it * again as a refused sleep does, or the display stays dark until a later sleep gets through. From 5c91097341eabaf96cf7cb33bbe7d25041661ce3 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:09:38 -0500 Subject: [PATCH 59/73] soc: apple: t8140-sleep, mfd: macsmc: take a lid closing during sleep as a dark wake The SMC wakes the J700 for the lid closing as well as for it opening, with the same wake record bit ("lid"). The machine then woke fully with the lid closed, and logind ignores the lid for 30 s after a resume, so it stayed awake (measured on a J700). macOS takes that wake and goes back to sleep. In the 25G83 kernel, read statically, AppleSMCPMU's _sendLidStateToRootDomainGated sends 0x10 (kIOPMClamshellClosed) to the root domain, whose dark wake policy then calls privateSleepSystem() with 0x6c (Maintenance) unless the machine is in desktop mode on AC. Do the same through the PM core's dark wake. The sleep driver records in .wake whether only the SMC's own sources woke the SoC, which it finds by their names in its table of the ADT's wake sources. The SMC's noirq resume, still polled, reads MSLD when the wake record is the lid alone, the bit taken from its table of reasons by name, and reports a closed lid to the sleep driver, which sets the dark wake flag when nothing else woke the SoC. Any other source or reason, an open lid or a failed read gives a normal wake. A display plugged in during the sleep wakes the AP with its own reason. Unlike macOS, a closed lid sleeps again with an external display attached on AC too: desktop mode isn't modelled here. The wake record's wakeup event is reported after the lid check, and not for a dark wake: with wakeup_count in use, it would abort the sleep that follows. A KUnit case checks that MSLD is read only for the lid alone, after MBSE = waka. Signed-off-by: Diljit Singh --- drivers/mfd/macsmc-test.c | 42 ++++++++++++++++++++++++++ drivers/mfd/macsmc.c | 41 +++++++++++++++++++++++-- drivers/soc/apple/t8140-sleep.c | 53 +++++++++++++++++++++++++++++++-- include/linux/soc/apple/sleep.h | 11 +++++++ 4 files changed, 143 insertions(+), 4 deletions(-) diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index 5e38da0dabcfa..16d6fdae7c12e 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -13,6 +13,7 @@ struct macsmc_test_context { u32 seen[8]; unsigned int seen_count; int ap_power_result; + u64 record; }; #define MACSMC_TEST_AP_IDLE 0 @@ -279,6 +280,46 @@ static void macsmc_notice_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(MBSW)); } +/* MBSW reads ctx->record, MSLD reads closed. */ +static int macsmc_test_lid_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = key; + memset(buf, 0, size); + if (key == SMC_KEY(MBSW) && size == sizeof(ctx->record)) + put_unaligned_le64(ctx->record, buf); + else if (key == SMC_KEY(MSLD)) + *(u8 *)buf = 1; + return size; +} + +static void macsmc_lid_wake_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_read, macsmc_test_lid_read); + ctx->result = 4; + + /* The lid alone: MSLD is read last, after macOS's own order. */ + ctx->record = BIT_ULL(10); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 5U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MSLD)); + + /* The lid with anything else is a normal wake: the lid isn't read. */ + ctx->seen_count = 0; + ctx->record = BIT_ULL(10) | BIT_ULL(0); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 4U); +} + static void macsmc_wake_reason_test(struct kunit *test) { char names[128]; @@ -304,6 +345,7 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_sleep_order_test), KUNIT_CASE(macsmc_sleep_refused_test), KUNIT_CASE(macsmc_notice_test), + KUNIT_CASE(macsmc_lid_wake_test), KUNIT_CASE(macsmc_wake_reason_test), {} }; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 1b9ffc9bf2d3a..c7bb10a87fb99 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -769,8 +769,41 @@ static void apple_smc_notice_on(struct apple_smc *smc) apple_smc_wake_reason_names(smc->wake_record, names, sizeof(names)); dev_info(smc->dev, "wake record: MBSE %#x, MBSW %8ph%s%s\n", event, record, names[0] ? ": " : "", names); - if (smc->wake_record) - pm_wakeup_event(smc->dev, 0); +} + +/* MBSW's bit for a wake reason, by its name, or -1. */ +static int apple_smc_wake_reason_bit(const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (!strcmp(apple_smc_wake_reasons[i].name, name)) + return apple_smc_wake_reasons[i].bit; + + return -1; +} + +/* + * Woken for the lid alone, which the SMC reports for the lid opening and for it closing: + * tell the sleep driver if it's closed now. + */ +static void apple_smc_lid_wake(struct apple_smc *smc) +{ + int bit = apple_smc_wake_reason_bit("lid"); + u8 lid = 0; + int ret; + + if (bit < 0 || smc->wake_record != BIT_ULL(bit)) + return; + + ret = apple_smc_read_u8(smc, SMC_KEY(MSLD), &lid); + if (ret < 0) { + dev_warn(smc->dev, "Failed to read the lid after a wake: %d\n", ret); + return; + } + dev_info(smc->dev, "woken for the lid, which is %s\n", lid ? "closed" : "open"); + if (lid) + apple_sleep_lid_closed_wake(); } static void apple_smc_sleep_wake(struct apple_smc *smc) @@ -778,6 +811,10 @@ static void apple_smc_sleep_wake(struct apple_smc *smc) apple_smc_notice_on(smc); apple_smc_write_mbse(smc, SMC_KEY(waka)); smc->sleep_warned = false; + apple_smc_lid_wake(smc); + /* A dark wake isn't one: with wakeup_count in use, it would abort the sleep after it. */ + if (smc->wake_record && !pm_dark_wake()) + pm_wakeup_event(smc->dev, 0); } static int apple_smc_prepare(struct device *dev) diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c index dd37629042962..a24d395b8837d 100644 --- a/drivers/soc/apple/t8140-sleep.c +++ b/drivers/soc/apple/t8140-sleep.c @@ -332,7 +332,7 @@ static struct { #define IPI_SR_PENDING BIT(0) #define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) -/* The names of ADT pmgr/ap-wake-sources, for the log only */ +/* The names of ADT pmgr/ap-wake-sources: for the log, and to tell the SMC's apart */ static const struct { u32 id; const char *name; @@ -410,6 +410,8 @@ struct t8140_sleep { u32 wake_enable[WAKE_SOURCES_MAX]; int wake_enable_count; u32 wake_saved[WAKE_WORDS]; + /* The sources this sleep left enabled. */ + u32 wake_armed[WAKE_WORDS]; struct { struct device_node *np; @@ -479,6 +481,8 @@ struct t8140_sleep { int radio_perst; bool radio_perst_low; bool returned; + /* This sleep's wake came from the SMC's sources alone. */ + bool smc_woke_alone; }; static struct t8140_sleep *t8140_sleep; @@ -1444,8 +1448,10 @@ static void t8140_wake_arm(struct t8140_sleep *s) for (i = 0; i < WAKE_WORDS; i++) s->wake_saved[i] = readl(s->wake + WAKE_MASK(i)); t8140_wake_masks(s, val); - for (i = 0; i < WAKE_WORDS; i++) + for (i = 0; i < WAKE_WORDS; i++) { writel(val[i], s->wake + WAKE_MASK(i)); + s->wake_armed[i] = ~val[i]; + } } static void t8140_wake_restore(struct t8140_sleep *s) @@ -1649,6 +1655,21 @@ bool apple_sleep_debug_link_cut(void) } EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); +/* + * macOS takes the wake for a lid closing during the sleep as a dark one: every driver wakes, + * the display stays off, and the root domain sleeps again as soon as it sees the lid closed. + */ +void apple_sleep_lid_closed_wake(void) +{ + struct t8140_sleep *s = t8140_sleep; + + if (!s || !s->smc_woke_alone) + return; + dev_info(s->dev, "the lid closed during the sleep: dark wake\n"); + pm_set_dark_wake(); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_wake); + /* * The radio's link and the engine that starts it, in macOS's order, measured on one * sleep: the link into L2, the engine stopped, PERST# asserted, the PHY's clocks off. @@ -2576,6 +2597,7 @@ static int t8140_sleep_begin(suspend_state_t state) if (t8140_sleep) { t8140_sleep->apcie_port_stopped = false; t8140_sleep->usb_down_ran = false; + t8140_sleep->smc_woke_alone = false; t8140_pins_save(t8140_sleep); } t8140_sleep_set_wd2(t8140_sleep, WDT_SUSPEND_SECS); @@ -2630,14 +2652,41 @@ static int t8140_sleep_prepare(void) return 0; } +/* Only the SMC's sources woke the SoC. */ +static bool t8140_sleep_smc_woke_alone(struct t8140_sleep *s) +{ + u32 smc[WAKE_WORDS] = {}; + bool any = false; + int i; + + /* SMC.CPUWakeupAP and SMC.OutboxNotEmpty, by their names. */ + for (i = 0; i < ARRAY_SIZE(t8140_wake_names); i++) { + u32 id = t8140_wake_names[i].id; + + if (strstarts(t8140_wake_names[i].name, "SMC.") && id / 32 < WAKE_WORDS) + smc[id / 32] |= BIT(id % 32); + } + for (i = 0; i < WAKE_WORDS; i++) { + u32 woke = s->wake_status[i] & s->wake_armed[i]; + + if (woke & ~smc[i]) + return false; + any |= !!woke; + } + + return any; +} + static void t8140_sleep_report(void) { struct t8140_sleep *s = t8140_sleep; int i; + s->smc_woke_alone = false; if (!s->returned) return; s->returned = false; + s->smc_woke_alone = t8140_sleep_smc_woke_alone(s); dev_info(s->dev, "back from sleep, boot object resumes %llu -> %llu\n", s->resumes_before, s->resumes_after); diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 39f2675178ebb..c167ae877030b 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -83,6 +83,13 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev); */ bool apple_sleep_debug_link_cut(void); +/* + * From the SMC's driver, in its noirq resume: the SMC woke the AP for the lid alone, and the + * lid reads closed. If nothing else woke the SoC, the PM core goes back to sleep once every + * device has resumed, as macOS does from a dark wake. + */ +void apple_sleep_lid_closed_wake(void); + /* The radio's host has its port back after a real sleep, so the next commit may stop it. */ void apple_sleep_radio_port_up(void); @@ -165,6 +172,10 @@ static inline bool apple_sleep_debug_link_cut(void) return false; } +static inline void apple_sleep_lid_closed_wake(void) +{ +} + static inline void apple_sleep_radio_port_up(void) { } From 978e9b2409b188aff8efe0b87e90fd6bb26d70f2 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:26 -0500 Subject: [PATCH 60/73] soc: apple: aop: stop the endpoints for a firmware-driven suspend The AOP keeps running across a sleep. From two traced macOS sleeps on T8140, and the same order in both, macOS stops its five AFK endpoints one at a time with 0xa4, each answered by 0x87, then sends the AP power state as idle. After the wake it sends the AP power state as on, then starts them again with the existing 0xa3, each answered by 0x86. Nothing restarts: no HELLO, no ring setup. So do that in suspend_late, after every child and the sound card have suspended, and undo it in resume, which also runs if the suspend is aborted later. A stop or notice that isn't answered within 1 s is undone and refuses the suspend. No EPIC call starts in between: a call checks for the sleep again once it holds its turn, so one that waited for it doesn't slip past the stop. The order is per-compatible match data, with T8140's taken from the trace. Other SoCs get endpoint order, and only a firmware-driven suspend reaches any of this, so s2idle is unchanged. Whether the AOP's setup port, which macOS's secure side owns, also hears about the sleep hasn't been traced, so nothing is sent there. Once the AOP's firmware has crashed it answers nothing, so every later sleep was refused when its AP idle notice went unanswered. Nothing short of a reboot restarts it, so a crash RTKit reports leaves the AOP out of every later sleep and wake: nothing is sent to it, the sleep goes ahead, and the sensors behind it stay gone until a reboot, as they already were. Signed-off-by: Diljit Singh --- drivers/soc/apple/aop.rs | 257 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 256 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index f36d1c7675181..6dca96203f9db 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -63,6 +63,7 @@ use kernel::{ atomic::{ Acquire, Atomic, + Relaxed, Release, // }, Arc, @@ -108,12 +109,16 @@ const AFK_OPC_INIT_UNK: u64 = 0x8c; const AFK_OPC_SEND: u64 = 0xa2; const AFK_OPC_START_ACK: u64 = 0x86; const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_SLEEP_ACK: u64 = 0x87; const AFK_OPC_RECV: u64 = 0x85; const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; const AFK_MSG_START: u64 = 0xa3 << 48; const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +/// Stops an endpoint for a system sleep. It keeps its rings, and `AFK_MSG_START` +/// brings it back. +const AFK_MSG_SLEEP: u64 = 0xa4 << 48; const AFK_RB_BLOCK_STEP: usize = 0x40; const EPIC_TYPE_NOTIFY: u32 = 0; const EPIC_CATEGORY_REPORT: u8 = 0x00; @@ -136,6 +141,8 @@ const EPIC_CALL_TIMEOUT_MS: u32 = 5000; /// Bound on the wait for the shutdown acknowledgments of all endpoints /// together, so that a firmware that stopped answering costs one timeout. const AFK_SHUTDOWN_TIMEOUT_MS: u32 = 5000; +/// Bound on the answer to each stop or start around a system sleep. +const AFK_SLEEP_TIMEOUT_MS: u32 = 1000; // The T8140 AOP boots through a second mailbox, its "setup port", besides the // RTKit one. Its management endpoint runs a HELLO / endpoint map / power @@ -202,6 +209,32 @@ mod aop_ring; /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one /// of the endpoints this driver drives. +/// Iterates over `list`, or over every AFK endpoint if `list` is empty. +struct AfkOrder { + list: &'static [u8], + next: usize, +} + +impl AfkOrder { + fn new(list: &'static [u8]) -> AfkOrder { + AfkOrder { list, next: 0 } + } +} + +impl Iterator for AfkOrder { + type Item = u8; + + fn next(&mut self) -> Option { + let at = self.next; + self.next += 1; + if self.list.is_empty() { + (at < AFK_ENDPOINT_COUNT as usize).then(|| AFK_ENDPOINT_START + at as u8) + } else { + self.list.get(at).copied() + } + } +} + fn afk_endpoint_index(ep: u8) -> Option { ep.checked_sub(AFK_ENDPOINT_START) .filter(|i| *i < AFK_ENDPOINT_COUNT) @@ -465,7 +498,7 @@ impl AFKEndpoint { self.recv_get_buf(client.dev.clone(), rtkit, msg)?; } AFK_OPC_INIT_UNK => {} // no-op - AFK_OPC_START_ACK => {} + AFK_OPC_START_ACK | AFK_OPC_SLEEP_ACK => client.sleep_answered(self.index), AFK_OPC_INIT_RX => { if self.rxbuf.is_some() { dev_err!( @@ -1251,6 +1284,21 @@ struct AopData { call_turn: [Mutex<()>; AFK_ENDPOINT_COUNT as usize], #[pin] ep_shutdown: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// Answers to the stop and start of each endpoint around a system sleep. + #[pin] + ep_sleep: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// The order endpoints stop for a system sleep and start after it. Empty + /// means all started ones in endpoint order. + sleep_endpoints: &'static [u8], + wake_endpoints: &'static [u8], + /// No call starts until the system has resumed. + sleeping: Atomic, + /// RTKit reported the firmware crashed. Nothing restarts it short of a reboot. + crashed: Atomic, + /// The endpoints stopped for this sleep, by index. + sleep_stopped: Atomic, + /// The firmware was told the AP is going to sleep. + ap_idle: Atomic, #[pin] hid_listeners: Mutex>, #[pin] @@ -1385,6 +1433,13 @@ impl AopData { }), call_turn <- pin_init::pin_init_array_from_fn(|_| new_mutex!(())), ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + ep_sleep <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + sleep_endpoints: cfg.sleep_endpoints, + wake_endpoints: cfg.wake_endpoints, + sleeping: Atomic::new(false), + crashed: Atomic::new(false), + sleep_stopped: Atomic::new(0), + ap_idle: Atomic::new(false), hid_listeners <- new_mutex!(KVec::new()), report_listeners <- new_mutex!(KVec::new()), source_ring <- new_mutex!(None), @@ -1592,6 +1647,127 @@ impl AopData { ret } + fn sleep_answered(&self, endpoint: u8) { + if let Some(index) = afk_endpoint_index(endpoint) { + self.ep_sleep[index].complete(()); + } + } + + /// Sends `msg` to one endpoint and waits for its answer. + fn sleep_switch(&self, endpoint: u8, msg: u64) -> Result<()> { + let index = afk_endpoint_index(endpoint).ok_or(EINVAL)?; + self.ep_sleep[index].reset(); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + rtk.send_message(endpoint, msg)?; + } + self.ep_sleep[index] + .wait_timeout(AFK_SLEEP_TIMEOUT_MS) + .ok_or(ETIMEDOUT) + } + + /// Tells the firmware about the AP's power state. Waits with the RTKit lock + /// dropped, as the answer shares a worker with messages that take it. + fn ap_power(&self, on: bool) -> Result<()> { + let rtk = (*self.rtkit.lock()) + .as_ref() + .map(|rtk| rtk.as_raw()) + .ok_or(ENODEV)?; + // SAFETY: the handle lives until removal, which holds the device lock + // that system sleep also runs under. + to_result(unsafe { bindings::apple_rtkit_ap_power(rtk, on) }) + } + + /// A crashed firmware hears nothing more, so it's left out of every sleep from then on, + /// rather than refusing them all. + fn crashed_skip(&self) -> bool { + self.crashed.load(Acquire) + } + + /// Puts back what `sleep_enter` did, in the order the firmware expects. + fn sleep_undo(&self) { + if self.crashed_skip() { + self.sleep_stopped.store(0, Relaxed); + self.ap_idle.store(false, Relaxed); + self.sleeping.store(false, Release); + return; + } + let stopped = self.sleep_stopped.xchg(0, Relaxed); + let was_idle = self.ap_idle.load(Relaxed); + if self.ap_idle.xchg(false, Relaxed) { + if let Err(e) = self.ap_power(true) { + dev_err!(self.dev, "Wake notice not answered ({:?})", e); + } + } + let order = AfkOrder::new(self.wake_endpoints); + for endpoint in order { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if stopped & (1 << index) == 0 { + continue; + } + if let Err(e) = self.sleep_switch(endpoint, AFK_MSG_START) { + dev_err!(self.dev, "Endpoint {:#04x} did not restart ({:?})", endpoint, e); + } + } + self.sleeping.store(false, Release); + if was_idle { + dev_dbg!(self.dev, "AP on notice sent and endpoints restarted after a sleep\n"); + } + } + + /// The AOP keeps running through a firmware-driven suspend. It stops its + /// endpoints and hears the AP's power state, then the reverse after a wake, + /// with no firmware restart. Measured on T8140, unverified on other SoCs. + fn sleep_enter(&self) -> Result<()> { + if !self.cpu_started.load(Acquire) { + return Ok(()); + } + if self.crashed_skip() { + dev_warn!(self.dev, "firmware crashed: sleeping without it\n"); + self.sleeping.store(true, Release); + return Ok(()); + } + self.sleeping.store(true, Release); + // Any call in progress finishes first; new ones see `sleeping`. + for turn in &self.call_turn { + drop(turn.lock()); + } + + let mut result = Ok(()); + for endpoint in AfkOrder::new(self.sleep_endpoints) { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if !self.endpoints[index].lock().started { + continue; + } + result = self.sleep_switch(endpoint, AFK_MSG_SLEEP).inspect_err(|e| { + dev_err!(self.dev, "Endpoint {:#04x} did not stop ({:?})", endpoint, e); + }); + if result.is_err() { + break; + } + self.sleep_stopped + .store(self.sleep_stopped.load(Relaxed) | (1 << index), Relaxed); + } + if result.is_ok() { + // Set first: a notice without an answer may still have been taken. + self.ap_idle.store(true, Relaxed); + result = self.ap_power(false).inspect_err(|e| { + dev_err!(self.dev, "Sleep notice not answered ({:?})", e); + }); + } + if result.is_err() { + self.sleep_undo(); + } else { + dev_dbg!(self.dev, "Endpoints stopped and AP idle notice answered for a sleep\n"); + } + result + } + fn patch_bootargs( &self, aop_mmio: &Mmio<'_, Region>, @@ -1902,8 +2078,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -1922,8 +2105,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -2541,6 +2731,7 @@ impl rtkit::Operations for AopData { } fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + data.crashed.store(true, Release); dev_err!(data.dev, "AOP firmware crashed"); } } @@ -2563,6 +2754,10 @@ struct AopHwConfig { epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. setup_port: bool, + /// The order endpoints stop for a system sleep; empty means endpoint order. + sleep_endpoints: &'static [u8], + /// The order they start after one. + wake_endpoints: &'static [u8], /// Shut the co-processor down on unbind and retain its buffers if the /// shutdown cannot be confirmed. quiesce_on_unbind: bool, @@ -2585,6 +2780,8 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2599,6 +2796,8 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2613,6 +2812,8 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2627,6 +2828,8 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2646,6 +2849,8 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: true, required_endpoints: &[], fw_regions: true, @@ -2669,6 +2874,8 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: true, + sleep_endpoints: &[0x2b, 0x23, 0x22, 0x20, 0x21], + wake_endpoints: &[0x20, 0x21, 0x22, 0x2b, 0x23], quiesce_on_unbind: true, required_endpoints: &[0x20, 0x21, 0x22, 0x23, 0x2b], fw_regions: false, @@ -2676,6 +2883,50 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { halt_on_failed_start: false, }; +/// The data of a bound AOP device. +/// +/// # Safety +/// +/// `dev` is an AOP device between the end of probe and the start of removal. +unsafe fn aop_from_dev<'a>(dev: *mut bindings::device) -> &'a AopData { + // SAFETY: by the caller's contract, drvdata is the `AopDriver` probe set. + let driver = unsafe { &*bindings::dev_get_drvdata(dev).cast::() }; + let data: *const dyn AOP = &*driver.0; + // SAFETY: probe made this `Arc` from an `Arc`. + unsafe { &*data.cast::() } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_suspend_late(dev: *mut bindings::device) -> c_int { + // SAFETY: only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: by this function's contract. + match unsafe { aop_from_dev(dev) }.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_resume(dev: *mut bindings::device) -> c_int { + // SAFETY: by this function's contract. + unsafe { aop_from_dev(dev) }.sleep_undo(); + 0 +} + +static AOP_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend_late: Some(aop_suspend_late), + resume: Some(aop_resume), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + kernel::of_device_table!( OF_TABLE, ::IdInfo, @@ -2696,6 +2947,10 @@ impl platform::Driver for AopDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, From d0d9c5f42002d5299cd250f275b7710b0964c546 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:27 -0500 Subject: [PATCH 61/73] ASoC: apple: take audio through a firmware-driven suspend The AOP keeps running across a firmware-driven suspend, so anything it was asked to run outlasts the streams: the serializers' clocks and the low-power microphone's reports keep going, and the latter would reach a sleeping AP. Nothing suspends the two microphone PCMs either, since their card isn't an ASoC one. On such a suspend, suspend those PCMs and return each of T8140's audio services to its idle state. None of these streams can resume, so userspace prepares them again, and prepare and trigger already ask for each service when it isn't running. MCA: hw_params is the only place a stream's cluster and serdes setup is written, and a stream that's merely suspended isn't given another. A sleep that takes those registers would leave it silent with no error, so refuse a firmware-driven suspend while a cluster is in use. ADMAC: a channel a client still holds keeps its carveout and configuration in registers, and a client only writes those at hw_params. Keep the client's last configuration and write it again before the next transfer starts, not earlier, as these controllers' registers may need the audio firmware to power them. A paused transfer's rings are gone as well, so resuming one starts it from the top. Signed-off-by: Diljit Singh --- drivers/dma/apple-admac.c | 66 ++++++++++++++++++++++++++++++++++-- sound/soc/apple/aop_audio.rs | 34 +++++++++++++++++++ sound/soc/apple/aop_t8140.rs | 47 +++++++++++++++++++++++++ sound/soc/apple/mca.c | 32 +++++++++++++++++ 4 files changed, 177 insertions(+), 2 deletions(-) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index b7336f97ee9fc..ee660cc38992f 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dmaengine.h" @@ -139,6 +140,12 @@ struct admac_chan { struct list_head issued; struct list_head to_free; + + /* The client's last configuration, set up again after a system sleep. */ + struct dma_slave_config cfg; + bool cfg_valid; + /* A system sleep took the channel's registers; set it up before a transfer. */ + bool lost; }; struct admac_sram { @@ -629,11 +636,30 @@ static void admac_reset_rings(struct admac_chan *adchan) writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, adchan->no)); } +static int admac_device_config(struct dma_chan *chan, struct dma_slave_config *config); + +/* Registers can't be reached before a client powers the fabric, so this waits for it. */ +static void admac_restore_chan(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + + if (!adchan->lost) + return; + adchan->lost = false; + + if (ad->split_channels) + writel_relaxed(0x18080001, ad->base + REG_CTL); + writel_relaxed(adchan->carveout, ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); + if (adchan->cfg_valid) + admac_device_config(&adchan->chan, &adchan->cfg); +} + static void admac_start_current_tx(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; int ch = adchan->no; + admac_restore_chan(adchan); admac_reset_rings(adchan); writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, ch)); if (ad->split_channels) { @@ -686,10 +712,15 @@ static int admac_resume(struct dma_chan *chan) int ret = 0; spin_lock_irqsave(&adchan->lock, flags); - if (adchan->current_tx && adchan->current_tx->error) + if (adchan->current_tx && adchan->current_tx->error) { ret = -EIO; - else + } else if (adchan->lost && adchan->current_tx) { + /* The rings went with the registers: start from the top. */ + adchan->nperiod_acks = 0; + admac_start_current_tx(adchan); + } else { admac_start_chan(adchan); + } spin_unlock_irqrestore(&adchan->lock, flags); return ret; @@ -1052,6 +1083,9 @@ static int admac_device_config(struct dma_chan *chan, | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), ad->base + REG_CHAN_FIFOCTL(ad, adchan->no)); + adchan->cfg = *config; + adchan->cfg_valid = true; + return 0; } @@ -1081,8 +1115,36 @@ static int admac_runtime_resume(struct device *dev) return 0; } +/* + * A system sleep that reached the hardware takes the registers of a channel a + * client still holds, and its client won't configure it again: only a new + * hw_params does that. + */ +static int __maybe_unused admac_resume_early(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + unsigned long flags; + int i; + + if (!pm_resume_via_firmware()) + return 0; + + mutex_lock(&ad->cache_alloc_lock); + ad->ctl_programmed = false; + mutex_unlock(&ad->cache_alloc_lock); + + for (i = 0; i < ad->nchannels; i++) { + spin_lock_irqsave(&ad->channels[i].lock, flags); + ad->channels[i].lost = true; + spin_unlock_irqrestore(&ad->channels[i].lock, flags); + } + + return 0; +} + /* Allocated channels must stay powered, including paused cyclic transfers. */ static const struct dev_pm_ops admac_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, admac_resume_early) RUNTIME_PM_OPS(admac_runtime_suspend, admac_runtime_resume, NULL) }; diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs index 4d3442bbfeafe..34f9c75b17d49 100644 --- a/sound/soc/apple/aop_audio.rs +++ b/sound/soc/apple/aop_audio.rs @@ -1092,6 +1092,36 @@ kernel::of_device_table!( ] ); +/// A system sleep with a firmware behind it. The firmware keeps running, so what it runs for +/// the streams has to stop first. Other variants' firmware isn't traced around a sleep. +/// +/// # Safety +/// +/// Called by the PM core with a live device of this driver. +unsafe extern "C" fn aop_audio_suspend(dev: *mut bindings::device) -> i32 { + // SAFETY: the helper only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: the PM core passes a live device, and keeps unbind away for this call. + let dev = unsafe { device::Device::::from_raw(dev) }; + let driver = dev.get_drvdata::(); + if driver.is_null() { + return 0; + } + // SAFETY: this driver stored it at probe, and it stays until unbind. + if let AopAudio::T8140(driver) = unsafe { &*driver } { + driver.sleep_stop(); + } + 0 +} + +static AOP_AUDIO_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend: Some(aop_audio_suspend), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + /// The bound driver, as the variant set it up. enum AopAudio { /// Held for its `Drop`, which frees the card. @@ -1105,6 +1135,10 @@ impl platform::Driver for AopAudio { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_AUDIO_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/sound/soc/apple/aop_t8140.rs b/sound/soc/apple/aop_t8140.rs index ff3bd53591276..a05964c76f43f 100644 --- a/sound/soc/apple/aop_t8140.rs +++ b/sound/soc/apple/aop_t8140.rs @@ -1032,6 +1032,34 @@ impl SndSocT8140AopData { Ok(()) } + /// `pw0 ` if running; the leaves stay with the stream. + fn service_stop(&self, svc: &Service) { + if svc.running.load(Relaxed) { + if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { + dev_err!(self.dev, "{}: pw0 failed: {:?}\n", svc.name, e); + } + svc.running.store(false, Relaxed); + } + } + + /// A system sleep: nothing the firmware runs may outlast the streams, whose DMA stops with + /// the sleep. Each one's next prepare or start asks for it again. + fn sleep_stop(&self) { + if self.powered.xchg(false, Relaxed) { + if let Err(e) = self.lpai_run_put() { + dev_err!(self.dev, "unable to return lpai to idle: {:?}\n", e); + } + } + self.flush_hpai_start(); + self.service_stop(&self.hpai); + if let Err(e) = self.spkr_prepare() { + dev_err!(self.dev, "spkr: stop failed: {:?}\n", e); + } + self.service_stop(&self.tap); + self.service_stop(&self.cout); + self.service_stop(&self.cin); + } + /// Called from sleeping prepare/shutdown paths while ALSA serializes /// this stream: a queued start must finish before the mute and pw0. fn flush_spkr_start(&self) { @@ -2557,6 +2585,8 @@ impl AsocContext { pub(crate) struct SndSocT8140AopDriver { card: *mut bindings::snd_card, + /// The card's two microphone PCMs, null until created. + pcms: [*mut bindings::snd_pcm; 2], data: Arc, asoc: KBox, asoc_registered: bool, @@ -2612,6 +2642,7 @@ impl SndSocT8140AopDriver { // whatever was set up. let mut this = SndSocT8140AopDriver { card: ptr::null_mut(), + pcms: [ptr::null_mut(); 2], asoc: AsocContext::new(data.clone())?, data: data.clone(), asoc_registered: false, @@ -2725,6 +2756,7 @@ impl SndSocT8140AopDriver { (*pcm).private_free = Some(pcm_free_private); (*pcm).info_flags = 0; } + self.pcms[device as usize] = pcm; // SAFETY: as above. copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; if device == 0 { @@ -2745,6 +2777,21 @@ impl SndSocT8140AopDriver { Ok(()) } + /// A system sleep: stop the card's own streams, then whatever the firmware runs for them + /// and for the sound card's. Userspace prepares its streams again afterwards, since none + /// of them can resume. + pub(crate) fn sleep_stop(&self) { + for pcm in self.pcms.iter().copied().filter(|pcm| !pcm.is_null()) { + // SAFETY: the PCMs live as long as the card, and the driver core keeps + // unbind away from a system sleep. + let ret = unsafe { bindings::snd_pcm_suspend_all(pcm) }; + if ret < 0 { + dev_err!(self.data.dev, "unable to suspend a PCM: {}\n", ret); + } + } + self.data.sleep_stop(); + } + /// Undo probe: stop the reports, unregister the ASoC component and free /// the card (which closes every stream, releasing its channel and power /// references), then return lpai to idle. Runs from unbind, and again diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 69abfcda7e813..af59d46784386 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -1456,10 +1457,41 @@ static const struct of_device_id apple_mca_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_mca_of_match); +/* + * hw_params is the only place a stream's cluster and serdes setup is written. A + * sleep that takes the registers would leave an open stream silent, and nothing + * here restores them yet. + */ +static int __maybe_unused apple_mca_suspend_late(struct device *dev) +{ + struct mca_data *mca = dev_get_drvdata(dev); + int i; + + if (!pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < mca->nclusters; i++) { + struct mca_cluster *cl = &mca->clusters[i]; + + if (cl->pd_link || cl->clocks_in_use[SNDRV_PCM_STREAM_PLAYBACK] || + cl->clocks_in_use[SNDRV_PCM_STREAM_CAPTURE]) { + dev_err(dev, "cluster %d is in use, refusing the sleep\n", i); + return -EBUSY; + } + } + + return 0; +} + +static const struct dev_pm_ops apple_mca_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_mca_suspend_late, NULL) +}; + static struct platform_driver apple_mca_driver = { .driver = { .name = "apple-mca", .of_match_table = apple_mca_of_match, + .pm = pm_sleep_ptr(&apple_mca_pm_ops), }, .probe = apple_mca_probe, .remove = apple_mca_remove, From a08244032108d314d47f35aa57c7f72bc6b49b4c Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:27 -0500 Subject: [PATCH 62/73] ASoC: apple: aop: t8140: switch the mic array's rail around each capture After a deep sleep the mic array recorded digital silence: every step succeeded and every period arrived, and neither a fresh open nor a rebind helped. The array has a power rail Linux never touched. macOS switches it on for an input stream as it starts the audio, and off once the transport has stopped, so under Linux it worked only for as long as it stayed up from boot, and the deep sleep drops it. Do as macOS does: take the "mic" supply before the service's pwrd and release it after its pw0, including the stop for a sleep. Probe defers until the supply can be had. Without one in the tree, the regulator core hands out a dummy and nothing changes. Signed-off-by: Diljit Singh --- sound/soc/apple/aop_t8140.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/aop_t8140.rs b/sound/soc/apple/aop_t8140.rs index a05964c76f43f..66948ae2d3d9f 100644 --- a/sound/soc/apple/aop_t8140.rs +++ b/sound/soc/apple/aop_t8140.rs @@ -60,6 +60,10 @@ use kernel::{ new_spinlock, platform, prelude::*, + regulator::{ + self, + Regulator, // + }, soc::apple::aop::{ from_fourcc, EPICService, @@ -490,6 +494,10 @@ struct SndSocT8140AopData { spkr_want_run: AtomicFlag, /// A microphone-array stream has been triggered and not yet stopped. hpai_want_run: AtomicFlag, + /// The array's rail (ADT function-input_power_enable), held while its + /// service runs. + #[pin] + hpai_supply: Mutex>>, /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. spkr_volume: Atomic, /// The sleeping firmware starts queued from the atomic triggers. @@ -953,7 +961,11 @@ impl SndSocT8140AopData { if !self.hpai_want_run.load(Relaxed) || self.hpai.running.load(Relaxed) { return Ok(()); } - self.service_run(&self.hpai) + // As macOS does: the rail, then the stream. A failed start drops it again. + let supply = Regulator::::get(&self.dev, c"mic")?; + self.service_run(&self.hpai)?; + *self.hpai_supply.lock() = Some(supply); + Ok(()) } /// Called from sleeping close paths: a queued start must finish before @@ -967,7 +979,9 @@ impl SndSocT8140AopData { fn hpai_shutdown(&self) { self.flush_hpai_start(); - self.service_shutdown(&self.hpai, &self.hpai_domains()) + self.service_shutdown(&self.hpai, &self.hpai_domains()); + // As macOS does: the stream's stop, then the rail. + *self.hpai_supply.lock() = None; } // ---- internal speakers: the spkr service on LEAP TX0 (leap-ns ADMAC @@ -1052,6 +1066,7 @@ impl SndSocT8140AopData { } self.flush_hpai_start(); self.service_stop(&self.hpai); + *self.hpai_supply.lock() = None; if let Err(e) = self.spkr_prepare() { dev_err!(self.dev, "spkr: stop failed: {:?}\n", e); } @@ -2884,6 +2899,9 @@ impl SndSocT8140AopDriver { // SAFETY: the pointer was checked above and stays valid with the child. let svc = unsafe { *service }; + // A capture takes the array's rail, which needs the SMC's GPIOs bound. + Regulator::::get(&dev, c"mic")?; + // The source ring: 512 KiB through our own IOMMU stream (dart-aop 9), // owned by the AOP driver because the firmware binds it once per boot. // SAFETY: no DMA is in flight on this device yet. @@ -2935,6 +2953,7 @@ impl SndSocT8140AopDriver { pd_rx0, spkr_want_run: AtomicFlag::new(false), hpai_want_run: AtomicFlag::new(false), + hpai_supply <- new_mutex!(None), spkr_volume: Atomic::new(SPKR_VOLUME_MAX), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), hpai_start_work <- new_work!("SndSocT8140AopData::hpai_start_work"), From f559a3fcfd8ba5255f37d463cf6430ae27c48333 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:28 -0500 Subject: [PATCH 63/73] soc: apple: sep: tell the enclave about a firmware-driven suspend A sleep powers the enclave down, and nothing told it. In five traced macOS sleeps on T8140 the words are the same every time: one control message, tag 8, type 0x0c, param 2, answered 18 ms later, after which the enclave's power state reads gated. After the wake macOS clears DEV_DISABLE in that state word, reprograms the enclave's DART, and 0.3 to 0.4 ms later rings a doorbell. 5.4 ms after that the enclave sends two messages of its own, and normal traffic follows with no new shared memory or discovery. macOS also clears bit 0 of the mailbox's receive control word straight after the answer, and sets it before the doorbell. In .suspend, when the suspend is firmware-driven, close a gate so that only answers to the enclave's own requests go out and wait for requests in flight. A gated enclave comes back only through a real sleep, so where the T8140 sleep controller takes it, the request is left to the sleep's very last refusal: the driver holds its receive interrupt and hands the shim the request and its answer's match (the low 24 bits of the traced answer). In .enter, with interrupts off, the shim takes the mailbox's receive callback and passes what was waiting on to the driver, saves the DART with macOS's writes, sends the request and polls for the answer, and stops reading the mailbox once DEV_DISABLE reads set. It then clears the receive enable and waits up to 5 s for DEV_DISABLE, as XNU does. A sleep refused anywhere before finds the enclave never told and its DART untouched: the receive interrupt is released and the gate opens again. Elsewhere, save the DART through its consumer hooks, send the request and wait for its answer in .suspend, and in .suspend_noirq wait for DEV_DISABLE in the domain's state word, which the enclave sets itself as it gates, refusing if it doesn't. In .resume after a real sleep, clear DEV_DISABLE, restore the DART, release the mailbox and ring the doorbell, then wait for the two messages. Nothing may read a gated enclave's mailbox: on a running J700 a wake path that did stalled the CPU with interrupts off. Only the power cycle of a real sleep brings it back, so after a sleep refused once it was told, which only the request's own failure can do where the sleep controller sends it, the enclave is left asleep with its receive interrupt held off, and requests fail until the next real sleep or a reboot. An enclave that never finished coming up isn't told of the sleep; the sleep controller refuses a sleep while it still runs. The SEP's domains are syscore devices for the sleep, since the enclave gates itself before genpd would. Signed-off-by: Diljit Singh --- drivers/soc/apple/control.rs | 92 +++++++- drivers/soc/apple/pm.rs | 198 +++++++++++++++- drivers/soc/apple/pm_shim.c | 394 ++++++++++++++++++++++++++++++- drivers/soc/apple/proto.rs | 68 ++++++ drivers/soc/apple/sep.rs | 304 +++++++++++++++++++++++- drivers/soc/apple/shim.h | 11 +- rust/kernel/soc/apple/mailbox.rs | 5 + 7 files changed, 1060 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 946c95d7a46d3..644354f5ddb06 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -164,6 +164,7 @@ pub(crate) struct ControlState { retired: [u64; 4], retired_count: u32, entropy: EntropySink, + wake_seen: u8, } impl ControlState { @@ -181,6 +182,7 @@ impl ControlState { retry_at: None, retry_ms: ENTROPY_RETRY_MIN_MS, }, + wake_seen: 0, } } @@ -218,7 +220,7 @@ impl ControlState { } else { tag + 1 }; - if !self.tag_in_flight(tag) && !self.is_retired(tag) { + if tag != proto::TAG_SLEEP && !self.tag_in_flight(tag) && !self.is_retired(tag) { self.slots[idx] = Slot { used: true, tag, @@ -230,6 +232,45 @@ impl ControlState { Err(EBUSY) } + /// Claims the sleep request's own tag, which the pool never hands out. A + /// request still in flight under it means the last one was never answered. + pub(crate) fn alloc_sleep(&mut self) -> Result { + let tag = proto::TAG_SLEEP; + if self.tag_in_flight(tag) { + return Err(EBUSY); + } + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + self.reclaim_retired(tag); + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + Ok(idx) + } + + /// Whether any request is waiting for its reply. + pub(crate) fn requests_in_flight(&self) -> bool { + self.slots.iter().any(|s| s.used) + } + + /// Forget the wake messages seen, before asking for them. + pub(crate) fn wake_reset(&mut self) { + self.wake_seen = 0; + } + + /// A control message that isn't a reply: note it if it's a wake message. + pub(crate) fn wake_note(&mut self, ty: u8) { + if let Some(n) = proto::WAKE_TYPES.iter().position(|&t| t == ty) { + self.wake_seen |= 1 << n; + } + } + + /// All the wake messages have come. + pub(crate) fn wake_complete(&self) -> bool { + self.wake_seen == (1 << proto::WAKE_TYPES.len()) - 1 + } + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { self.slots[idx].reply.take() } @@ -419,6 +460,55 @@ mod tests { } } + #[test] + fn the_pool_never_hands_out_the_sleep_tag() { + let mut state = ControlState::new(); + for _ in 0..POOL_LEN * 2 { + let (idx, tag) = state.alloc().unwrap(); + assert_ne!(tag, proto::TAG_SLEEP); + state.release(idx); + } + } + + #[test] + fn a_sleep_request_gets_its_own_answer() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + assert!(state.requests_in_flight()); + assert_eq!(state.alloc_sleep(), Err(EBUSY)); + let reply = proto::ControlReply { + tag: proto::TAG_SLEEP, + data_lo: 0, + }; + assert!(matches!(state.deliver(reply), Delivery::Matched)); + assert!(state.take_reply(idx).is_some()); + state.release(idx); + assert!(!state.requests_in_flight()); + } + + #[test] + fn a_sleep_request_can_follow_one_that_timed_out() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + state.abandon(idx); + assert!(state.alloc_sleep().is_ok()); + } + + #[test] + fn a_wake_takes_both_messages() { + let mut state = ControlState::new(); + state.wake_reset(); + assert!(!state.wake_complete()); + state.wake_note(0x33); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[0]); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[1]); + assert!(state.wake_complete()); + state.wake_reset(); + assert!(!state.wake_complete()); + } + #[test] fn a_request_is_answered() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 3701f08c18487..a65e5920102f4 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -8,6 +8,10 @@ //! is not frozen either, so entropy requests are held over the same window. //! The notifier itself is in `pm_shim.c`; the capture and entropy handling is //! on [`SepData`]. +//! +//! On a firmware-driven suspend the enclave also saves its state and gates its +//! own power domain when asked, as the sleep powers it down, and has to be +//! started again after a wake. See [`SleepGate`] and `SepData::sleep_enter`. use core::sync::atomic::{AtomicPtr, Ordering}; @@ -23,9 +27,138 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; + fn sep_pm_register( + dev: *mut c_void, + event: unsafe extern "C" fn(bool), + sleep: unsafe extern "C" fn() -> c_int, + wake: unsafe extern "C" fn(), + told: unsafe extern "C" fn() -> bool, + ) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; + fn sep_pm_wait_gated(dev: *mut c_void, timeout_ms: c_uint) -> c_int; + fn sep_pm_clear_dev_disable(dev: *mut c_void) -> c_int; + fn sep_pm_dart_prepare(dev: *mut c_void) -> c_int; + fn sep_pm_dart_restore(dev: *mut c_void); + fn sep_pm_commit_prepare( + dev: *mut c_void, + mbox: *mut c_void, + msg0: u64, + msg1: u32, + reply: u64, + mask: u64, + ) -> c_int; + fn sep_pm_commit_finish(dev: *mut c_void) -> c_int; + pub(crate) static sep_pm_ops: kernel::bindings::dev_pm_ops; +} + +/// How long the enclave gets to gate its domain once it has answered. +const GATE_TIMEOUT_MS: u32 = 5000; + +/// Who may use the mailbox around a firmware-driven suspend. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SleepGate { + /// Anyone. + Open = 0, + /// Replies to what the enclave asked, and the sleep request itself. + Draining = 1, + /// Nobody until the wake: the enclave is gated or about to be. + Told = 2, + /// Nobody: the enclave did not come back. A later sleep may revive it. + Lost = 3, +} + +impl SleepGate { + pub(crate) fn from_u32(v: u32) -> SleepGate { + match v { + 0 => SleepGate::Open, + 1 => SleepGate::Draining, + 2 => SleepGate::Told, + _ => SleepGate::Lost, + } + } + + pub(crate) fn may_request(self) -> bool { + self == SleepGate::Open + } + + pub(crate) fn may_reply(self) -> bool { + matches!(self, SleepGate::Open | SleepGate::Draining) + } + + /// The enclave may be gated, so a wake has to look. + pub(crate) fn needs_wake(self) -> bool { + matches!(self, SleepGate::Told | SleepGate::Lost) + } +} + +/// Whether the enclave gated itself within the bound. +pub(crate) fn wait_gated(dev: &kernel::device::Device) -> bool { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_wait_gated(dev.as_raw().cast(), GATE_TIMEOUT_MS) == 0 } +} + +/// Saves the enclave's DART while it can be read, before the enclave is told. +pub(crate) fn dart_prepare(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_dart_prepare(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Leaves `request` to the sleep driver's last refusal, which sends it with interrupts off +/// and saves the DART just before. Its answer is the message whose `mask` bits read `reply`. +/// False where there's no such step, and the request goes now. +pub(crate) fn leave_for_commit( + dev: &kernel::device::Device, + mbox: *mut kernel::bindings::apple_mbox, + request: kernel::soc::apple::mailbox::Message, + reply: u64, + mask: u64, +) -> bool { + // SAFETY: `dev` is the bound SEP device and `mbox` its mailbox, which outlives the sleep: + // `commit_finish` takes the request back at the resume. + let rc = unsafe { + sep_pm_commit_prepare( + dev.as_raw().cast(), + mbox.cast(), + request.msg0, + request.msg1, + reply, + mask, + ) + }; + rc == 0 +} + +/// After a sleep whose request was left to its last refusal: whether it went out. `None` if +/// none was left. +pub(crate) fn commit_finish(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_commit_finish(dev.as_raw().cast()) }; + if rc < 0 { + None + } else { + Some(rc > 0) + } +} + +/// Puts the enclave's DART back if it lost its state, before the doorbell. +pub(crate) fn dart_restore(dev: &kernel::device::Device) { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_dart_restore(dev.as_raw().cast()) }; +} + +/// Lets a gated enclave start again. Writes only if the bit is set. +pub(crate) fn clear_dev_disable(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_clear_dev_disable(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) } /// Attaches every power domain of a multi-domain SEP node and keeps them on @@ -92,7 +225,41 @@ unsafe extern "C" fn pm_event(entering: bool) { sleep_event(&*sep, entering); } +/// Runs `f` on the registered instance. Only for callbacks that +/// `sep_pm_unregister` waits for or that the bound device excludes. +fn with_sep(none: R, f: impl FnOnce(&SepData) -> R) -> R { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return none; + } + // SAFETY: as in `pm_event`. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + f(&*sep) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's suspend callback. +unsafe extern "C" fn pm_sleep() -> c_int { + with_sep(0, |sep| match sep.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + }) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's resume callback. +unsafe extern "C" fn pm_wake() { + with_sep((), |sep| sep.sleep_exit()); +} + +/// # Safety +/// Called by `pm_shim.c`, whether or not the device suspended. +unsafe extern "C" fn pm_told() -> bool { + with_sep(false, |sep| sep.sleep_gate() == SleepGate::Told) +} + pub(crate) fn register(sep: Arc) -> Result<()> { + let dev: *mut c_void = sep.dev.as_raw().cast(); let ptr = sep.into_foreign(); if SEP @@ -111,7 +278,7 @@ pub(crate) fn register(sep: Arc) -> Result<()> { // SAFETY: the shim stores the callback and registers a notifier block // that lives for the module's lifetime. - let rc = unsafe { sep_pm_register(pm_event) }; + let rc = unsafe { sep_pm_register(dev, pm_event, pm_sleep, pm_wake, pm_told) }; if rc != 0 { let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); if !raw.is_null() { @@ -187,6 +354,33 @@ mod tests { } } + #[test] + fn the_gate_lets_replies_out_until_the_enclave_is_told() { + assert!(SleepGate::Open.may_request()); + assert!(SleepGate::Open.may_reply()); + assert!(!SleepGate::Draining.may_request()); + assert!(SleepGate::Draining.may_reply()); + assert!(!SleepGate::Told.may_request()); + assert!(!SleepGate::Told.may_reply()); + assert!(!SleepGate::Lost.may_request()); + assert!(!SleepGate::Lost.may_reply()); + } + + #[test] + fn only_a_told_or_lost_enclave_needs_waking() { + assert!(!SleepGate::Open.needs_wake()); + assert!(!SleepGate::Draining.needs_wake()); + assert!(SleepGate::Told.needs_wake()); + assert!(SleepGate::Lost.needs_wake()); + } + + #[test] + fn the_gate_round_trips() { + for gate in [SleepGate::Open, SleepGate::Draining, SleepGate::Told, SleepGate::Lost] { + assert_eq!(SleepGate::from_u32(gate as u32), gate); + } + } + #[test] fn going_to_sleep_holds_entropy_then_captures() { let calls = Calls::new(); diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 884ecbb354a06..adfb9c90688a4 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -3,18 +3,41 @@ * System sleep for the SEP driver: the notifier only forwards "about to sleep" * and "back from sleep" to Rust, which ends any Touch ID capture before the * system suspends (see pm.rs), and the enclave's power domains are kept on - * across sleep. + * across sleep. The device's suspend and resume callbacks are the firmware + * suspend's: the enclave is told, and gates its own domain, whose state word is + * read here. On T8140 at EL2 the sleep driver sends the request, as its last + * refusal. */ #include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include #include #include "shim.h" +/* The state word of a PMGR power domain, as pmgr-pwrstate has it. */ +#define SEP_PS_DEV_DISABLE BIT(10) + static void (*sep_pm_event)(bool entering); +static int (*sep_pm_sleep)(void); +static void (*sep_pm_wake)(void); +static bool (*sep_pm_told)(void); +/* In the device's .suspend, the only caller that can leave the request to the sleep. */ +static bool sep_pm_suspending; static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) @@ -44,15 +67,375 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(void (*event)(bool entering)) +/* The enclave's own domain is the first one its node lists. */ +static int sep_pm_domain(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + + *regmap = syscon_node_to_regmap(np->parent); + ret = PTR_ERR_OR_ZERO(*regmap); + if (!ret) + ret = of_property_read_u32(np, "reg", offset); + of_node_put(np); + + return ret; +} + +/* + * Returns 0 once the enclave has gated itself, which shows as DEV_DISABLE, and + * -ENOENT if there is no domain to read. The actual state is no guide: a running + * enclave idles at 0 with auto-enable on. + */ +int sep_pm_wait_gated(void *ptr, unsigned int timeout_ms) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + return regmap_read_poll_timeout(regmap, offset, val, + val & SEP_PS_DEV_DISABLE, + 1000, timeout_ms * USEC_PER_MSEC); +} + +int sep_pm_clear_dev_disable(void *ptr) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + ret = regmap_read(regmap, offset, &val); + if (ret || !(val & SEP_PS_DEV_DISABLE)) + return ret; + + return regmap_write(regmap, offset, val & ~SEP_PS_DEV_DISABLE); +} + +/* The enclave's DART, the first one its node lists, with a reference held. */ +static struct device *sep_pm_dart(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + + np = of_parse_phandle(dev->of_node, "iommus", 0); + if (!np) + return NULL; + pdev = of_find_device_by_node(np); + of_node_put(np); + + return pdev ? &pdev->dev : NULL; +} + +int sep_pm_dart_prepare(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + int ret; + + if (!dart) + return -ENODEV; + ret = apple_dart_sleep_prepare(dart); + put_device(dart); + + return ret; +} + +void sep_pm_dart_restore(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + + if (dart) { + apple_dart_sleep_restore(dart); + put_device(dart); + } +} + +#define SEP_PM_POLL_US 20 +#define SEP_PM_TIMEOUT_US (5000 * USEC_PER_MSEC) + +/* + * The sleep request, sent by the sleep driver as its very last refusal, with interrupts off: + * an enclave that gated doesn't come back with its power kept, so a sleep refused anywhere + * before must find it never told. The driver hands over the request and its answer's match + * at .suspend, with its receive interrupt held. + */ +static struct sep_pm_commit { + struct device *dev; + struct device *dart; + struct apple_mbox *mbox; + void __iomem *ps; + struct apple_mbox_msg request; + u64 reply, reply_mask; + void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); + void *cookie; + bool registered, sent, answered; + unsigned int answer_us; + int gate_ret; + u32 state; +} sep_pm_commit; + +/* The answer stops here. Anything else goes to the driver, which reads it after the wake. */ +static void sep_pm_commit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct sep_pm_commit *c = cookie; + + if (c->sent && (msg.msg0 & c->reply_mask) == c->reply) { + c->answered = true; + return; + } + if (c->rx) + c->rx(mbox, msg, c->cookie); +} + +static void sep_pm_commit_take_rx(struct sep_pm_commit *c, bool take) +{ + unsigned long flags; + + spin_lock_irqsave(&c->mbox->rx_lock, flags); + if (take) { + c->rx = c->mbox->rx; + c->cookie = c->mbox->cookie; + c->mbox->rx = sep_pm_commit_rx; + c->mbox->cookie = c; + } else { + c->mbox->rx = c->rx; + c->mbox->cookie = c->cookie; + } + spin_unlock_irqrestore(&c->mbox->rx_lock, flags); +} + +/* 0 once the enclave has gated. A sleep refused after it was sent finds it told. */ +static int sep_pm_commit_tell(void *data) +{ + struct sep_pm_commit *c = data; + unsigned int us; + int ret = 0; + + c->sent = false; + c->answered = false; + c->answer_us = 0; + c->gate_ret = -EINVAL; + + sep_pm_commit_take_rx(c, true); + /* What was already waiting first. */ + apple_mbox_poll(c->mbox); + for (us = 0; !apple_mbox_send_ready(c->mbox); us += SEP_PM_POLL_US) { + if (us >= SEP_PM_TIMEOUT_US) { + ret = -ETIMEDOUT; + goto out; + } + udelay(SEP_PM_POLL_US); + } + /* Its DART is saved, with macOS's writes, right before the request, as at .suspend. */ + ret = apple_dart_sleep_prepare(c->dart); + if (ret) + goto out; + ret = apple_mbox_send(c->mbox, c->request, true); + if (ret) { + /* Not a real sleep: this only drops the saved copy. */ + apple_dart_sleep_restore(c->dart); + goto out; + } + c->sent = true; + /* Never read a gated enclave's mailbox: that stalls the CPU for good. */ + while (!c->answered && !(readl(c->ps) & SEP_PS_DEV_DISABLE) && + c->answer_us < SEP_PM_TIMEOUT_US) { + apple_mbox_poll(c->mbox); + if (!c->answered) { + udelay(SEP_PM_POLL_US); + c->answer_us += SEP_PM_POLL_US; + } + } +out: + sep_pm_commit_take_rx(c, false); + if (!c->sent) + return ret; + if (c->answered) + apple_mbox_set_recv_enabled(c->mbox, false); + c->gate_ret = readl_poll_timeout_atomic(c->ps, c->state, c->state & SEP_PS_DEV_DISABLE, + SEP_PM_POLL_US, SEP_PM_TIMEOUT_US); + + return c->gate_ret; +} + +/* + * From .suspend, with the enclave drained and its receive interrupt held: leaves the + * request in @msg0 and @msg1 to the sleep driver's last refusal. Its answer is the message + * whose @mask bits read @reply. -EOPNOTSUPP where there's no such step. + */ +int sep_pm_commit_prepare(void *ptr, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask) +{ + struct sep_pm_commit *c = &sep_pm_commit; + struct device *dev = ptr; + struct device_node *np; + struct resource res; + u32 offset; + int ret; + + if (!READ_ONCE(sep_pm_suspending)) + return -EOPNOTSUPP; + if (c->registered) + return -EBUSY; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + ret = of_property_read_u32(np, "reg", &offset); + if (!ret) + ret = np->parent ? of_address_to_resource(np->parent, 0, &res) : -EINVAL; + of_node_put(np); + if (ret) + return ret; + if (offset + 4 > resource_size(&res)) + return -EINVAL; + + c->dart = sep_pm_dart(dev); + if (!c->dart || !dev_get_drvdata(c->dart)) { + ret = -ENODEV; + goto err; + } + c->ps = ioremap(res.start + offset, 4); + if (!c->ps) { + ret = -ENOMEM; + goto err; + } + c->dev = dev; + c->mbox = mbox; + c->request.msg0 = msg0; + c->request.msg1 = msg1; + c->reply = reply; + c->reply_mask = mask; + c->sent = false; + ret = apple_sleep_enclave_register(dev, sep_pm_commit_tell, c); + if (ret) + goto err; + c->registered = true; + + return 0; + +err: + if (c->ps) + iounmap(c->ps); + c->ps = NULL; + if (c->dart) + put_device(c->dart); + c->dart = NULL; + return ret; +} + +/* After the sleep, refused or not: 1 if the request went out, 0 if not, <0 if none was left. */ +int sep_pm_commit_finish(void *ptr) +{ + struct sep_pm_commit *c = &sep_pm_commit; + + if (!c->registered || c->dev != ptr) + return -ENOENT; + apple_sleep_enclave_unregister(c->dev); + c->registered = false; + if (c->sent) + dev_dbg(c->dev, "sleep request at the last refusal: %s in %u us, gate %d, state %#x\n", + c->answered ? "answered" : "no answer", c->answer_us, c->gate_ret, + c->state); + iounmap(c->ps); + c->ps = NULL; + put_device(c->dart); + c->dart = NULL; + + return c->sent; +} + +/* + * Only a firmware-driven suspend tells the enclave anything, so every other sleep + * stays as it was. Refusing is safe: the PM core runs the resume callback of a + * device whose suspend had passed when a later one failed, and sleep_enter undoes + * what it did if its own step fails. + */ +static int __maybe_unused sep_pm_suspend(struct device *dev) +{ + int (*sleep)(void) = READ_ONCE(sep_pm_sleep); + int ret; + + if (!pm_suspend_via_firmware() || !sleep) + return 0; + + WRITE_ONCE(sep_pm_suspending, true); + ret = sleep(); + WRITE_ONCE(sep_pm_suspending, false); + + return ret; +} + +/* The last step before the platform's own: the domain has to be down by now. */ +static int __maybe_unused sep_pm_suspend_noirq(struct device *dev) +{ + bool (*told)(void) = READ_ONCE(sep_pm_told); + int ret; + + if (!pm_suspend_via_firmware() || !told || !told()) + return 0; + + ret = sep_pm_wait_gated(dev, 5000); + if (ret == -ENOENT) + return 0; + if (ret) { + dev_err(dev, "the enclave did not gate its power domain: %d\n", ret); + } else { + struct regmap *regmap; + u32 offset, val = 0; + + if (!sep_pm_domain(dev, ®map, &offset)) + regmap_read(regmap, offset, &val); + dev_dbg(dev, "the enclave gated its power domain, state %#x\n", val); + } + + return ret; +} + +static int __maybe_unused sep_pm_resume(struct device *dev) +{ + void (*wake)(void) = READ_ONCE(sep_pm_wake); + + if (wake) + wake(); + + return 0; +} + +const struct dev_pm_ops sep_pm_ops = { + .suspend = pm_sleep_ptr(sep_pm_suspend), + .resume = pm_sleep_ptr(sep_pm_resume), + .suspend_noirq = pm_sleep_ptr(sep_pm_suspend_noirq), +}; + +int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)) { int ret; WRITE_ONCE(sep_pm_event, event); + WRITE_ONCE(sep_pm_sleep, sleep); + WRITE_ONCE(sep_pm_wake, wake); + WRITE_ONCE(sep_pm_told, told); ret = register_pm_notifier(&sep_pm_nb); - if (ret) + if (ret) { WRITE_ONCE(sep_pm_event, NULL); - return ret; + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); + return ret; + } + + return 0; } /* Returns only once no notifier call is still running. */ @@ -60,6 +443,9 @@ void sep_pm_unregister(void) { unregister_pm_notifier(&sep_pm_nb); WRITE_ONCE(sep_pm_event, NULL); + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); } static void sep_pm_release_domains(void *data) diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 8fc7278550d9a..1bdea86a4c51e 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -90,6 +90,32 @@ pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; pub(crate) const TAG_ENTROPY: u8 = 0xE7; +/// The tag of the sleep request, kept out of the pool. +pub(crate) const TAG_SLEEP: u8 = 0x08; + +/// The sleep request's answer, in the bits [`SLEEP_REPLY_MASK`] keeps: the control endpoint, +/// its tag and the reply type. +pub(crate) const SLEEP_REPLY: u64 = (EP_CONTROL as u64) + | ((TAG_SLEEP as u64) << MSG_TAG_SHIFT) + | ((CONTROL_REPLY_TYPE as u64) << MSG_TYPE_SHIFT); +pub(crate) const SLEEP_REPLY_MASK: u64 = (1 << MSG_PARAM_SHIFT) - 1; + +// The answer in the trace, 0x0000_0000_0001_0800. +static_assert!(0x0000_0000_0001_0800 & SLEEP_REPLY_MASK == SLEEP_REPLY); + +/// How long the enclave gets to answer the sleep request. +pub(crate) const SLEEP_TIMEOUT_MS: u32 = 5000; + +/// The first message macOS sends a gated enclave after a sleep. The contents +/// vary from sleep to sleep, so it is taken to be a doorbell. This is one of them. +pub(crate) const WAKE_KICK: Message = Message { + msg0: 0xffff_ffff_a482_9303, + msg1: 0, +}; + +/// The control messages a woken enclave sends without being asked, in order. +pub(crate) const WAKE_TYPES: [u8; 2] = [0x0d, 0x10]; + pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; pub(crate) const TAG_POOL_FIRST: u8 = 0x01; @@ -97,6 +123,7 @@ pub(crate) const TAG_POOL_LAST: u8 = 0x7e; static_assert!(TAG_POOL_FIRST > 0); static_assert!(TAG_POOL_LAST < TAG_ENTROPY); +static_assert!(TAG_SLEEP >= TAG_POOL_FIRST && TAG_SLEEP <= TAG_POOL_LAST); pub(crate) struct ControlOp { ty: u8, @@ -161,6 +188,19 @@ pub(crate) fn op_get_entropy() -> ControlOp { } } +/// Asks the enclave to save its state and gate its own power domain. +pub(crate) fn op_sleep() -> ControlOp { + ControlOp { + ty: 0x0c, + param: 0x02, + data: 0, + expect_reply: true, + timeout_ms: SLEEP_TIMEOUT_MS, + reserved_tag: false, + name: c"SLEEP", + } +} + pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { Message { msg0: u64::from(EP_CONTROL) @@ -282,6 +322,34 @@ mod tests { assert_eq!(f.data_lo, 0x1234_5678); } + #[test] + fn sleep_request_matches_the_trace() { + let msg = encode_control(&op_sleep(), TAG_SLEEP); + assert_eq!(msg.msg0, 0x0000_0000_020c_0800); + assert_eq!(msg.msg1, 0); + } + + #[test] + fn its_answer_matches_the_trace() { + let reply = Message { + msg0: 0x0000_0000_0001_0800, + msg1: 0, + }; + let f = decode(&reply); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, CONTROL_REPLY_TYPE); + assert_eq!(f.tag, TAG_SLEEP); + } + + #[test] + fn wake_messages_match_the_trace() { + for (raw, ty) in [(0x000d_0000u64, WAKE_TYPES[0]), (0x0010_0000u64, WAKE_TYPES[1])] { + let f = decode(&Message { msg0: raw, msg1: 0 }); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, ty); + } + } + #[test] fn only_entropy_uses_the_reserved_tag() { assert!(op_get_entropy().uses_reserved_tag()); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 660bb9d1854b8..fac83c41ba5d9 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -60,8 +60,10 @@ use kernel::{ sync::{ aref::ARef, atomic::{ + Acquire, Atomic, - Relaxed, // + Relaxed, + Release, // }, barrier::{ dma_mb, @@ -636,6 +638,12 @@ struct SepData { // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, + // A `pm::SleepGate`. + sleep_gate: Atomic, + + // The mailbox's receive interrupt is held off for the enclave's sleep. + sleep_rx_held: Atomic, + #[pin] ool_xarm: Mutex>, @@ -1037,6 +1045,8 @@ impl SepData { sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), + sleep_gate: Atomic::new(pm::SleepGate::Open as u32), + sleep_rx_held: Atomic::new(false), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), @@ -1286,7 +1296,276 @@ impl SepData { if self.shutting_down.load(Relaxed) { return Err(ENODEV); } - Self::ring(self.mbox.lock().as_ref().ok_or(ENODEV)?, msg) + // Checked with the mailbox lock held: a sender can wait for it for longer + // than the system takes to tell the enclave it's going to sleep. + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_request() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + /// As [`Self::send`], for an answer to something the enclave asked: it + /// waits for these before it can answer the sleep request. + fn send_reply(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_reply() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + pub(crate) fn sleep_gate(&self) -> pm::SleepGate { + pm::SleepGate::from_u32(self.sleep_gate.load(Acquire)) + } + + /// Under the mailbox lock, so that a sender holding it finishes first. + fn set_sleep_gate(&self, gate: pm::SleepGate) { + let _mbox = self.mbox.lock(); + self.sleep_gate.store(gate as u32, Release); + } + + /// Asks the enclave to go to sleep. The request is sent past the gate. + fn sleep_request(&self) -> Result<()> { + let op = proto::op_sleep(); + let tag = proto::TAG_SLEEP; + let idx = self.control.lock().alloc_sleep()?; + let msg = proto::encode_control(&op, tag); + + let sent = match self.mbox.lock().as_ref() { + Some(mbox) => Self::ring(mbox, msg), + None => Err(ENODEV), + }; + if let Err(e) = sent { + self.control.lock().abandon(idx); + return Err(e); + } + match self.control_wait(&op, idx, tag)? { + Some(_) => Ok(()), + None => Err(ETIMEDOUT), + } + } + + /// A firmware-driven suspend: tell the enclave, which answers, saves its + /// state and gates its own domain, and read and send nothing more until the + /// wake. + /// A failure leaves the enclave as it was if it can. + pub(crate) fn sleep_enter(&self) -> Result<()> { + match self.sleep_gate() { + // Gated already, or never coming back: nothing to tell it. + pm::SleepGate::Lost => return Ok(()), + pm::SleepGate::Open => {} + _ => return Err(EBUSY), + } + // Only an enclave that came up is told anything. The sleep goes on without one that didn't. + if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { + dev_warn!( + self.dev, + "the enclave did not finish coming up, so it isn't told of this sleep\n" + ); + return Ok(()); + } + + self.set_sleep_gate(pm::SleepGate::Draining); + let idle = control::wait_idle( + proto::CONTROL_TIMEOUT_MS, + 10, + || self.control.lock().requests_in_flight(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_err!(self.dev, "sleep refused: a request is still waiting for its reply\n"); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(EBUSY); + } + + if self.sleep_leave_for_commit() { + return Ok(()); + } + + if let Err(e) = pm::dart_prepare(&self.dev) { + dev_err!(self.dev, "sleep refused: its DART could not be saved ({:?})\n", e); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(e); + } + + match self.sleep_request() { + Ok(()) => { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.set_recv_enabled(false); + } + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Told); + dev_dbg!(self.dev, "the enclave answered the sleep request\n"); + Ok(()) + } + Err(e) => { + dev_err!(self.dev, "sleep refused: no answer to the sleep request ({:?})\n", e); + // A request that timed out may still have been taken. + self.set_sleep_gate(pm::SleepGate::Told); + self.sleep_exit(); + Err(e) + } + } + } + + /// Where the sleep driver takes it, the request goes out as its very last refusal, with + /// interrupts off: a gated enclave comes back only through a real sleep, so any earlier + /// refusal has to find it never told. Its answer is polled there, so the receive + /// interrupt is held from now on. The gate stays at draining until the wake. + fn sleep_leave_for_commit(&self) -> bool { + let request = proto::encode_control(&proto::op_sleep(), proto::TAG_SLEEP); + let left = match self.mbox.lock().as_ref() { + Some(mbox) => pm::leave_for_commit( + &self.dev, + mbox.as_raw(), + request, + proto::SLEEP_REPLY, + proto::SLEEP_REPLY_MASK, + ), + None => false, + }; + if left { + self.sleep_hold_rx(); + dev_dbg!(self.dev, "the sleep request is left for the sleep's last refusal\n"); + } + left + } + + /// Holds off the receive interrupt, whose handler reads the mailbox. + fn sleep_hold_rx(&self) { + if !self.sleep_rx_held.xchg(true, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.hold_rx(); + } + } + } + + fn sleep_release_rx(&self) { + if self.sleep_rx_held.xchg(false, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.release_rx(); + } + } + } + + /// Back from a sleep, or a suspend that was aborted. The enclave has kept + /// its state and takes no new endpoint setup. + /// + /// Reading the mailbox of a gated enclave can stall the CPU for good, so only + /// the power cycle of a real sleep ends one. A sleep refused after the request, + /// which only the request itself can do where the sleep driver sends it, leaves + /// the enclave asleep, and requests fail until the next real sleep or a reboot. + /// If it never gated it was never stopped, and a doorbell would reach an + /// endpoint: ask it something instead. + pub(crate) fn sleep_exit(&self) { + // Left to the sleep's last refusal: told only if it went out. + match pm::commit_finish(&self.dev) { + Some(true) => self.set_sleep_gate(pm::SleepGate::Told), + Some(false) => self.sleep_release_rx(), + None => {} + } + let gate = self.sleep_gate(); + if gate == pm::SleepGate::Draining { + self.set_sleep_gate(pm::SleepGate::Open); + return; + } + if !gate.needs_wake() { + return; + } + // SAFETY: only reads a flag. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if !real && gate == pm::SleepGate::Lost { + return; + } + let answered = if real { + self.sleep_kick() + } else if pm::wait_gated(&self.dev) { + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + dev_warn!( + self.dev, + "the enclave stays asleep until the next real sleep or a reboot\n" + ); + return; + } else { + dev_warn!(self.dev, "the enclave never gated its power domain\n"); + self.sleep_probe() + }; + match answered { + Ok(()) => { + self.set_sleep_gate(pm::SleepGate::Open); + // One line per sleep it was told of: nothing else says it came back. + dev_info!(self.dev, "the enclave is back from its sleep request\n"); + } + Err(e) => { + dev_err!(self.dev, "the enclave did not come back ({:?}); it stays unused\n", e); + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + } + } + } + + /// Starts a gated enclave and waits for the messages that say it's up. + fn sleep_kick(&self) -> Result<()> { + pm::clear_dev_disable(&self.dev)?; + pm::dart_restore(&self.dev); + self.sleep_release_rx(); + self.control.lock().wake_reset(); + match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, proto::WAKE_KICK)?; + } + None => return Err(ENODEV), + } + let mut remaining = time::msecs_to_jiffies(proto::CONTROL_TIMEOUT_MS); + let mut guard = self.control.lock(); + loop { + if guard.wake_complete() { + return Ok(()); + } + if remaining == 0 { + return Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + /// Whether an enclave that was told and never gated still answers. + fn sleep_probe(&self) -> Result<()> { + pm::dart_restore(&self.dev); + self.sleep_release_rx(); + let op = proto::op_nop(0); + let (idx, tag) = self.control.lock().alloc()?; + let msg = proto::encode_control(&op, tag); + let sent = match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, msg) + } + None => Err(ENODEV), + }; + if let Err(e) = sent { + self.control.lock().abandon(idx); + return Err(e); + } + match self.control_wait(&op, idx, tag)? { + Some(_) => Ok(()), + None => Err(ETIMEDOUT), + } } /// Puts `msg` in the mailbox once the enclave can see what the CPU wrote @@ -1312,6 +1591,16 @@ impl SepData { return Err(e); } + self.control_wait(op, idx, tag) + } + + /// Waits for the answer to the request sent under `idx` and `tag`. + fn control_wait( + &self, + op: &proto::ControlOp, + idx: usize, + tag: u8, + ) -> Result> { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { @@ -1449,7 +1738,7 @@ impl SepData { } let mbox = self.mbox.lock(); let mbox = mbox.as_ref().ok_or(ENODEV)?; - if !self.control.lock().entropy_may_send() { + if !self.sleep_gate().may_request() || !self.control.lock().entropy_may_send() { return Err(EAGAIN); } Self::ring(mbox, msg) @@ -1862,7 +2151,7 @@ impl SepData { fn send_xarm_reply(&self, reply: &crate::xarm::XarmReply) { let msg = crate::xarm::encode_xarm_reply(reply); - let _ = self.send(msg); + let _ = self.send_reply(msg); } fn release_deferred_query(&self) { @@ -2567,6 +2856,8 @@ impl SepData { fn on_control(&self, msg: Message, f: proto::Fields) { if f.ty != proto::CONTROL_REPLY_TYPE { + self.control.lock().wake_note(f.ty); + self.control_wq.notify_all(); return; } @@ -2911,6 +3202,11 @@ impl platform::Driver for SepDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pm_shim.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(pm::sep_pm_ops) }) + } + fn probe<'bound>( pdev: &'bound platform::Device>, _info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index ca0d3d8a797ee..e9188184095e9 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -19,15 +19,24 @@ #include struct apple_sep_fv_new_file_key; +struct dev_pm_ops; void sep_cancel_work_sync(void *work); void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void (*event)(bool entering)); +int sep_pm_register(void *dev, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); +int sep_pm_wait_gated(void *dev, unsigned int timeout_ms); +int sep_pm_clear_dev_disable(void *dev); +int sep_pm_dart_prepare(void *dev); +void sep_pm_dart_restore(void *dev); +int sep_pm_commit_prepare(void *dev, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask); +int sep_pm_commit_finish(void *dev); +extern const struct dev_pm_ops sep_pm_ops; /* -- hwrng_shim.c ------------------------------------------------------- */ diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index 03b17af8851c9..e0c4be586801d 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -91,6 +91,11 @@ impl Mailbox { unsafe { device::Device::from_raw((*self.mbox).dev) } } + /// The C mailbox, for code that drives it with interrupts off. + pub fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + /// Sends the specified message pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { // SAFETY: Calling the c function, `mbox` is a valid pointer From 65dbbd8dbaf8d4d3bb97149405597a5fd324822e Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 11:19:39 -0500 Subject: [PATCH 64/73] soc: apple: sep: power the fingerprint sensor off for system sleep The sensor driver had no system sleep callbacks. A sensor left powered by the attach-time bring-up stayed powered through every s2idle and deep sleep, and a capture work item that outlived the suspend drain could still reach the bus and the power rails while devices went down. macOS 27.2 (25G83) does this around a sleep, read statically from AppleMesaSEPDriver, AppleBiometricSensor and IOBiometricFamily: AppleMesaSEPDriver has two power states, and its on state needs its parent's power, so a system sleep calls setPowerState(0). That starts a zero-delay timer and returns a 20 ms acknowledgment bound; the handler cancels the driver's timers and pending command, then sensorDisable(true, userOverride = false) has AppleMesa wait for its state machine, mask the data-ready event source, send 80 31 00 07 00 00 00 (opcode 0x31, framed like the idle command 0x30) transmit-only, and cut the power only when the user turned the sensor off. The generic transfer endpoint to the enclave is then disabled on the AP side, with no message to the enclave, and the power change is acknowledged. On wake it unmasks the event source, reads the 16-byte status and runs a power cycle and patch reload if the status is bad or unreadable, then restarts whatever the current biometric command needs. Here the sleep command goes out and the power is then cut, whatever the reason for the sleep. Every Linux bring-up starts with a power cycle and registers the sensor with the enclave again, so a sensor powered off at suspend is powered on demand by the next enrol or verify, which is what the wake-time recovery on macOS amounts to. The resume only lets requests through again. Between the two, bus and power requests fail with -ESHUTDOWN. A suspend that finds the sensor already off, which it is after any enrol or verify, touches no hardware, and the power-off skips the 10 ms settling delay, which the next power cycle has anyway. A power-off that fails is logged and does not fail the sleep. Nothing else needs to change for the sleep. The PM notifier already ends any capture and drains the work before devices suspend. The enclave keeps its state across the sleep, and the next verify clears and registers the sensor again as every verify does. spi-apple programs the mode, chip select timing, clock divider, word size, counts and interrupts for each message or transfer, because runtime PM gates its power domain between messages, so a controller that lost its state in a deep sleep is already handled. The AP GPIO restore after a deep sleep puts back the data-ready pin's trigger mode. The callbacks belong to the sensor's SPI driver, so a machine without the sensor runs none of this. KUnit covers the sleep sequence and the power tracking with faked hardware. They pass, built with the shim's own code on the host with -Wall -Wextra -Werror, and six mutations of the sequence each fail at least one case. Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 5 +- drivers/soc/apple/sensor_shim.c | 152 ++++++++++++++++++------ drivers/soc/apple/sensor_shim_test.c | 169 +++++++++++++++++++++++++++ 3 files changed, 286 insertions(+), 40 deletions(-) create mode 100644 drivers/soc/apple/sensor_shim_test.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ae3dc45793a7d..05dfd0855bbc5 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -219,8 +219,9 @@ config APPLE_SEP_KUNIT_TEST mailbox message encoding and decoding, the shared-memory registration limits, the rules that keep a keybag's lookup identity apart from the identity the enclave reports, the format and validation of the - Linux host-state store, and the xART guard's decisions and its list - of Linux's records. The tests touch no hardware. + Linux host-state store, the xART guard's decisions and its list + of Linux's records, and how the fingerprint sensor is put to sleep. + The tests touch no hardware. If unsure, say N. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index bed228b4d1ca3..5536de938e5ba 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,21 @@ MODULE_PARM_DESC(sensor_spi_mode, struct sep_sensor_lifetime { /* Serializes sensor operations with this device's resource teardown. */ struct mutex lock; + /* The power last written; a failed write counts as on. */ + bool powered; + /* Set across a system sleep: bus and power requests are refused. */ + bool asleep; }; +/* The power line or rails and the bus, with no settling delays. */ +struct sep_sensor_hw { + int (*set_power)(int on); + int (*send)(const void *tx, size_t len); +}; + +/* Puts the sensor in its low-power state; macOS sends it before a sleep. */ +static const u8 sep_sensor_cmd_sleep[] = { 0x80, 0x31, 0x00, 0x07, 0x00, 0x00, 0x00 }; + static DEFINE_MUTEX(sep_sensor_registry_lock); static struct sep_sensor_lifetime *sep_active_sensor; @@ -58,6 +72,7 @@ static struct sep_sensor_lifetime *sep_lock_sensor(void) } static void sep_sensor_irq_teardown_locked(void); +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len); static struct spi_device *sep_spi; static struct gpio_desc *sep_power; @@ -328,6 +343,52 @@ static void sep_release_power(void) sep_power_source = SEP_POWER_NONE; } +static int sep_sensor_set_power_now(int on) +{ + if (sep_power_source == SEP_POWER_SMC_RAILS) + return sep_smc_set_power(on); + if (!sep_power) + return -ENODEV; + gpiod_set_value_cansleep(sep_power, on); + return 0; +} + +static const struct sep_sensor_hw sep_hw = { + .set_power = sep_sensor_set_power_now, + .send = sep_sensor_xfer_tx_locked, +}; + +static int sep_sensor_write_power(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw, int on) +{ + int rc = hw->set_power(on); + + /* A failed write may have taken effect. */ + sensor->powered = rc || on; + return rc; +} + +/* + * As macOS does for a sleep: the sleep command, then the power off. The next + * bring-up's power cycle powers the sensor again, so a resume does not. + * Returns the power-off's error. + */ +static int sep_sensor_sleep_locked(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw) +{ + WRITE_ONCE(sensor->asleep, true); + if (!sensor->powered) + return 0; + /* Best effort: only the power-off matters. */ + hw->send(sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + return sep_sensor_write_power(sensor, hw, 0); +} + +static int sep_sensor_awake_locked(struct sep_sensor_lifetime *sensor) +{ + return READ_ONCE(sensor->asleep) ? -ESHUTDOWN : 0; +} + #define SEP_CS_TIMING_SOFTWARE 0 #define SEP_CS_TIMING_HOOK_BYPASSED 1 #define SEP_CS_TIMING_HARDWARE 2 @@ -462,6 +523,35 @@ static void sep_sensor_remove(struct spi_device *spi) mutex_unlock(&sep_sensor_registry_lock); } +static int sep_sensor_suspend(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + bool was_on; + int rc; + + mutex_lock(&sensor->lock); + was_on = sensor->powered; + rc = sep_sensor_sleep_locked(sensor, &sep_hw); + mutex_unlock(&sensor->lock); + /* Failing the sleep would keep the machine awake; the next bring-up recovers. */ + if (rc) + dev_warn(dev, "sep sensor: power-off for sleep failed (%d)\n", rc); + else if (was_on) + dev_dbg(dev, "sep sensor: off for sleep\n"); + return 0; +} + +/* No lock: a capture still waiting on the data-ready line may hold it. */ +static int sep_sensor_resume(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + + WRITE_ONCE(sensor->asleep, false); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(sep_sensor_pm_ops, sep_sensor_suspend, sep_sensor_resume); + /* * Its own compatible, deliberately not spidev's: capture must stay in the * kernel, and a spidev node would leak an image to userspace. No @@ -485,6 +575,7 @@ static struct spi_driver sep_sensor_driver = { .driver = { .name = "apple-mesa", .of_match_table = sep_sensor_of_match, + .pm = pm_sleep_ptr(&sep_sensor_pm_ops), }, .id_table = sep_sensor_spi_ids, .probe = sep_sensor_probe, @@ -528,26 +619,16 @@ static int sep_sensor_cs_timing_mode_locked(void) } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -static int sep_sensor_power_cycle_locked(void) +static int sep_sensor_power_cycle_locked(struct sep_sensor_lifetime *sensor) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(0); - - if (rc) - return rc; - msleep(SEP_SENSOR_OFF_DELAY_MS); - rc = sep_smc_set_power(1); - if (rc) - return rc; - msleep(SEP_SENSOR_ON_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; + int rc = sep_sensor_write_power(sensor, &sep_hw, 0); - gpiod_set_value_cansleep(sep_power, 0); + if (rc) + return rc; msleep(SEP_SENSOR_OFF_DELAY_MS); - gpiod_set_value_cansleep(sep_power, 1); + rc = sep_sensor_write_power(sensor, &sep_hw, 1); + if (rc) + return rc; msleep(SEP_SENSOR_ON_DELAY_MS); return 0; } @@ -588,24 +669,13 @@ static const char *sep_sensor_firmware_name_locked(unsigned int index) } /* Powers the sensor on/off, holding the hardware settling delay. */ -static int sep_sensor_power_locked(int on) +static int sep_sensor_power_locked(struct sep_sensor_lifetime *sensor, int on) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(on ? 1 : 0); + int rc = sep_sensor_write_power(sensor, &sep_hw, on ? 1 : 0); - if (rc) - return rc; - msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; - - gpiod_set_value_cansleep(sep_power, on ? 1 : 0); - if (on) - msleep(SEP_SENSOR_ON_DELAY_MS); - else - msleep(SEP_SENSOR_OFF_DELAY_MS); + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); return 0; } @@ -846,7 +916,7 @@ int sep_sensor_power_cycle(void) if (!sensor) return -ENODEV; - result = sep_sensor_power_cycle_locked(); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_cycle_locked(sensor); mutex_unlock(&sensor->lock); return result; } @@ -894,7 +964,7 @@ int sep_sensor_power(int on) if (!sensor) return -ENODEV; - result = sep_sensor_power_locked(on); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_locked(sensor, on); mutex_unlock(&sensor->lock); return result; } @@ -962,7 +1032,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_locked(tx, rx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_locked(tx, rx, len); mutex_unlock(&sensor->lock); return result; } @@ -974,7 +1044,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_tx_locked(tx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_tx_locked(tx, len); mutex_unlock(&sensor->lock); return result; } @@ -986,7 +1056,13 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + result = sep_sensor_awake_locked(sensor) ?: + sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); mutex_unlock(&sensor->lock); return result; } + +/* Like the module's Rust tests, which need KUnit built in. */ +#if IS_ENABLED(CONFIG_APPLE_SEP_KUNIT_TEST) && IS_BUILTIN(CONFIG_KUNIT) +#include "sensor_shim_test.c" +#endif diff --git a/drivers/soc/apple/sensor_shim_test.c b/drivers/soc/apple/sensor_shim_test.c new file mode 100644 index 0000000000000..a0374e65bbf5f --- /dev/null +++ b/drivers/soc/apple/sensor_shim_test.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* KUnit tests for the fingerprint sensor's system sleep. No hardware is touched. */ + +#include +#include + +struct sep_fake_hw { + /* 's' for a send, '0' and '1' for power writes, in order. */ + char calls[8]; + unsigned int ncalls; + u8 sent[16]; + size_t sent_len; + int send_rc; + int power_rc; +}; + +static struct sep_fake_hw *sep_fake(void) +{ + return kunit_get_current_test()->priv; +} + +static void sep_fake_note(struct sep_fake_hw *f, char call) +{ + if (f->ncalls < ARRAY_SIZE(f->calls)) + f->calls[f->ncalls++] = call; +} + +static int sep_fake_send(const void *tx, size_t len) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, 's'); + f->sent_len = min(len, sizeof(f->sent)); + memcpy(f->sent, tx, f->sent_len); + return f->send_rc; +} + +static int sep_fake_set_power(int on) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, on ? '1' : '0'); + return f->power_rc; +} + +static const struct sep_sensor_hw sep_fake_hw = { + .set_power = sep_fake_set_power, + .send = sep_fake_send, +}; + +static int sep_sensor_test_init(struct kunit *test) +{ + test->priv = kunit_kzalloc(test, sizeof(struct sep_fake_hw), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, test->priv); + return 0; +} + +static void sep_sensor_sleep_sends_the_command_then_cuts_power(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_EQ(test, f->sent_len, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_MEMEQ(test, f->sent, sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_leaves_an_unpowered_sensor_alone(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = false }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_EXPECT_EQ(test, f->ncalls, 0U); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_cuts_power_when_the_command_fails(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->send_rc = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_FALSE(test, sensor.powered); +} + +static void sep_sensor_sleep_counts_a_failed_power_off_as_on(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_requests_wait_for_the_resume(struct kunit *test) +{ + struct sep_sensor_lifetime sensor = { .powered = true }; + struct device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + sep_sensor_sleep_locked(&sensor, &sep_fake_hw); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), -ESHUTDOWN); + + dev_set_drvdata(dev, &sensor); + KUNIT_EXPECT_EQ(test, sep_sensor_resume(dev), 0); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + /* The resume powers nothing: the next bring-up does. */ + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, ((struct sep_fake_hw *)test->priv)->ncalls, 2U); +} + +static void sep_sensor_power_writes_are_tracked(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = {}; + + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), 0); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), 0); + KUNIT_EXPECT_FALSE(test, sensor.powered); + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + sensor.powered = false; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); +} + +/* Framed like the driver's other commands: 0x80, opcode, then the length at 3. */ +static void sep_sensor_sleep_command_is_framed(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, sizeof(sep_sensor_cmd_sleep), (size_t)7); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[0], (u8)0x80); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[1], (u8)0x31); + KUNIT_EXPECT_EQ(test, (size_t)get_unaligned_le16(&sep_sensor_cmd_sleep[3]), + sizeof(sep_sensor_cmd_sleep)); +} + +static struct kunit_case sep_sensor_sleep_cases[] = { + KUNIT_CASE(sep_sensor_sleep_sends_the_command_then_cuts_power), + KUNIT_CASE(sep_sensor_sleep_leaves_an_unpowered_sensor_alone), + KUNIT_CASE(sep_sensor_sleep_cuts_power_when_the_command_fails), + KUNIT_CASE(sep_sensor_sleep_counts_a_failed_power_off_as_on), + KUNIT_CASE(sep_sensor_requests_wait_for_the_resume), + KUNIT_CASE(sep_sensor_power_writes_are_tracked), + KUNIT_CASE(sep_sensor_sleep_command_is_framed), + {} +}; + +static struct kunit_suite sep_sensor_sleep_suite = { + .name = "apple_sep_sensor_sleep", + .init = sep_sensor_test_init, + .test_cases = sep_sensor_sleep_cases, +}; + +kunit_test_suite(sep_sensor_sleep_suite); From 4f7e92c1fbe191e1b0230627693f29ff62cfd459 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:29 -0500 Subject: [PATCH 65/73] soc: apple: pmp: take the PMP and its report through a system sleep Two traced macOS sleeps on T8140 show the same thirteen mailbox messages. About 61 ms before the commit the AP state then the IOP state go to 0x201, each acknowledged. About 37 ms after the wake the firmware sends HELLO unasked, then its endpoint map, reports IOP state 0x20 unasked, and takes AP state 0x20. The idle is a point of no return: an idled PMP doesn't come back with its power kept. So where the sleep controller can take it, the idle and the stop run in the sleep's last step, after every refusal, with interrupts off: the PMP registers there once, readies itself in suspend_late, and a refused sleep finds it still running. Elsewhere it idles in suspend_late, after the consumers' suspend and any NVMe stop. It comes back in resume_early, before the devices it serves resume, through the shared RTKit return. Nothing the firmware asked for at probe is repeated. As macOS does, after the idle pair the CPU is stopped and the firmware's SRAM range (0x3c000 to 0x98000 on T8140, match data) is copied out; after the wake it's written back before the run bit. The copy's memory is taken before anything is asked of the firmware, so running out refuses the sleep cleanly. A sleep refused after the stop leaves the firmware stopped, since a restart with the power kept can't work, and the next suspend finds it stopped and goes on. The report: entries' power requests reach the PMP through it, and each waits for an acknowledgment that a stopped PMP can't give. So requests are withdrawn just before the PMP stops, as macOS's request word reads 0 by then, and held in the books until the firmware is back. After the wake it's handed everything again, with the request word from the sleep's start for the bits no entry owns, since the report's SRAM loses power. A driver that switches its device's power itself around a sleep can set its one entry's request (macOS does that for the GPU, clearing its bit before lowering GFX and setting it after raising it). A power command during the sleep gets -EAGAIN and doesn't latch a failure. The GPU's stop moves to the commit as well, and withdraws the GPU's request there, while the PMP still runs. So an entry's driver can mark it as withdrawn at the commit: apple_pmp_report_sleep() leaves it set when the PMP stops at the commit too, and apple_pmp_report_entry_withdraw_polled() withdraws it with interrupts off, past the hold on requests, polling for the PMP's acknowledgment. On T8140 the PMP offers no sysfs unbind, as on the retained-vote J713: an unbind leaves its firmware running, and a commit with the PMP running reset the SoC, so its stale sleep registration would refuse every sleep until reboot. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmp-report-test.c | 11 + .../pmdomain/apple/pmp-report-validation.h | 9 + drivers/pmdomain/apple/pmp-report.c | 248 +++++++++++++++++- drivers/soc/apple/pmp-export.c | 150 +++++++++++ drivers/soc/apple/pmp.rs | 194 +++++++++++++- drivers/soc/apple/pmp_bootargs.rs | 14 + include/linux/soc/apple/pmp-report.h | 30 +++ include/linux/soc/apple/pmp.h | 20 ++ tools/asahi/display26-tests.py | 40 ++- tools/asahi/t6030-pmp-graph-tests.c | 8 + tools/asahi/t6030-pmp-supplier-tests.c | 2 + .../drivers/gpu/asahi-g15/pmp-vote.c | 2 + 12 files changed, 719 insertions(+), 9 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report-test.c b/drivers/pmdomain/apple/pmp-report-test.c index d99ece1ac4b6f..1d816cf715da7 100644 --- a/drivers/pmdomain/apple/pmp-report-test.c +++ b/drivers/pmdomain/apple/pmp-report-test.c @@ -12,6 +12,16 @@ static void pmp_range(u8 *table, u32 id, u32 base, u32 count) put_unaligned_le32(count, table + 8); } +static void pmp_request_word_replay(struct kunit *test) +{ + /* A request an entry withdrew stays off; one nobody here owns stays on. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x31, 0x03, 0x02), 0x32ULL); + /* The found word can be empty: only the requests come back. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0, 0x03, 0x01), 0x01ULL); + /* Nothing requested, nothing known. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x10, 0, 0), 0x10ULL); +} + static void pmp_ranges_t6030(struct kunit *test) { u8 table[2 * PMP_PTD_RANGE_SIZE]; @@ -256,6 +266,7 @@ static void pmp_bootargs_reject_integer_width(struct kunit *test) static struct kunit_case apple_pmp_report_cases[] = { KUNIT_CASE(pmp_ranges_t6030), + KUNIT_CASE(pmp_request_word_replay), KUNIT_CASE(pmp_ranges_t8122), KUNIT_CASE(pmp_ranges_t6031), KUNIT_CASE(pmp_ranges_reject_bad_records), diff --git a/drivers/pmdomain/apple/pmp-report-validation.h b/drivers/pmdomain/apple/pmp-report-validation.h index 94a682a9f18f2..be0e6a500a351 100644 --- a/drivers/pmdomain/apple/pmp-report-validation.h +++ b/drivers/pmdomain/apple/pmp-report-validation.h @@ -13,6 +13,15 @@ #define PMP_PTD_RANGE_REQUEST 10 #define PMP_PTD_RANGE_ACK 11 +/* + * The request word for a PMP that is starting: requests made through entries, + * and every other bit as it was found. + */ +static inline u64 apple_pmp_request_word(u64 found, u64 known, u64 shadow) +{ + return (found & ~known) | shadow; +} + /* The PTD request read, request update and acknowledgment apertures. */ static inline bool apple_pmp_ranges_valid(const u8 *table, size_t len, u32 request_read, u32 request_write, diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index b3dceb84afa5a..312c63dd631ba 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -101,8 +101,17 @@ struct apple_pmp_report { /* Requests made through entries: bits requested on, all bits. */ u64 shadow; u64 known; - /* Requests are held in memory until the PMP runs (@defer_requests). */ + /* Bits whose drivers withdraw them at the sleep's commit, with the PMP still running. */ + u64 commit_owned; + /* + * Requests are held in memory until the PMP runs (@defer_requests), and + * while it's stopped for a system sleep. + */ bool defer; + /* The request word when a system sleep stopped the PMP. */ + u64 sleep_request; + /* The last SOC-DEV-DVFS word written to each entry, 0 if none. */ + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; }; @@ -409,7 +418,7 @@ static void apple_pmp_report_ready_work(struct work_struct *work) spin_lock_irqsave(&rep->lock, flags); val = readq(rep->base + offs->tgt_read); - val = (val & ~rep->known) | rep->shadow; + val = apple_pmp_request_word(val, rep->known, rep->shadow); writeq(val, rep->base + offs->tgt_write); want = rep->shadow; rep->defer = false; @@ -423,6 +432,97 @@ static void apple_pmp_report_ready_work(struct work_struct *work) readq(rep->base + offs->status), want); } +/* A report names its PMP where a SoC has more than one kind of link to it. */ +static bool apple_pmp_report_of(const struct apple_pmp_report *rep, const struct device *pmp) +{ + struct device_node *np = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + bool match = !np || np == pmp->of_node; + + of_node_put(np); + return match; +} + +/** + * apple_pmp_report_sleep() - withdraw and hold requests before a system sleep stops the PMP + * @pmp: the PMP device, still running + * @at_commit: the PMP stops at the sleep's commit, so requests whose drivers withdraw them + * there are left set + * + * The devices behind these requests have stopped by now, so the PMP is told before + * it goes. Requests made from here on only update the books, so nothing waits on an + * acknowledgment from a stopped PMP. Bits no entry owns are left alone: nothing here + * knows whether their devices have stopped. + */ +void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 val, withdrawn; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + rep->sleep_request = readq(rep->base + offs->tgt_read); + withdrawn = rep->sleep_request & rep->known & ~(at_commit ? rep->commit_owned : 0); + if (withdrawn) + writeq(rep->sleep_request & ~withdrawn, rep->base + offs->tgt_write); + rep->defer = true; + spin_unlock_irqrestore(&rep->lock, flags); + + if (withdrawn && readq_poll_timeout(rep->base + offs->actual, val, + !(val & withdrawn), 100, 50000)) + dev_warn(rep->dev, "PMP still acknowledges withdrawn requests %#llx (ack %#llx)\n", + withdrawn, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_sleep); + +/** + * apple_pmp_report_wake() - hand the PMP every request after a system sleep + * @pmp: the PMP device, running again + * @lost_power: the sleep reached the hardware, so the report may have lost its + * contents: bits no entry owns come from the word kept at the sleep + * + * The power requests go first and the performance requests after them, the + * order an entry uses when it powers on. + */ +void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ + struct apple_pmp_report *rep; + unsigned long flags; + unsigned int i; + u64 val, want; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + val = lost_power ? rep->sleep_request : readq(rep->base + offs->tgt_read); + writeq(apple_pmp_request_word(val, rep->known, rep->shadow), + rep->base + offs->tgt_write); + for (i = 0; rep->dvfs_shadow && i < offs->dvfs_count; i++) + if (rep->dvfs_shadow[i]) + writeq(rep->dvfs_shadow[i], + rep->base + offs->update + 8 * (offs->dvfs_first + i)); + want = rep->shadow; + rep->defer = false; + spin_unlock_irqrestore(&rep->lock, flags); + + if (want && readq_poll_timeout(rep->base + offs->actual, val, + (val & want) == want, 1000, 1000000)) + dev_warn(rep->dev, "PMP did not acknowledge requests %#llx (ack %#llx)\n", + want, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_wake); + /* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) { @@ -1342,6 +1442,12 @@ static int apple_pmp_report_probe(struct platform_device *pdev) } rep->defer = rep->offsets->defer_requests; + if (rep->offsets->dvfs_count) { + rep->dvfs_shadow = devm_kcalloc(dev, rep->offsets->dvfs_count, + sizeof(*rep->dvfs_shadow), GFP_KERNEL); + if (!rep->dvfs_shadow) + return -ENOMEM; + } INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); @@ -1651,7 +1757,13 @@ EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); */ static void apple_pmp_report_dvfs_write(struct apple_pmp_report *rep, u32 entry, u64 val) { - writeq(val, rep->base + rep->offsets->update + 8 * entry); + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + rep->dvfs_shadow[entry - rep->offsets->dvfs_first] = val; + if (!rep->defer) + writeq(val, rep->base + rep->offsets->update + 8 * entry); + spin_unlock_irqrestore(&rep->lock, flags); } static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) @@ -1679,6 +1791,136 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo return ret; } +/** + * apple_pmp_report_entry_request() - set or withdraw one entry's request + * @entry: the report entry's node + * @enable: make the request, or withdraw it + * + * For a driver that switches its device's power itself around a system sleep. The + * PMP may use a device whose request is set, so that driver withdraws the request + * before its power goes and makes it again after its power is back. A withdrawn + * request isn't replayed after the PMP restarts. + * + * Return: 0, also with no report bound or the PMP not running, or the error from + * waiting for its acknowledgment. + */ +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable) +{ + struct device_node *report; + struct apple_pmp_report *rep; + u32 id; + int ret = 0; + + if (of_property_read_u32(entry, "reg", &id)) + return -EINVAL; + + report = of_get_parent(entry); + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != report) + continue; + ret = apple_pmp_report_set_state(rep, id, enable, false); + break; + } + rcu_read_unlock(); + of_node_put(report); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_request); + +static struct apple_pmp_report *apple_pmp_report_of_entry(const struct device_node *entry, + u32 *id) +{ + struct device_node *report; + struct apple_pmp_report *rep, *found = NULL; + + if (of_property_read_u32(entry, "reg", id) || *id >= 64) + return NULL; + report = of_get_parent(entry); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == report) { + found = rep; + break; + } + } + of_node_put(report); + + return found; +} + +/** + * apple_pmp_report_entry_commit_owned() - mark an entry withdrawn by its driver at the commit + * @entry: the report entry's node + * @owned: its driver withdraws it at the sleep's commit, or no longer does + * + * For a device whose stop moves to the sleep's commit: a PMP that stops there too keeps the + * request through apple_pmp_report_sleep(), until apple_pmp_report_entry_withdraw_polled(). + * + * Return: 0, or -ENODEV with no report bound for the entry. + */ +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned) +{ + struct apple_pmp_report *rep; + unsigned long flags; + int ret = -ENODEV; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + spin_lock_irqsave(&rep->lock, flags); + if (owned) + rep->commit_owned |= BIT_ULL(id); + else + rep->commit_owned &= ~BIT_ULL(id); + spin_unlock_irqrestore(&rep->lock, flags); + ret = 0; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_commit_owned); + +/** + * apple_pmp_report_entry_withdraw_polled() - withdraw an entry's request at the sleep's commit + * @entry: the report entry's node + * + * Interrupts off, one CPU: the request goes even with requests held for the sleep, and the + * PMP's acknowledgment is polled for while it runs. One already withdrawn returns at once. + * + * Return: 0, also with no report bound, or -ETIMEDOUT with no acknowledgment. + */ +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 bit, val; + int ret = 0; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + bit = BIT_ULL(id); + spin_lock_irqsave(&rep->lock, flags); + rep->known |= bit; + rep->shadow &= ~bit; + val = readq(rep->base + rep->offsets->tgt_read); + if (val & bit) + writeq(val & ~bit, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + if (apple_pmp_report_running(rep)) + ret = readq_poll_timeout_atomic(rep->base + rep->offsets->actual, val, + !(val & bit), 10, 50000); + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_withdraw_polled); + /* * System sleep. An external display processor's request keeps the PMP's * external display state up while nothing is shown, about 0.25 W in diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 41091755a77c6..93a8aa9123757 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -1,15 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include #include #include +#include +#include #include +#include #include +#include +#include +#include /* Join removal and serialize the firmware's untagged power reply stream. */ static DEFINE_MUTEX(pmp_lock); static const void *pmp_data; static struct device *pmp_device; +/* The firmware is stopped for a system sleep. */ +static bool pmp_asleep; int apple_pmp_register(struct device *dev, const void *data) { @@ -66,6 +75,8 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; /* The matched native report must be ready; never use an elapsed-time fallback. */ ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) @@ -104,9 +115,148 @@ int apple_pmp_t6030_gpu_ready(void) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); if (ret) return ret; return apple_pmp_report_set_t6030_gfx(pmp_device, true); } EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); + +/* + * The PMP's idle is a point of no return: an idled PMP doesn't come back with its power kept, + * and only a real sleep restarts it. Where the sleep driver can take them, its idle + * and stop move to the sleep's last step, after every refusal: a refused sleep then finds it + * still running, and the GPU's request, withdrawn at the commit, reaches a running PMP. + */ +static struct apple_pmp_commit_stop pmp_commit; +static bool pmp_commit_registered, pmp_commit_used; + +/* Interrupts off, one CPU: what apple_pmp_sleep() does, through pointers taken before. */ +static int apple_pmp_commit_stop_fn(void *unused) +{ + struct apple_pmp_commit_stop *c = &pmp_commit; + unsigned int us; + size_t i; + int ret; + + ret = apple_rtkit_sleep_idle_polled(c->rtk, c->control, &us); + apple_rtkit_mailbox_recv_off(c->rtk); + for (i = 0; c->sram && i < c->count; i++) + c->words[i] = readl_relaxed(c->sram + 4 * i); + /* Told to idle, it can't carry on whatever it answered: the sleep goes on. */ + return ret ? 1 : 0; +} + +/* + * Retained J713 power votes have no teardown. On T8140 an unbind leaves the firmware running, + * and a commit with it running reset the SoC: its sleep registration is what refuses + * then, every sleep until reboot, since a rebind can't register again. + */ +bool apple_pmp_keep_bound(void) +{ + return apple_pmp_display_retained_board() || of_machine_is_compatible("apple,t8140"); +} + +/* Whether the sleep's last step takes the PMP: registered with the sleep driver, once. */ +static bool apple_pmp_commit_available(struct device *dev) +{ + void __iomem *asc; + int idx; + + if (pmp_commit_registered) + return true; + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return false; + if (apple_sleep_coproc_register(dev, asc, false, apple_pmp_commit_stop_fn, NULL)) { + iounmap(asc); + return false; + } + pmp_commit_registered = true; + + return true; +} + +static int apple_pmp_commit_prepare(struct device *dev) +{ + int ret; + + ret = apple_pmp_sleep_prepare(pmp_data, &pmp_commit); + if (ret) + return ret; + apple_sleep_coproc_set_ready(dev, true); + pmp_commit_used = true; + dev_dbg(dev, "idle and stop left for the sleep's commit\n"); + + return 0; +} + +static int apple_pmp_suspend_late(struct device *dev) +{ + bool at_commit; + int ret; + + if (!pm_suspend_via_firmware()) + return 0; + /* Retained power votes have no teardown. */ + if (apple_pmp_display_retained_board()) + return -EBUSY; + + guard(mutex)(&pmp_lock); + if (pmp_device != dev || pmp_asleep) + return 0; + at_commit = apple_pmp_commit_available(dev); + apple_pmp_report_sleep(dev, at_commit); + ret = at_commit ? apple_pmp_commit_prepare(dev) : apple_pmp_sleep(pmp_data); + if (ret) { + apple_pmp_report_wake(dev, false); + return dev_err_probe(dev, ret, "firmware did not go idle for sleep\n"); + } + pmp_asleep = true; + + return 0; +} + +static int apple_pmp_resume_early(struct device *dev) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_asleep) + return 0; + /* Left for the commit: if the sleep was refused before it, the firmware never idled. */ + if (pmp_commit_used) { + bool stopped = apple_sleep_coproc_stopped(dev); + + pmp_commit_used = false; + apple_sleep_coproc_set_ready(dev, false); + if (!pm_resume_via_firmware() && !stopped) { + pmp_asleep = false; + apple_pmp_report_wake(dev, true); + dev_info(dev, "sleep refused before the commit: firmware never stopped\n"); + return 0; + } + } + /* A stopped firmware doesn't restart with its power kept: the next real sleep does. */ + if (!pm_resume_via_firmware()) { + dev_warn(dev, "sleep refused: firmware stays stopped until a real sleep\n"); + return 0; + } + pmp_asleep = false; + ret = apple_pmp_wake(pmp_data); + if (ret) + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + else + dev_dbg(dev, "firmware back from sleep\n"); + /* Requests go back either way: consumers read them from the report too. */ + apple_pmp_report_wake(dev, true); + + return 0; +} + +const struct dev_pm_ops apple_pmp_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmp_suspend_late, apple_pmp_resume_early) +}; diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 2212509efd82c..503ec1005351e 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -71,6 +71,7 @@ const BOOTARGS_UUID_MAGIC: u32 = 0x6469_7575; // "uuid" const BOOTARGS_UUID_VERSION: u32 = 5; const BOOTARGS_OFFSET: usize = 0x22c; const BOOTARGS_SIZE: usize = 0x230; +const ASC_CPU_CONTROL_OFFSET: usize = 0x44; const PMP_ENDPOINT: u8 = 0x20; const PM_COMMAND_MARKER: u64 = pmp_power::MARKER; const PM_COMMAND_SHIFT: u32 = pmp_power::SHIFT; @@ -120,11 +121,19 @@ unsafe impl AsBytes for IovaTableEntry {} // SAFETY: TODO: unsafe impl FromBytes for IovaTableEntry {} +/// A range of the firmware's SRAM as it was when the firmware stopped. +struct SramCopy { + offset: usize, + words: KVVec, +} + struct PmpState { iova_table: Option>, allocs: KVec, value_buf: Option, ioreg_entries: KVec, + /// Held from a system sleep's start until its end. + sram: Option, } impl PmpState { @@ -134,6 +143,7 @@ impl PmpState { allocs: KVec::with_capacity(10, GFP_KERNEL)?, value_buf: None, ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + sram: None, }) } fn find_alloc(&self, addr: u64) -> Option { @@ -169,10 +179,12 @@ struct PmpData { display: AtomicPtr, /// The first failure seen before or after the bridge was published. display_error: AtomicI32, + /// See [`PmpHwConfig::sram_backup`]. + sram_backup: Option<(usize, usize)>, } impl PmpData { - fn new(dev: &platform::Device>) -> Result> { + fn new(dev: &platform::Device>, cfg: &PmpHwConfig) -> Result> { let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = pmp_req.iomap_sized::()?.into_devres()?; let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; @@ -189,6 +201,7 @@ impl PmpData { asc_mmio, display: AtomicPtr::new(core::ptr::null_mut()), display_error: AtomicI32::new(0), + sram_backup: cfg.sram_backup, } ), GFP_KERNEL, @@ -438,6 +451,112 @@ impl PmpData { ) }; } + /// The handle, to wait on the firmware with the lock free: its messages take that lock. + fn rtkit_handle(&self) -> Result<*mut bindings::apple_rtkit> { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + Ok(rtk.as_raw()) + } + /// The ASC's CPU control register. + fn asc_cpu_control(&self) -> Result<*mut kernel::ffi::c_void> { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let asc_mmio = self.asc_mmio.access(bound_dev)?.as_view(); + Ok(MmioBackend::as_ptr(asc_mmio) + .cast::() + .wrapping_add(ASC_CPU_CONTROL_OFFSET) + .cast()) + } + /// Room for what SRAM needs kept, taken first so that running out refuses a sleep cleanly. + fn sram_alloc(&self, sram_size: usize) -> Result> { + let Some((offset, size)) = self.sram_backup else { + return Ok(None); + }; + if !pmp_bootargs::word_range_valid(offset, size, sram_size) { + return Err(EINVAL); + } + let mut words = KVVec::new(); + words.resize(size / 4, 0u32, GFP_KERNEL)?; + Ok(Some(SramCopy { offset, words })) + } + /// Stops the firmware for a system sleep, then keeps what its restart needs from SRAM. + fn sleep(&self) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + let mut copy = self.sram_alloc(io.size())?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + if let Err(error) = to_result(unsafe { bindings::apple_rtkit_sleep_stop(rtk, control) }) { + // SAFETY: as above. + if to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }).is_err() { + self.fail_display(error); + } + return Err(error); + } + if let Some(copy) = copy.as_mut() { + for (idx, word) in copy.words.iter_mut().enumerate() { + // The range was checked against the mapping when it was allocated. + *word = io.try_read32(copy.offset + idx * 4).unwrap_or(0); + } + } + self.state.lock().sram = copy; + Ok(()) + } + /// For a stop at the commit: the RTKit handle, the CPU control and the SRAM copy's room, + /// whose buffer stays where it is once moved into the state. + fn sleep_prepare(&self, stop: &mut bindings::apple_pmp_commit_stop) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let size = pmp_mmio.as_view().relaxed().size(); + let mut copy = self.sram_alloc(size)?; + stop.rtk = self.rtkit_handle()?; + stop.control = self.asc_cpu_control()?; + stop.sram = core::ptr::null_mut(); + stop.words = core::ptr::null_mut(); + stop.count = 0; + if let Some(copy) = copy.as_mut() { + stop.sram = MmioBackend::as_ptr(pmp_mmio) + .cast::() + .wrapping_add(copy.offset) + .cast(); + stop.words = copy.words.as_mut_ptr(); + stop.count = copy.words.len(); + } + self.state.lock().sram = copy; + Ok(()) + } + /// Writes SRAM back after a sleep that took its power, while the firmware is stopped. + fn sram_restore(&self) -> Result { + let Some(copy) = self.state.lock().sram.take() else { + return Ok(()); + }; + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + for (idx, word) in copy.words.iter().enumerate() { + io.try_write32(*word, copy.offset + idx * 4)?; + } + // The firmware mustn't start ahead of writes still on their way. + let _ = io.try_read32(copy.offset); + dev_dbg!(self.dev, "SRAM restored: {} bytes\n", copy.words.len() * 4); + Ok(()) + } + /// Brings the firmware back after a system sleep. It keeps its endpoints and everything the + /// host gave it, so nothing here repeats what it asked for at probe. + fn wake(&self) -> Result { + self.sram_restore()?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }) + } fn recv_message(&self, msg: u64) -> Result<()> { let pm_command = (msg >> PM_COMMAND_SHIFT) & 0xff; if msg & PM_COMMAND_MARKER != 0 && (pm_command == 0x0f || pm_command == 0x10) { @@ -609,6 +728,58 @@ pub unsafe extern "C" fn apple_pmp_send_power_command( 0 } +/// Stops the firmware for a system sleep. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.sleep() { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Takes what the sleep's last step needs to stop the firmware with interrupts off, and the room +/// for the SRAM it keeps. Nothing is stopped here. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver, and `stop` to writable memory. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep_prepare( + data: *const core::ffi::c_void, + stop: *mut bindings::apple_pmp_commit_stop, +) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + // SAFETY: The caller passes a valid pointer. + let stop = unsafe { &mut *stop }; + match data.sleep_prepare(stop) { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Brings the firmware back after a system sleep. A firmware that doesn't come back disables +/// further power commands. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_wake(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.wake() { + Ok(()) => 0, + Err(error) => { + data.fail_display(error); + error.to_errno() + } + } +} + struct NoBuffer; impl rtkit::Buffer for NoBuffer { fn iova(&self) -> Result { @@ -657,6 +828,9 @@ struct PmpHwConfig { /// When the firmware needs system endpoint 7 (CoreAnalytics) started /// and given its buffer before it acknowledges the IOP power state. coreanalytics: Ep7, + /// The range of the firmware's SRAM, as (offset, size) in bytes, that a system sleep keeps + /// and writes back. `None` where no sleep has been traced. + sram_backup: Option<(usize, usize)>, } /// Whether a PMP firmware needs endpoint 7 to finish its RTKit boot. @@ -674,6 +848,7 @@ const HW_CFG_DEFAULT: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; /// T8140: the DART translates 42 bits and its DMA window starts at 1 TiB. @@ -681,6 +856,7 @@ const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: Some((0x3c000, 0x5c000)), }; /// T6041: the DART window starts at 1 TiB; the boot argument header is @@ -690,6 +866,7 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: true, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 @@ -699,12 +876,14 @@ const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::Always, + sram_backup: None, }; const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::RetainedBoard, + sram_backup: None, }; impl Drop for PmpDriver { @@ -731,9 +910,14 @@ impl platform::Driver for PmpDriver { type Data<'bound> = PmpDriver; fn suppress_bind_attrs() -> bool { - // Retained J713 power votes have no teardown path. - // SAFETY: read-only board identity check; other boards remain unchanged. - unsafe { bindings::apple_pmp_display_retained_board() } + // Retained J713 power votes have no teardown path, and an unbound T8140 PMP still runs. + // SAFETY: read-only machine identity checks; other machines remain unchanged. + unsafe { bindings::apple_pmp_keep_bound() } + } + + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pmp-export.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(bindings::apple_pmp_pm_ops) }) } const OF_ID_TABLE: Option> = Some(&OF_TABLE); @@ -749,7 +933,7 @@ impl platform::Driver for PmpDriver { // yet. unsafe { pdev.dma_set_mask_and_coherent(mask)? }; } - let data = PmpData::new(pdev)?; + let data = PmpData::new(pdev, cfg)?; let pmp_mmio = data.pmp_mmio.access(pdev.as_ref())?.as_view(); let asc_mmio = data.asc_mmio.access(pdev.as_ref())?.as_view(); let node = dev.fwnode().ok_or(EIO)?; diff --git a/drivers/soc/apple/pmp_bootargs.rs b/drivers/soc/apple/pmp_bootargs.rs index c085a323fee47..bd2abbc94029b 100644 --- a/drivers/soc/apple/pmp_bootargs.rs +++ b/drivers/soc/apple/pmp_bootargs.rs @@ -7,6 +7,11 @@ pub(super) fn region_valid(offset: usize, size: usize, sram_size: usize) -> bool size != 0 && offset <= sram_size && size <= sram_size - offset } +/// A range of SRAM that can be copied as 32-bit words. +pub(super) fn word_range_valid(offset: usize, size: usize, sram_size: usize) -> bool { + region_valid(offset, size, sram_size) && offset % 4 == 0 && size % 4 == 0 +} + fn header(bytes: &[u8]) -> Option<(u32, usize)> { let key = u32::from_le_bytes(bytes.get(..4)?.try_into().ok()?); let size = u32::from_le_bytes(bytes.get(4..8)?.try_into().ok()?) as usize; @@ -69,6 +74,15 @@ mod tests { assert!(!region_valid(1, usize::MAX, usize::MAX)); } + #[test] + fn a_word_range_is_aligned_and_inside_the_window() { + assert!(word_range_valid(0x3c000, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c002, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x5c002, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x64004, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0, 0xa0000)); + } + #[test] fn larger_sram_uses_its_actual_extent() { assert!(region_valid(0x90000, 0x1000, 0x100000)); diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index dd3373517bf90..9f9347e3f271e 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -24,7 +24,37 @@ int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); +void apple_pmp_report_sleep(struct device *pmp, bool at_commit); +void apple_pmp_report_wake(struct device *pmp, bool lost_power); +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable); +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned); +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry); #else +static inline int apple_pmp_report_entry_request(const struct device_node *entry, + bool enable) +{ + return 0; +} + +static inline int apple_pmp_report_entry_commit_owned(const struct device_node *entry, + bool owned) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + return 0; +} + +static inline void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ +} + +static inline void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ +} + static inline int apple_pmp_report_ane_available(const struct device_node *np) { return -EOPNOTSUPP; diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 8b329bf5d4dd5..8663ca195febe 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -5,6 +5,7 @@ #include #include struct device; +struct dev_pm_ops; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); @@ -25,8 +26,27 @@ static inline int apple_pmp_t6030_gpu_ready(void) } #endif /* Registration is private to the PMP Rust driver and its C bridge. */ +bool apple_pmp_keep_bound(void); int apple_pmp_register(struct device *dev, const void *data); void apple_pmp_unregister(const void *data); int apple_pmp_send_power_command(const void *data, u8 command, u16 device_id, u32 enabled); +extern const struct dev_pm_ops apple_pmp_pm_ops; +int apple_pmp_sleep(const void *data); +int apple_pmp_wake(const void *data); + +/* + * What the sleep's last step needs to idle and stop the PMP with interrupts off: its RTKit and + * CPU control, and where the SRAM it keeps goes (no range: sram NULL). Filled in suspend_late. + */ +struct apple_rtkit; +struct apple_pmp_commit_stop { + struct apple_rtkit *rtk; + void __iomem *control; + void __iomem *sram; + u32 *words; + size_t count; +}; + +int apple_pmp_sleep_prepare(const void *data, struct apple_pmp_commit_stop *stop); #endif diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index 2366766042dc2..c43323f0d856f 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -89,13 +89,44 @@ def main(): # Platform functions are mocked; the bridge's actual decisions and cleanup execute. for name in ('device', 'export', 'mutex', 'of'): write(tmp, 'linux/'+name+'.h', '#include "mock.h"\n') - write(tmp, 'linux/errno.h', '#define EPROBE_DEFER 517\n#define ENODEV 19\n#define EINVAL 22\n#define EBUSY 16\n#define EIO 5\n#define ETIMEDOUT 110\n') + write(tmp, 'linux/errno.h', '#define EPROBE_DEFER 517\n#define ENODEV 19\n#define EINVAL 22\n#define EBUSY 16\n#define EIO 5\n#define ETIMEDOUT 110\n#define EAGAIN 11\n') write(tmp, 'linux/kconfig.h', '#define IS_REACHABLE(x) 1\n#define IS_ENABLED(x) 1\n') + # The PMP's system-sleep callbacks live in the bridge; this harness never suspends. + write(tmp, 'linux/pm.h', '#include "mock.h"\n' + 'struct dev_pm_ops { int (*suspend_late)(struct device *); int (*resume_early)(struct device *); };\n' + '#define LATE_SYSTEM_SLEEP_PM_OPS(s, r) .suspend_late = (s), .resume_early = (r),\n') + write(tmp, 'linux/suspend.h', '#include \n' + 'static inline bool pm_suspend_via_firmware(void) { return false; }\n' + 'static inline bool pm_resume_via_firmware(void) { return false; }\n') + # The bridge's sleep-commit stop: MMIO, RTKit and the sleep driver, reached only in a suspend. + write(tmp, 'linux/io.h', '#include "mock.h"\n' + 'static inline u32 readl_relaxed(const volatile void __iomem *a) { assert(!"no system sleep here"); return 0; }\n' + 'static inline void iounmap(volatile void __iomem *a) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/of_address.h', '#include "mock.h"\n' + 'static inline void __iomem *of_iomap(struct device_node *n, int i) { assert(!"no system sleep here"); return NULL; }\n') + write(tmp, 'linux/soc/apple/rtkit.h', '#include "mock.h"\n#include \n' + 'struct apple_rtkit;\n' + 'static inline int apple_rtkit_sleep_idle_polled(struct apple_rtkit *r, void __iomem *c,\n' + '\t\t\t\t\t\tunsigned int *us) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_rtkit_mailbox_recv_off(struct apple_rtkit *r) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/soc/apple/sleep.h', '#include "mock.h"\n#include \n' + 'static inline int apple_sleep_coproc_register(struct device *d, void __iomem *a, bool p,\n' + '\t\t\t\t\t int (*f)(void *), void *x) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_sleep_coproc_set_ready(struct device *d, bool r) { assert(!"no system sleep here"); }\n' + 'static inline bool apple_sleep_coproc_stopped(struct device *d) { assert(!"no system sleep here"); return false; }\n') + write(tmp, 'linux/soc/apple/pmp-display.h', '#include \nbool apple_pmp_display_retained_board(void);\n') write(tmp, 'mock.h', '''#pragma once #include #include #include #include +#define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_err_probe(d, e, ...) (e) +#define dev_info(...) ((void)0) +#include +#define __iomem #include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; @@ -121,6 +152,7 @@ def main(): /* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} +static inline int of_property_match_string(const struct device_node*n,const char*p,const char*s){assert(!"no system sleep here");return -EINVAL;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" @@ -129,6 +161,12 @@ def main(): int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} +bool apple_pmp_display_retained_board(void){return false;} +void apple_pmp_report_sleep(struct device*d,bool at_commit){assert(!"no system sleep here");} +void apple_pmp_report_wake(struct device*d,bool real){assert(!"no system sleep here");} +int apple_pmp_sleep(const void*data){assert(!"no system sleep here");return -EIO;} +int apple_pmp_sleep_prepare(const void*data,struct apple_pmp_commit_stop*stop){assert(!"no system sleep here");return -EIO;} +int apple_pmp_wake(const void*data){assert(!"no system sleep here");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 5f6d0c4d6214b..d974c724827f3 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -159,6 +159,9 @@ struct apple_pmp_report { /* Requests held until the PMP runs; T6030 does not defer them. */ u64 shadow, known; bool defer; + /* Firmware-driven sleep: the request word at its start, the DVFS words kept. */ + u64 sleep_request; + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; struct apple_pmp_ane ane; @@ -627,6 +630,11 @@ static void *devm_kzalloc(struct device *d, size_t n, int g) allocations[allocation_count++] = v; return v; } + +static void *devm_kcalloc(struct device *d, size_t n, size_t size, int g) +{ + return devm_kzalloc(d, n * size, g); +} #define kzalloc_obj(v) calloc(1, sizeof(v)) #define kfree free static char *kstrdup(const char *s, int g) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index 69f832bff9cb3..468edbf9446a1 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -126,6 +126,8 @@ static bool passive_present, passive_lockable, passive_data; static const char *machine; static const void *pmp_data; static struct device *pmp_device; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c index 806148f64d1bf..8154098ee4157 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -19,6 +19,8 @@ struct device { void *of_node; }; static struct device owner; static struct device *pmp_device = &owner; static const void *pmp_data = &owner; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static bool t6030, requested; static int ready, check, power_result[2], report_result[3], powers, reports; static char calls[16]; From ae8686a0bcf4fd49bbefe0b7037fea326ee7a288 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:30 -0500 Subject: [PATCH 66/73] dmaengine: apple-sio: take the firmware through a system sleep The SIO takes the same messages as the PMP around two traced macOS sleeps on T8140, 64 ms before the commit and 39 ms after the wake, and macOS stops it the same way: the idle pair, then the run bit cleared, and set again about 300 us before HELLO after the wake. There's nothing on endpoint 0x20 in either direction, so a restart keeps the firmware's parameters and nothing is sent again. It keeps none of its SRAM. A channel with a transfer can't cross a restart, so a suspend refuses with one running; its client has to end it first. A KUnit test covers that check. An idled firmware doesn't wake with its power kept, and SIO_CPU shares SIO's power switch, so it can't be power-cycled alone. After a refused sleep the sleep controller's own reset of its domain restarts it, as the sleep engine does; if that fails it stays stopped until the next real sleep. Nothing on T8140 uses the SIO's DMA, and its service messages and parameters aren't in any trace, so on T8140 the driver only starts the firmware, stops it for a sleep and restarts it after, with no DMA device registered. Add the T8140 compatible for that. Other SoCs are unchanged, and suspend-to-idle still takes the same runtime PM calls. Signed-off-by: Diljit Singh --- drivers/dma/apple-sio-boot-test.c | 24 +++++++ drivers/dma/apple-sio-probe-test.c | 2 + drivers/dma/apple-sio.c | 110 ++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 1 deletion(-) diff --git a/drivers/dma/apple-sio-boot-test.c b/drivers/dma/apple-sio-boot-test.c index 3d8f93d11d8bb..5994518f619b0 100644 --- a/drivers/dma/apple-sio-boot-test.c +++ b/drivers/dma/apple-sio-boot-test.c @@ -187,12 +187,36 @@ static void sio_boot_dma_arrays_test(struct kunit *test) } } +/* Only a channel with nothing on it may pass through a firmware restart. */ +static void sio_sleep_busy_test(struct kunit *test) +{ + struct sio_chan *chan = kunit_kzalloc(test, sizeof(*chan), GFP_KERNEL); + struct sio_tx *tx = kunit_kzalloc(test, sizeof(*tx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, chan); + KUNIT_ASSERT_NOT_NULL(test, tx); + spin_lock_init(&chan->vc.lock); + + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); + chan->current_tx = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->current_tx = NULL; + chan->idle = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->idle = NULL; + chan->ring = true; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->ring = false; + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); +} + static struct kunit_case sio_boot_test_cases[] = { KUNIT_CASE(sio_boot_order_test), KUNIT_CASE(sio_boot_optional_tables_test), KUNIT_CASE(sio_boot_bad_params_test), KUNIT_CASE(sio_boot_control_ack_failure_test), KUNIT_CASE(sio_boot_dma_arrays_test), + KUNIT_CASE(sio_sleep_busy_test), {} }; diff --git a/drivers/dma/apple-sio-probe-test.c b/drivers/dma/apple-sio-probe-test.c index c486c5072f07c..bb1dd130c6d76 100644 --- a/drivers/dma/apple-sio-probe-test.c +++ b/drivers/dma/apple-sio-probe-test.c @@ -72,6 +72,8 @@ static void sio_match_data_test(struct kunit *test) KUNIT_EXPECT_EQ(test, hw->fifo_threshold, 0x800U); KUNIT_EXPECT_FALSE(test, hw->clear_period_desc); KUNIT_EXPECT_FALSE(test, hw->check_boot_tables); + KUNIT_EXPECT_FALSE(test, hw->firmware_only); + KUNIT_EXPECT_TRUE(test, sio_hw_t8140.firmware_only); } static struct kunit_case sio_probe_test_cases[] = { diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index f2d3afaaf9bc3..ca75beb2bf431 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include "dmaengine.h" @@ -154,6 +156,11 @@ typedef void (*sio_ack_callback)(struct sio_chan *, void *, bool); struct sio_hw_data { u32 fifo_threshold; bool clear_period_desc; + /* + * Only run the firmware, so that it can be stopped for a system sleep. Nothing + * uses its DMA here, and its service parameters haven't been traced. + */ + bool firmware_only; int (*check_boot_tables)(struct device *dev); }; @@ -163,6 +170,8 @@ struct sio_data { struct dma_device dma; struct device *dev; struct apple_rtkit *rtk; + /* The firmware is stopped for a system sleep. */ + bool asleep; u32 protocol; void *shmem; dma_addr_t shmem_iova; @@ -373,8 +382,13 @@ static const struct sio_hw_data sio_hw_t6030 = { .check_boot_tables = sio_t6030_check_boot_tables, }; +static const struct sio_hw_data sio_hw_t8140 = { + .firmware_only = true, +}; + static const struct of_device_id sio_of_match[] = { { .compatible = "apple,t6030-sio", .data = &sio_hw_t6030 }, + { .compatible = "apple,t8140-sio", .data = &sio_hw_t8140 }, { .compatible = "apple,sio", .data = &sio_hw_generic }, { } }; @@ -1709,6 +1723,11 @@ static int sio_probe(struct platform_device *pdev) if (err) return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + if (hw->firmware_only) { + dev_info(&pdev->dev, "firmware running, no DMA service\n"); + goto rpm_put; + } + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); if (err) return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); @@ -1748,6 +1767,9 @@ static void sio_remove(struct platform_device *pdev) { struct sio_data *sio = platform_get_drvdata(pdev); + if (sio->hw->firmware_only) + return; + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&sio->dma); } @@ -1765,7 +1787,93 @@ static __maybe_unused int sio_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); +/* A channel with a transfer can't be carried through a firmware restart. */ +static bool sio_chan_busy(struct sio_chan *siochan) +{ + unsigned long flags; + bool busy; + + spin_lock_irqsave(&siochan->vc.lock, flags); + busy = siochan->current_tx || siochan->idle || siochan->ring; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return busy; +} + +static __maybe_unused int sio_suspend_late(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int i, ret; + + if (!pm_suspend_via_firmware() || sio->asleep) + return 0; + + for (i = 0; i < sio->nchannels; i++) { + if (sio_chan_busy(&sio->channels[i])) { + dev_err(dev, "channel %d is busy, refusing the sleep\n", i); + return -EBUSY; + } + } + + ret = apple_rtkit_sleep_stop(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) { + dev_err(dev, "firmware did not stop for sleep: %d\n", ret); + if (apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) + dev_err(dev, "firmware did not come back\n"); + return ret; + } + sio->asleep = true; + + return 0; +} + +static __maybe_unused int sio_resume_early(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int ret; + + if (!sio->asleep) + return 0; + /* + * A refused sleep: an idled firmware doesn't wake with its power kept, and SIO_CPU shares + * SIO's switch, so it can't be power-cycled alone. The sleep engine's own PMGR reset of it + * restarts it (S); if that fails it stays stopped until the next real sleep. + */ + if (!pm_resume_via_firmware()) { + if (!apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) { + sio->asleep = false; + return 0; + } + ret = apple_sleep_engine_reset(dev); + if (!ret) + ret = apple_rtkit_power_cycled_return(sio->rtk, + sio->base + REG_CPU_CONTROL); + if (ret) { + dev_warn(dev, "sleep refused: firmware not back after a reset (%d), stopped until a real sleep\n", + ret); + return 0; + } + sio->asleep = false; + dev_info(dev, "sleep refused: firmware back after a reset\n"); + return 0; + } + sio->asleep = false; + + /* Endpoints and parameters stay with the firmware: nothing is sent again. */ + ret = apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + else + dev_dbg(dev, "firmware back from sleep\n"); + + return 0; +} + +static const struct dev_pm_ops sio_pm_ops = { + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + LATE_SYSTEM_SLEEP_PM_OPS(sio_suspend_late, sio_resume_early) + RUNTIME_PM_OPS(sio_suspend, sio_resume, NULL) +}; static struct platform_driver apple_sio_driver = { .driver = { From c8515595061e8a74df49054af59226b484c15c07 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:32 -0500 Subject: [PATCH 67/73] drm/asahi: g17: restart the GPU firmware across a firmware-driven suspend With no sleep callbacks, a suspend-to-RAM goes through with the firmware running and loses it: an idle GPU reads actual state 0, so the sleep driver's check passes, and the next submit after the wake times out. macOS doesn't reload the firmware. Measured on T8140 under aura, it idles GFX then GFX1, lowers the GFX power state, and after the wake takes HELLO on GFX1 then GFX with no initialization, no endpoint starts and no buffer requests, then sends POWER_ON_DONE on both. So rings and firmware state stay in DRAM and only the RTKit handshake is repeated. Do the same, in macOS's order and values: - Down: GFX's idle, the gating word's bit 2, GFX1's idle, each instance's mailbox off after its IOP ack and its run bit 5 us later, the PMP's request for the GPU withdrawn, then automatic power management, the bridge words, and the target, polled until its busy bit clears. Both idle first, so a refusal by the second can still wake the first. - Up: the target, the bridge, automatic power management, the PMP's request, gating, then GFX1's run bit and later GFX's. The prepare callback closes publication, gives work in flight 3 s to retire and refuses the sleep otherwise. It runs before any device suspends. An idled firmware can't be woken with its power kept, so where the sleep controller can take it, the down sequence runs in C as the first of the sleep's last stops, after every refusal and with interrupts off: both instances idled through RTKit's polled idle, each mailbox's receive and run bit off, the PMP's request withdrawn while the PMP still runs, then the bridge and the domain. Told to idle, the firmware can't carry on, so the sleep goes on whatever it answers. .complete asks whether that stop ran: if it didn't, the sleep was refused before the commit, the firmware never stopped and publication reopens; if it did, the books are settled without touching the registers and the wake's restart follows. Elsewhere .prepare stops the GPU itself. A sleep refused after a stop leaves the firmware stopped and publication closed, but brings the domain back: userspace runs again during the hold, and ioctls, debugfs and unbind can all reach what's behind it. GFX's domain is driver-owned for the sleep, so the power-domain resync leaves it to this order. The GPU's OSLOG counters stay unanswered, as before: RTKit now echoes every T8140 coprocessor's, and no soak has echoed the GPU's yet. The ordering logic is a pure function over a hardware trait, with KUnit tests. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/driver.rs | 68 ++ drivers/gpu/drm/asahi/g15/mod.rs | 4 + drivers/gpu/drm/asahi/g16/mod.rs | 4 + drivers/gpu/drm/asahi/g17/mod.rs | 90 +- drivers/gpu/drm/asahi/g17/port.rs | 5 + drivers/gpu/drm/asahi/g17/queue/render.rs | 4 + .../gpu/drm/asahi/g17/queue/render/parked.rs | 6 + drivers/gpu/drm/asahi/g17/reports.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 73 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 22 + drivers/gpu/drm/asahi/g17/sleep.c | 350 ++++++++ drivers/gpu/drm/asahi/g17/sleep.h | 22 + drivers/gpu/drm/asahi/g17/sleep.rs | 808 ++++++++++++++++++ drivers/gpu/drm/asahi/hw/t6040.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 1 + drivers/gpu/drm/asahi/hw/t8140.rs | 1 + drivers/gpu/drm/asahi/regs.rs | 51 ++ 18 files changed, 1495 insertions(+), 19 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/sleep.c create mode 100644 drivers/gpu/drm/asahi/g17/sleep.h create mode 100644 drivers/gpu/drm/asahi/g17/sleep.rs diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index db13d06336b3c..c32b1bf5f055d 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/sleep.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 93c52f1de9968..35e495893878e 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -231,12 +231,80 @@ asahi_of_device_table!( #[cfg(CONFIG_DRM_ASAHI_G15)] asahi_of_device_table!(); +struct SleepOps(bindings::dev_pm_ops); + +// SAFETY: Immutable: the callbacks are plain functions. +unsafe impl Sync for SleepOps {} + +/// # Safety +/// +/// `dev` must be a device this driver is bound to, as for a PM core callback. +unsafe fn sleep_data<'a>(dev: *mut bindings::device) -> &'a AsahiPlatformDriverData<'a> { + // SAFETY: Probe stored a `Pin>` here, and the PM core keeps the + // device bound for the duration of a callback. + unsafe { &*bindings::dev_get_drvdata(dev).cast() } +} + +/// Runs before any device suspends, so the GPU goes first and work still in flight can finish. +unsafe extern "C" fn sleep_prepare(dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: Called by the PM core. + let data = unsafe { sleep_data(dev) }; + match data.g17.as_ref() { + Some(gpu) => kernel::error::from_result(|| gpu.sleep_prepare().map(|_| 0)), + None => EBUSY.to_errno(), + } +} + +/// Runs after every device resumed. Does nothing unless [`sleep_prepare`] stopped the firmware. +unsafe extern "C" fn sleep_complete(dev: *mut bindings::device) { + // SAFETY: Called by the PM core. + if let Some(gpu) = unsafe { sleep_data(dev) }.g17.as_ref() { + gpu.sleep_complete(); + } +} + +static SLEEP_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_prepare), + complete: Some(sleep_complete), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// For a driver with no firmware restart: a sleep that powers the SoC down would lose its firmware. +unsafe extern "C" fn sleep_refuse(_dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if unsafe { bindings::pm_suspend_via_firmware() } { + EBUSY.to_errno() + } else { + 0 + } +} + +static SLEEP_REFUSE_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_refuse), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// The callbacks of a driver that refuses a sleep that powers the SoC down. +pub(crate) fn sleep_refuse_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_REFUSE_OPS.0) +} + /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { type IdInfo = ProbeConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_OPS.0) + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 9ffd858598b1c..ee9708c88cee5 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -179,6 +179,10 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(driver) = &this._driver { driver.close_binding(); diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 4cf9057f04fb0..8ab2c88244fbf 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -145,6 +145,10 @@ impl platform::Driver for Driver { type Data<'bound> = Option>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ffb0c20d0267f..dd6698e7b7a9c 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -57,6 +57,7 @@ mod recovery; mod replay; mod reports; mod runtime; +mod sleep; pub(crate) mod stats; pub(crate) mod status; mod teardown; @@ -191,6 +192,9 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, + /// The firmware's warm restart across a system sleep is known for this SoC. Otherwise the + /// driver refuses a sleep that powers the SoC down. + pub(crate) sleep_restart: bool, /// The SoC's values of region C. pub(crate) region_c: &'static fw::initdata::RegionCConfig, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, @@ -531,7 +535,11 @@ impl Coprocessor { ) -> Result> { for attempt in 0..RECEIVER_ATTEMPTS { match rtkit::RtKit::::new(dev.as_ref().as_ref(), None, index, state.clone()) { - Ok(receiver) => return Ok(receiver), + Ok(receiver) => { + // Its OSLOG counters stay unanswered until a soak shows echoing them is safe. + receiver.set_oslog_echo(false); + return Ok(receiver); + } Err(error) if error == EINVAL || error == ETIMEDOUT => { if attempt + 1 < RECEIVER_ATTEMPTS { fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); @@ -573,6 +581,15 @@ impl Coprocessor { self.rtkit.send(EP_DOORBELL, msg) } + /// Starts the CPU of firmware that lost power with its state intact, and waits for RTKit to + /// come up. No endpoint starts and no initialization: it already has both. + fn restart(&mut self) -> Result { + self.rtkit.with(|rtkit| rtkit.warm_restart())?; + self.cpu.start()?; + self.running = true; + self.rtkit.with(|rtkit| rtkit.boot()) + } + /// Stops the CPU. The firmware keeps its state, but no longer runs. fn stop(&mut self) { if self.running { @@ -580,6 +597,14 @@ impl Coprocessor { self.running = false; } } + + /// The books for a CPU that the sleep's commit stopped, with no register access. + fn stopped_at_commit(&mut self) { + if self.running { + self.cpu.stopped_elsewhere(); + self.running = false; + } + } } impl Drop for Coprocessor { @@ -612,6 +637,44 @@ impl Bridge { } } + /// The bridge across a power-off of the GPU's domain. Nothing on a target that runs without + /// it. Teardown still clears it afterwards. + fn set(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + return self.regs.enable_g17_bridge(); + } + self.regs.disable_g17_bridge(); + Ok(()) + } + + /// Core gating across a power-off of the GPU's domain. Nothing on a target that runs without + /// the bridge. + fn set_gating(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + self.regs.enable_g17_gating() + } else { + self.regs.hold_g17_gating() + } + } + + /// Both bridge words and the gating word, on a target that has them. + fn words(&self) -> Option<[u32; 3]> { + if !self.enabled { + return None; + } + self.regs.g17_fabric_words().ok() + } + + fn forget(&mut self) { + self.enabled = false; + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -686,7 +749,10 @@ impl Firmware { /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { use recovery::Memory; - if self.recovery.pending() || self.init.recovery_state()? != 0 { + if self.primary.state.shared.sleeping() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + { return Err(EAGAIN); } let target = channel::Rings::publish_control(&self.init, record)?; @@ -738,7 +804,7 @@ impl Firmware { /// Observe a telemetry epoch boundary without changing GPU power policy. fn observe_statistics_power(&mut self) { - if self.rings.statistics_suspended() { + if self.rings.statistics_suspended() || self.primary.state.shared.sleeping() { return; } let roles = self.roles(); @@ -874,6 +940,14 @@ struct Shared { compute_blocks: usize, descriptor_flags: [u32; 2], crashed: AtomicBool, + /// The firmware is stopped, or about to be: nothing may read or publish to it. + sleeping: AtomicBool, + /// A sleep holds preparations closed, so a recovery settling meanwhile doesn't reopen them. + sleep_gate: AtomicBool, + /// The power state word as it was before the firmware stopped. + sleep_ps: AtomicU32, + /// The stop the sleep's commit makes, from `.prepare` to `.complete`; null otherwise. + sleep_commit: core::sync::atomic::AtomicPtr, feed: Arc, #[pin] feed_work: Work, @@ -943,6 +1017,9 @@ impl_has_work! { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } if let Err(error) = this.grow_render() { if this.state.lock().is_some() { dev_err!( @@ -966,6 +1043,9 @@ impl WorkItem for Shared { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } let result = { let mut state = this.state.lock(); match (*state).as_deref_mut() { @@ -1256,6 +1336,10 @@ impl Gpu { compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), + sleeping: AtomicBool::new(false), + sleep_gate: AtomicBool::new(false), + sleep_ps: AtomicU32::new(0), + sleep_commit: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 425eb6f517b71..e155fa40f5bfb 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -528,6 +528,11 @@ impl Port { self.demand.set_active(full); } + /// The next statistics drain and reader discard what the decoder holds. + pub(super) fn statistics_epoch_bump(&self) { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } + pub(super) fn statistics_epoch(&self) -> u64 { self.statistics_epoch.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 76447a297c5ec..ba770a46a5f2a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2207,6 +2207,10 @@ impl Pair { } } } + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } + /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. /// The steady state in which a command may publish without the device diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c..9d8c108b3cab3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -257,6 +257,12 @@ impl ParkedPair { } } +impl ParkedPair { + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } +} + impl Prepared { /// Called after the registry revalidates its preparation epoch and parked pair identity. pub(crate) fn finish_install( diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 543612cd211a6..bbbb73d63dc5a 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -104,7 +104,7 @@ fn write_power( firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>, ) -> kernel::fmt::Result { - if firmware.rings.statistics_suspended() { + if firmware.rings.statistics_suspended() || firmware.primary.state.shared.sleeping() { return writeln!(f, "power_management=firmware suspended=1"); } let core_power = firmware.bridge.regs.g17_core_power(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4af2704be0b44..0ebdb0ca972c9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -301,6 +301,57 @@ impl Firmware { Ok(()) } + /// A step of the event pass is waiting on the firmware with no event to wake it. + fn runtime_polling(&self, now: u64) -> bool { + self.render_control_backpressured() + || self.queues.replay_outer_wait.is_some() + || self.queues.render.replay_polling() + || self.queues.teardown.polling(now) + || self.compute_exit_polling(now) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()) + } + + /// Nothing is published that hasn't retired, nothing is waiting on the firmware, its rings + /// are empty, and the idle check has taken the effort and power holds down. + pub(in crate::g17) fn sleep_quiet(&self, now: u64) -> Result { + use crate::g17::recovery::Memory; + let compute = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + if compute + || self.queues.render_in_flight() + || !self.queues.render_settled() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + || self.runtime_polling(now) + || self.hold.active() + || self.registration_power_release_pending + || self.effort.applied() != 0 + { + return Ok(false); + } + let [consumer, unknown, producer] = self.init.control_counters()?; + if consumer != producer || unknown != producer { + return Ok(false); + } + for &role in self.roles() { + if channel::Events::pending(&self.init, role)? { + return Ok(false); + } + } + Ok(true) + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -377,6 +428,10 @@ impl crate::g17::Shared { /// callbacks are finished. Shutdown can detach Firmware during an off-lock /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { + // The way back queues a pass, which also handles a crash noticed meanwhile. + if self.sleeping() { + return; + } if self.crashed.load(core::sync::atomic::Ordering::Acquire) { self.stop_runtime(); return; @@ -442,7 +497,9 @@ impl crate::g17::Shared { } if recover { firmware.service_runtime_recovery(deferred)?; - if !firmware.recovery.pending() { + if !firmware.recovery.pending() + && !self.sleep_gate.load(core::sync::atomic::Ordering::Acquire) + { if !self.preparations.reopen() { return Ok(()); } @@ -518,19 +575,7 @@ impl crate::g17::Shared { self.queue_reclaims(); } } - let backpressured = firmware.render_control_backpressured(); - let polling = backpressured - || firmware.queues.replay_outer_wait.is_some() - || firmware.queues.render.replay_polling() - || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now) - || firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); + let polling = firmware.runtime_polling(now); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index fc2e01be761ae..8d99333b86e25 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -1038,6 +1038,28 @@ impl Registry { } Ok(()) } + /// No graph is being built or grown. + pub(in crate::g17) fn render_settled(&self) -> bool { + self.render + .entries + .iter() + .flatten() + .all(|entry| entry.preparing.is_none() && !entry.growing) + } + + /// The firmware restarted with no power on the cores: nothing it was told about a pair through + /// their registers can be relied on, so the next command installs it again. + pub(in crate::g17) fn forget_render_registrations(&mut self) { + for entry in self.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + pair.forget_registration(); + } + if let Some(parked) = entry.parked.as_mut() { + parked.forget_registration(); + } + } + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c new file mode 100644 index 0000000000000..57af9cc3dfef0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The GPU's power state word across a system sleep. The domain is always on as far + * as genpd knows, so nothing else lowers it. The word and its parent come from the + * device tree. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sleep.h" + +#define PS_AUTO_ENABLE BIT(28) +#define PS_AUTO GENMASK(27, 24) +#define PS_BUSY BIT(11) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTIVE 0xf +#define PS_POLL_US 100 +#define PS_TIMEOUT_US 50000 + +/* In the "sgx" range, as regs.rs has them. */ +#define SGX_GATING 0xd06030 +#define SGX_GATING_HOLD BIT(2) +#define SGX_BRIDGE 0x1000104 +#define SGX_BRIDGE_WORDS 2 +#define SGX_BRIDGE_ENABLE BIT(0) + +#define ASC_CPU_STATUS 0x4 + +/* The PMP's request for the GPU, where the device tree names one. */ +int asahi_g17_pmp_request(struct device *dev, bool enable) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (!np) + return 0; + ret = apple_pmp_report_entry_request(np, enable); + of_node_put(np); + + return ret; +} + +static int asahi_g17_ps(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -EINVAL; + ret = of_property_read_u32(np, "reg", offset); + if (!ret) { + *regmap = np->parent ? syscon_node_to_regmap(np->parent) : ERR_PTR(-EINVAL); + ret = PTR_ERR_OR_ZERO(*regmap); + } + of_node_put(np); + + return ret; +} + +/* Automatic power management off. Returns the word as it was in @saved. */ +int asahi_g17_power_auto_off(struct device *dev, u32 *saved) +{ + struct regmap *regmap; + u32 offset; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, saved); + if (ret) + return ret; + + return regmap_write(regmap, offset, *saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS)); +} + +/* The target off, then wait for the domain to finish going off. */ +int asahi_g17_power_lower(struct device *dev) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + regmap_write(regmap, offset, + (val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS); + + ret = regmap_read_poll_timeout(regmap, offset, cur, + !FIELD_GET(PS_ACTUAL, cur) && !(cur & PS_BUSY), + PS_POLL_US, PS_TIMEOUT_US); + dev_dbg(dev, "GPU domain %#x, now %#x\n", val, cur); + + return ret; +} + +static u32 asahi_g17_ps_on(u32 val, u32 saved) +{ + val &= ~(PS_AUTO_ENABLE | PS_AUTO | PS_BUSY | PS_FLAGS | PS_ACTUAL | PS_TARGET); + + return val | (saved & PS_AUTO) | FIELD_PREP(PS_TARGET, PS_ACTIVE); +} + +/* The target on, and wait for it. */ +int asahi_g17_power_raise(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + dev_dbg(dev, "GPU domain before raising it: %#x\n", val); + + regmap_write(regmap, offset, asahi_g17_ps_on(val, saved)); + + return regmap_read_poll_timeout(regmap, offset, cur, + FIELD_GET(PS_ACTUAL, cur) == PS_ACTIVE, + PS_POLL_US, PS_TIMEOUT_US); +} + +/* Automatic power management as it was. */ +int asahi_g17_power_auto_on(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + return regmap_write(regmap, offset, asahi_g17_ps_on(val, saved) | (saved & PS_AUTO_ENABLE)); +} + +/* + * What the sleep's commit needs to stop the GPU with interrupts off, taken in .prepare, and + * what it saw, for the log after the sleep. + */ +struct asahi_g17_commit { + struct device *dev; + unsigned int count; + struct apple_rtkit *rtk[2]; + struct apple_mbox *mbox[2]; + void __iomem *cpu[2]; + void __iomem *gating; + void __iomem *bridge; + void __iomem *ps; + struct device_node *request; + + bool ran; + int idle[2]; + unsigned int idle_us[2]; + u32 status[2]; + u32 words[SGX_BRIDGE_WORDS + 1]; + int request_ret; + u32 saved; + u32 lowered; + int lower_ret; +}; + +/* + * down() at the commit, every refusal behind it: both instances idled by polling, mailboxes + * and run bits off, the gating hold, the PMP request withdrawn while the PMP still runs, then + * the bridge and the domain. Told to idle, the firmware can't carry on, so the sleep goes on + * whatever it answers. + */ +static int asahi_g17_commit_stop(void *data) +{ + struct asahi_g17_commit *c = data; + unsigned int i; + u32 val; + + c->ran = true; + for (i = 0; i < c->count; i++) + c->idle[i] = apple_rtkit_sleep_idle_polled(c->rtk[i], c->cpu[i], &c->idle_us[i]); + for (i = 0; i < c->count; i++) { + c->status[i] = readl(c->cpu[i] + ASC_CPU_STATUS); + apple_rtkit_mailbox_recv_off(c->rtk[i]); + apple_mbox_ascwrap_v6_stop_cpu_atomic(c->mbox[i]); + if (!i && c->bridge) + writel(readl(c->gating) | SGX_GATING_HOLD, c->gating); + } + c->request_ret = c->request ? apple_pmp_report_entry_withdraw_polled(c->request) : 0; + + c->saved = readl(c->ps); + writel(c->saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS), c->ps); + if (c->bridge) { + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + c->words[i] = readl(c->bridge + 4 * i); + c->words[i] = readl(c->gating); + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + writel(c->words[i] & ~SGX_BRIDGE_ENABLE, c->bridge + 4 * i); + } + val = readl(c->ps); + writel((val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS, c->ps); + c->lower_ret = readl_poll_timeout_atomic(c->ps, c->lowered, + !FIELD_GET(PS_ACTUAL, c->lowered) && + !(c->lowered & PS_BUSY), + 10, PS_TIMEOUT_US); + + return (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) ? 1 : 0; +} + +static void __iomem *asahi_g17_map(struct device_node *np, const char *name, int index, + u32 offset, u32 size) +{ + struct resource res; + + if (name) + index = of_property_match_string(np, "reg-names", name); + if (index < 0 || of_address_to_resource(np, index, &res) || + offset + size > resource_size(&res)) + return NULL; + + return ioremap(res.start + offset, size); +} + +static void asahi_g17_commit_free(struct asahi_g17_commit *c) +{ + if (c->request) { + apple_pmp_report_entry_commit_owned(c->request, false); + of_node_put(c->request); + } + if (c->ps) + iounmap(c->ps); + if (c->bridge) + iounmap(c->bridge); + if (c->gating) + iounmap(c->gating); + kfree(c); +} + +/* + * Leaves the GPU's whole stop to the sleep's commit: maps what the stop touches, keeps the + * PMP request for it, and registers it with the sleep driver, ready. Nothing is stopped + * here. @secondary_rtk is NULL on a single instance, and @bridge false on a target that runs + * without the bridge. + */ +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge) +{ + struct asahi_g17_commit *c; + struct device_node *np; + unsigned int i; + u32 offset; + int ret; + + c = kzalloc_obj(*c); + if (!c) + return ERR_PTR(-ENOMEM); + c->dev = dev; + c->rtk[0] = primary_rtk; + c->mbox[0] = primary_mbox; + c->rtk[1] = secondary_rtk; + c->mbox[1] = secondary_mbox; + c->count = secondary_rtk ? 2 : 1; + + ret = -EOPNOTSUPP; + for (i = 0; i < c->count; i++) { + c->cpu[i] = apple_mbox_ascwrap_v6_cpu_control(c->mbox[i]); + if (!c->rtk[i] || !c->cpu[i]) + goto err; + } + + ret = -ENODEV; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + goto err; + if (!of_property_read_u32(np, "reg", &offset) && np->parent) + c->ps = asahi_g17_map(np->parent, NULL, 0, offset, 4); + of_node_put(np); + c->gating = asahi_g17_map(dev->of_node, "sgx", 0, SGX_GATING, 4); + if (bridge) + c->bridge = asahi_g17_map(dev->of_node, "sgx", 0, SGX_BRIDGE, + 4 * SGX_BRIDGE_WORDS); + if (!c->ps || !c->gating || (bridge && !c->bridge)) + goto err; + + c->request = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (c->request) + apple_pmp_report_entry_commit_owned(c->request, true); + + ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + if (ret) + goto err; + apple_sleep_coproc_set_ready(dev, true); + + return c; + +err: + asahi_g17_commit_free(c); + return ERR_PTR(ret); +} + +/* + * After the sleep, refused or not: whether the commit stopped the GPU, and in @saved the + * domain's word from before, if it did. Frees @c. + */ +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved) +{ + bool stopped = apple_sleep_coproc_stopped(c->dev) || c->ran; + + apple_sleep_coproc_unregister(c->dev); + if (c->ran) { + if (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) + dev_warn(c->dev, "GPU stop at the commit: idle %d %d, PMP request %d, domain %#x (%d)\n", + c->idle[0], c->idle[1], c->request_ret, c->lowered, c->lower_ret); + dev_dbg(c->dev, "GPU stopped at the commit: idle in %u %u us, status %#x %#x\n", + c->idle_us[0], c->idle_us[1], c->status[0], c->status[1]); + dev_dbg(c->dev, "GPU at the commit: bridge %#x %#x gating %#x, domain %#x now %#x\n", + c->words[0], c->words[1], c->words[2], c->saved, c->lowered); + *saved = c->saved; + } + asahi_g17_commit_free(c); + + return stopped; +} diff --git a/drivers/gpu/drm/asahi/g17/sleep.h b/drivers/gpu/drm/asahi/g17/sleep.h new file mode 100644 index 0000000000000..21b91dfca0a2e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SLEEP_H +#define ASAHI_G17_SLEEP_H +#include + +struct apple_mbox; +struct apple_rtkit; +struct asahi_g17_commit; +struct device; + +int asahi_g17_pmp_request(struct device *dev, bool enable); +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge); +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved); +int asahi_g17_power_auto_off(struct device *dev, u32 *saved); +int asahi_g17_power_lower(struct device *dev); +int asahi_g17_power_raise(struct device *dev, u32 saved); +int asahi_g17_power_auto_on(struct device *dev, u32 saved); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sleep.rs b/drivers/gpu/drm/asahi/g17/sleep.rs new file mode 100644 index 0000000000000..0db5f0a9a3d23 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.rs @@ -0,0 +1,808 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! System sleep: the firmware goes idle, its domain goes off, and it restarts warm. +//! +//! The firmware isn't initialized again. It keeps its rings and state in DRAM, so a restart is +//! the RTKit handshake and the end of the power-on transition. Publication stays closed from +//! before the idle until after the restart, and nothing is in flight across it: work that doesn't +//! finish in time refuses the sleep. +//! +//! Order, on both instances, is what macOS does on the chip whose [`super::Config`] sets +//! `sleep_restart`. Other chips refuse. +//! +//! Only a real wake restarts the firmware. A sleep refused after the firmware stopped leaves it +//! stopped and publication closed until the next real sleep, with the domain on so that everything +//! behind it stays reachable. That sleep then only has the domain left to lower. +//! +//! Where the sleep driver takes it, `.prepare` only drains and checks for quiet, and the sleep's +//! commit makes the whole stop in C, with interrupts off and every refusal behind it. A sleep +//! refused before then finds the firmware never stopped. Elsewhere `.prepare` stops it. + +use super::{Firmware, Role, Shared}; +use core::sync::atomic::Ordering; +use kernel::{ + bindings, + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, + workqueue::{DelayedWork, Work}, +}; + +/// Time work in flight gets to finish before the sleep is refused. +const DRAIN_NS: u64 = 3_000_000_000; +const DRAIN_TICK_MS: i64 = 2; + +unsafe extern "C" { + fn asahi_g17_pmp_request(dev: *mut core::ffi::c_void, enable: bool) -> core::ffi::c_int; + fn asahi_g17_power_auto_off(dev: *mut core::ffi::c_void, saved: *mut u32) -> core::ffi::c_int; + fn asahi_g17_power_lower(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_g17_power_raise(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_power_auto_on(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_commit_prepare( + dev: *mut core::ffi::c_void, + primary_rtk: *mut bindings::apple_rtkit, + primary_mbox: *mut core::ffi::c_void, + secondary_rtk: *mut bindings::apple_rtkit, + secondary_mbox: *mut core::ffi::c_void, + bridge: bool, + ) -> *mut core::ffi::c_void; + fn asahi_g17_commit_finish(commit: *mut core::ffi::c_void, saved: *mut u32) -> bool; +} + +/// What `.complete` does after a sleep whose stop was left to the commit. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum AfterCommit { + /// Refused before the commit: the firmware never stopped, so only publication reopens. + Running, + /// Stopped at the commit, or its power lost anyway: as after [`down`]. + Stopped, +} + +fn after_commit(stopped: bool, real: bool) -> AfterCommit { + if stopped || real { + AfterCommit::Stopped + } else { + AfterCommit::Running + } +} + +/// What stopping and restarting the firmware touches, in the order [`down`] and [`up`] use it. +pub(super) trait Hardware { + fn has_secondary(&self) -> bool; + /// RTKit AP and IOP idle, each acknowledged. + fn idle(&mut self, role: Role) -> Result; + /// Back from idle with no restart, after a refusal. One that fails loses the device. + fn wake(&mut self, role: Role); + /// Mailbox off, then run bit off, for an instance that acknowledged idle. + fn stop(&mut self, role: Role); + /// The PMP's request for the GPU. It may use a device it's told is in use, so the request is + /// only ever set with the domain on. + fn request(&mut self, enable: bool); + /// Core gating as it is while the firmware runs, or held for a domain about to go off. + fn gating(&mut self, enable: bool) -> Result; + /// The GPU-to-fabric bridge. + fn bridge(&mut self, enable: bool) -> Result; + /// Automatic power management off, keeping the word for the way back. + fn auto_off(&mut self) -> Result; + /// The domain off, waiting until it has settled. + fn lower(&mut self) -> Result; + /// The domain on, waiting until it is. + fn raise(&mut self) -> Result; + /// Automatic power management as it was. + fn auto_on(&mut self) -> Result; + /// Run bit on, RTKit handshake to AP on. + fn restart(&mut self, role: Role) -> Result; + /// Ends the power-on transition, on every instance. + fn power_on_done(&mut self) -> Result; +} + +/// Stops the firmware and lowers the domain. An error leaves the firmware running, so the caller +/// refuses the sleep. Both instances idle before either stops, so that a refusal by the second +/// still has a way back. A domain that is slow to go off isn't an error here: the sleep driver +/// checks it at the commit. +pub(super) fn down(hw: &mut H) -> Result { + if let Err(error) = hw.idle(Role::Primary) { + hw.wake(Role::Primary); + return Err(error); + } + if hw.has_secondary() { + if let Err(error) = hw.idle(Role::Secondary) { + hw.wake(Role::Secondary); + hw.wake(Role::Primary); + return Err(error); + } + } + hw.stop(Role::Primary); + let _ = hw.gating(false); + if hw.has_secondary() { + hw.stop(Role::Secondary); + } + hw.request(false); + lower_domain(hw); + Ok(()) +} + +/// Lowers the domain, with the firmware stopped. +fn lower_domain(hw: &mut H) { + let _ = hw.auto_off(); + let _ = hw.bridge(false); + let _ = hw.lower(); +} + +/// Brings the domain back for firmware that stays stopped after a refused sleep. +fn hold(hw: &mut H) -> Result { + hw.raise()?; + hw.auto_on() +} + +/// Restarts the firmware. Any error loses the device: there is no other way back. +pub(super) fn up(hw: &mut H) -> Result { + hw.raise()?; + hw.bridge(true)?; + hw.auto_on()?; + hw.request(true); + hw.gating(true)?; + if hw.has_secondary() { + hw.restart(Role::Secondary)?; + } + hw.restart(Role::Primary)?; + hw.power_on_done() +} + +impl Hardware for Firmware { + fn has_secondary(&self) -> bool { + self.secondary.is_some() + } + + fn idle(&mut self, role: Role) -> Result { + self.coprocessor(role)?.rtkit.with(|rtkit| rtkit.idle()) + } + + fn wake(&mut self, role: Role) { + let result = match self.coprocessor(role) { + Ok(coprocessor) => coprocessor.rtkit.with(|rtkit| rtkit.wake()), + Err(error) => Err(error), + }; + // EINVAL: it never left the on state. + if result.is_err_and(|error| error != EINVAL) { + self.primary.state.shared.lose_device_quietly(); + } + } + + fn stop(&mut self, role: Role) { + if let Ok(coprocessor) = self.coprocessor(role) { + coprocessor.rtkit.with(|rtkit| rtkit.mailbox_off()); + coprocessor.stop(); + } + } + + fn request(&mut self, enable: bool) { + self.primary.state.shared.pmp_request(enable); + } + + fn gating(&mut self, enable: bool) -> Result { + self.bridge.set_gating(enable) + } + + fn bridge(&mut self, enable: bool) -> Result { + if let Some([first, second, gating]) = self.bridge.words() { + dev_dbg!( + self.primary.state.shared.dev.as_ref(), + "GPU before bridge {}: {:#x} {:#x} gating {:#x}\n", + if enable { "on" } else { "off" }, + first, + second, + gating + ); + } + self.bridge.set(enable) + } + + fn auto_off(&mut self) -> Result { + self.primary.state.shared.power_auto_off() + } + + fn lower(&mut self) -> Result { + self.primary.state.shared.power_lower() + } + + fn raise(&mut self) -> Result { + let result = self.primary.state.shared.power_raise(); + if result.is_err() { + // The registers aren't reachable, so teardown can't clear the bridge. + self.bridge.forget(); + } + result + } + + fn auto_on(&mut self) -> Result { + self.primary.state.shared.power_auto_on() + } + + fn restart(&mut self, role: Role) -> Result { + let shared = self.primary.state.shared.clone(); + let coprocessor = self.coprocessor(role)?; + let result = coprocessor.restart(); + if let (Err(_), Ok((control, status))) = (&result, coprocessor.cpu.state()) { + dev_err!( + shared.dev.as_ref(), + "GPU firmware {:?}: control {:#x} status {:#x}\n", + role, + control, + status + ); + } + result + } + + fn power_on_done(&mut self) -> Result { + if !self.variant.power_on_done { + return Ok(()); + } + self.primary.notify(super::MSG_POWER_ON_DONE)?; + if let Some(secondary) = self.secondary.as_ref() { + secondary.notify(super::MSG_POWER_ON_DONE)?; + } + Ok(()) + } +} + +impl Firmware { + /// Hands the sleep's commit what its stop needs. Nothing stops here. + fn leave_for_commit(&mut self) -> Result<*mut core::ffi::c_void> { + let shared = self.primary.state.shared.clone(); + let primary_rtk = self.primary.rtkit.with(|rtkit| rtkit.as_raw()); + let primary_mbox = self.primary.cpu.as_raw().cast(); + let (secondary_rtk, secondary_mbox) = match self.secondary.as_ref() { + Some(secondary) => ( + secondary.rtkit.with(|rtkit| rtkit.as_raw()), + secondary.cpu.as_raw().cast(), + ), + None => (core::ptr::null_mut(), core::ptr::null_mut()), + }; + // SAFETY: `dev` is the live GPU device. The RTKit and mailbox pointers stay valid while + // the firmware exists, and `.complete` frees the result before anything drops them. + let commit = unsafe { + asahi_g17_commit_prepare( + shared.dev.as_raw().cast(), + primary_rtk, + primary_mbox, + secondary_rtk, + secondary_mbox, + self.bridge.enabled, + ) + }; + kernel::error::from_err_ptr(commit) + } + + /// The books for both instances, which the commit stopped. + fn stopped_at_commit(&mut self) { + if let Some(secondary) = self.secondary.as_mut() { + secondary.stopped_at_commit(); + } + self.primary.stopped_at_commit(); + } + + fn coprocessor(&mut self, role: Role) -> Result<&mut super::Coprocessor> { + match role { + Role::Primary => Ok(&mut self.primary), + Role::Secondary => self.secondary.as_mut().ok_or(ENODEV), + } + } +} + +impl Shared { + pub(super) fn sleeping(&self) -> bool { + self.sleeping.load(Ordering::Acquire) + } + + /// A PMP that doesn't answer in time only gets a warning: nothing better is left to do here. + fn pmp_request(&self, enable: bool) { + // SAFETY: `dev` is the live GPU device. + let result = kernel::error::to_result(unsafe { + asahi_g17_pmp_request(self.dev.as_raw().cast(), enable) + }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "PMP request for the GPU, {}: {:?}\n", enable, error); + } + } + + fn power_auto_off(&self) -> Result { + let mut saved = 0; + // SAFETY: `dev` is the live GPU device and `saved` is writable. + let result = kernel::error::to_result(unsafe { + asahi_g17_power_auto_off(self.dev.as_raw().cast(), &mut saved) + }); + self.sleep_ps.store(saved, Ordering::Relaxed); + result + } + + fn power_lower(&self) -> Result { + // SAFETY: `dev` is the live GPU device. + let result = + kernel::error::to_result(unsafe { asahi_g17_power_lower(self.dev.as_raw().cast()) }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "GPU domain slow to go off: {:?}\n", error); + } + result + } + + fn power_raise(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { asahi_g17_power_raise(self.dev.as_raw().cast(), saved) }) + } + + fn power_auto_on(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { + asahi_g17_power_auto_on(self.dev.as_raw().cast(), saved) + }) + } + + fn cancel_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: As in `disable_timer`: a pending callback owned one Arc that it can no longer + // reclaim. Future enqueues stay possible and run into the sleeping check. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, bindings::delayed_work, work); + bindings::cancel_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: See above. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// With the sleeping flag set, joins whatever may be reading firmware state. The growth work + /// isn't joined: it may be waiting on the closed gate, and can't publish while it is closed. + fn join_event_workers(self: &Arc) { + self.cancel_timer(&self.poll); + self.cancel_timer(&self.idle); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + } + + /// Waits for what's been published to retire. The event worker keeps running meanwhile, and + /// its own idle check takes the power and effort holds down. + fn wait_quiet(self: &Arc, deadline: u64) -> Result { + loop { + if self.crashed.load(Ordering::Acquire) { + return Err(ENODEV); + } + let now = super::now_ns(); + { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + // Nothing else retries this with no render publication coming. + if firmware.registration_power_release_pending { + let _ = firmware.release_registration_power(); + } + if firmware.sleep_quiet(now)? { + return Ok(()); + } + } + if now >= deadline { + return Err(EBUSY); + } + self.queue_events(); + fsleep(Delta::from_millis(DRAIN_TICK_MS)); + } + } + + /// Undoes everything short of a stopped firmware. A builder that never finishes would keep + /// publication closed for good, and that is a lost device. + fn sleep_cancel(self: &Arc) { + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + let drained = self + .preparations + .wait_drained_until(super::now_ns().saturating_add(DRAIN_NS)); + if (drained.is_err() || !self.preparations.reopen()) + && !self.crashed.load(Ordering::Acquire) + { + self.lose_device_quietly(); + } + self.queue_events(); + } + + /// Keeps the firmware stopped after a sleep that never reached the hardware. False if there's + /// no firmware to keep, or no domain to keep it in. + fn sleep_hold(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { + return false; + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return false; + }; + if hold(firmware).is_err() { + return false; + } + dev_warn!( + self.dev.as_ref(), + "sleep refused: GPU firmware stays stopped until a real sleep\n" + ); + true + } + + /// Leaves the stop to the commit. False, with the reason logged, if it can't be. + fn sleep_leave_for_commit(&self, firmware: &mut Firmware) -> bool { + match firmware.leave_for_commit() { + Ok(commit) => { + self.sleep_commit.store(commit, Ordering::Release); + dev_dbg!(self.dev.as_ref(), "GPU firmware left running for the sleep's commit\n"); + true + } + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "no GPU stop at the commit ({:?}): stopping it now\n", + error + ); + false + } + } + } + + /// Whether the commit stopped the firmware, if the stop was left to it. + fn sleep_commit_finish(&self) -> Option { + let commit = self.sleep_commit.swap(core::ptr::null_mut(), Ordering::AcqRel); + if commit.is_null() { + return None; + } + let mut saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `commit` came from `asahi_g17_commit_prepare` and is finished only once. + let stopped = unsafe { asahi_g17_commit_finish(commit, &mut saved) }; + self.sleep_ps.store(saved, Ordering::Relaxed); + Some(stopped) + } + + /// After the commit stopped the firmware: the books it left, as [`down`] would have them. + fn sleep_settle_commit(&self) { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.stopped_at_commit(); + } + } + + /// Closes publication, waits for quiet, and leaves the stop to the sleep's commit where the + /// sleep driver takes it, or stops the firmware. A refusal changes nothing. + fn sleep_down(self: &Arc) -> Result { + if self.sleeping() { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + lower_domain(firmware); + } + return Ok(()); + } + let deadline = super::now_ns().saturating_add(DRAIN_NS); + self.sleep_gate.store(true, Ordering::Release); + self.preparations.close(); + let result = self.preparations.wait_drained_until(deadline).map_err(|_| EBUSY) + .and_then(|_| self.wait_quiet(deadline)) + .and_then(|_| { + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !firmware.sleep_quiet(super::now_ns())? { + return Err(EBUSY); + } + if self.sleep_leave_for_commit(firmware) { + return Ok(()); + } + down(firmware) + }); + match result { + Ok(()) => Ok(()), + Err(error) if error == ENODEV || self.crashed.load(Ordering::Acquire) => { + // Nothing to save: let the worker finish detaching what's left, and then the + // domain only has to be off for the sleep. + self.sleeping.store(false, Ordering::Release); + self.queue_events(); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + self.pmp_request(false); + let _ = self.power_auto_off(); + let _ = self.power_lower(); + Ok(()) + } + Err(error) => { + self.sleep_cancel(); + Err(error) + } + } + } + + /// Restarts the firmware and reopens publication. A failure loses the device. + fn sleep_up(self: &Arc) -> Result { + let started = super::now_ns(); + let result = { + let mut state = self.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if !self.crashed.load(Ordering::Acquire) => { + let result = up(firmware); + if result.is_ok() { + // The cores lost power, so the next command on each pair installs it again. + firmware.queues.forget_render_registrations(); + firmware.rings.invalidate_statistics(); + firmware.port.statistics_epoch_bump(); + } + result + } + _ => self + .power_raise() + .and_then(|_| self.power_auto_on()) + .inspect(|_| self.pmp_request(true)), + } + }; + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + match result { + Ok(()) => { + if !self.crashed.load(Ordering::Acquire) { + dev_dbg!( + self.dev.as_ref(), + "GPU firmware back in {} us\n", + super::now_ns().saturating_sub(started) / 1000 + ); + } + self.preparations.reopen(); + self.queue_events(); + Ok(()) + } + Err(error) => { + dev_err!(self.dev.as_ref(), "GPU firmware did not come back: {:?}\n", error); + self.lose_device_quietly(); + Err(error) + } + } + } +} + +impl super::Gpu { + /// Called before any device suspends. Only a sleep that powers the SoC down gets here. + pub(crate) fn sleep_prepare(&self) -> Result { + if !self.cfg.sleep_restart { + dev_warn!( + self.shared.dev.as_ref(), + "no firmware restart on this chip, refusing sleep\n" + ); + return Err(EBUSY); + } + let result = self.shared.sleep_down(); + if let Err(error) = result { + dev_warn!(self.shared.dev.as_ref(), "GPU refused sleep: {:?}\n", error); + } + result + } + + /// Called after every device resumed, whether or not the sleep reached the hardware. + pub(crate) fn sleep_complete(&self) { + if !self.shared.sleeping() { + return; + } + // SAFETY: Reads a global flag word. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if let Some(stopped) = self.shared.sleep_commit_finish() { + if after_commit(stopped, real) == AfterCommit::Running { + dev_info!( + self.shared.dev.as_ref(), + "sleep refused before the commit: GPU firmware never stopped\n" + ); + self.shared.sleep_cancel(); + return; + } + self.shared.sleep_settle_commit(); + } + if !real && self.shared.sleep_hold() { + return; + } + let _ = self.shared.sleep_up(); + } +} + +#[kernel::macros::kunit_tests(asahi_g17_sleep)] +mod tests { + use super::*; + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + enum Call { + Idle(Role), + Wake(Role), + Stop(Role), + Request(bool), + Gating(bool), + Bridge(bool), + AutoOff, + Lower, + Raise, + AutoOn, + Restart(Role), + Done, + } + + struct Fake { + secondary: bool, + fail: Option, + calls: [Option; 14], + count: usize, + } + + impl Fake { + fn new(secondary: bool, fail: Option) -> Self { + Self { secondary, fail, calls: [None; 14], count: 0 } + } + + fn note(&mut self, call: Call) -> Result { + self.calls[self.count] = Some(call); + self.count += 1; + if self.fail == Some(call) { Err(EIO) } else { Ok(()) } + } + + fn called(&self, expected: &[Call]) -> bool { + self.count == expected.len() + && expected.iter().zip(self.calls.iter()).all(|(a, b)| Some(*a) == *b) + } + } + + impl Hardware for Fake { + fn has_secondary(&self) -> bool { self.secondary } + fn idle(&mut self, role: Role) -> Result { self.note(Call::Idle(role)) } + fn wake(&mut self, role: Role) { let _ = self.note(Call::Wake(role)); } + fn stop(&mut self, role: Role) { let _ = self.note(Call::Stop(role)); } + fn request(&mut self, enable: bool) { let _ = self.note(Call::Request(enable)); } + fn gating(&mut self, enable: bool) -> Result { self.note(Call::Gating(enable)) } + fn bridge(&mut self, enable: bool) -> Result { self.note(Call::Bridge(enable)) } + fn auto_off(&mut self) -> Result { self.note(Call::AutoOff) } + fn lower(&mut self) -> Result { self.note(Call::Lower) } + fn raise(&mut self) -> Result { self.note(Call::Raise) } + fn auto_on(&mut self) -> Result { self.note(Call::AutoOn) } + fn restart(&mut self, role: Role) -> Result { self.note(Call::Restart(role)) } + fn power_on_done(&mut self) -> Result { self.note(Call::Done) } + } + + use Role::{Primary as P, Secondary as S}; + + #[test] + fn down_idles_the_primary_first_and_lowers_the_domain_last() { + let mut hw = Fake::new(true, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Idle(S), + Call::Stop(P), + Call::Gating(false), + Call::Stop(S), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + } + + #[test] + fn up_restarts_the_secondary_first_and_ends_the_transition_last() { + let mut hw = Fake::new(true, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_single_instance_never_names_a_secondary() { + let mut hw = Fake::new(false, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Stop(P), + Call::Gating(false), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + let mut hw = Fake::new(false, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_refused_idle_touches_nothing_past_the_firmware() { + for call in [Call::Idle(P), Call::Idle(S)] { + let mut hw = Fake::new(true, Some(call)); + assert_eq!(down(&mut hw), Err(EIO)); + let past = |c: &Call| !matches!(c, Call::Idle(_) | Call::Wake(_)); + assert!(!hw.calls.iter().flatten().any(past)); + } + } + + #[test] + fn a_primary_that_refuses_idle_is_woken_and_nothing_stops() { + let mut hw = Fake::new(true, Some(Call::Idle(P))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Wake(P)])); + } + + #[test] + fn a_secondary_that_refuses_idle_wakes_both_and_stops_nothing() { + let mut hw = Fake::new(true, Some(Call::Idle(S))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Idle(S), Call::Wake(S), Call::Wake(P)])); + } + + #[test] + fn a_refused_sleep_only_brings_the_domain_back() { + let mut hw = Fake::new(true, None); + assert_eq!(hold(&mut hw), Ok(())); + assert!(hw.called(&[Call::Raise, Call::AutoOn])); + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(hold(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + } + + #[test] + fn the_sleep_after_a_refusal_only_lowers_the_domain() { + let mut hw = Fake::new(true, None); + lower_domain(&mut hw); + assert!(hw.called(&[Call::AutoOff, Call::Bridge(false), Call::Lower])); + } + + #[test] + fn a_slow_domain_does_not_fail_down() { + let mut hw = Fake::new(false, Some(Call::Lower)); + assert_eq!(down(&mut hw), Ok(())); + } + + #[test] + fn up_stops_at_the_first_failure() { + let mut hw = Fake::new(true, Some(Call::Restart(S))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + ])); + // No domain, so no bridge and no request either. + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + // No bridge, so the PMP isn't told about the GPU. + let mut hw = Fake::new(true, Some(Call::Bridge(true))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise, Call::Bridge(true)])); + } + + #[test] + fn a_stop_left_to_the_commit_is_undone_only_if_it_never_ran() { + assert_eq!(after_commit(false, false), AfterCommit::Running); + assert_eq!(after_commit(true, false), AfterCommit::Stopped); + assert_eq!(after_commit(true, true), AfterCommit::Stopped); + // No stop, but the power went anyway: the firmware has to restart. + assert_eq!(after_commit(false, true), AfterCommit::Stopped); + } + +} diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index d531e20c7e9f9..3258e08f9c2d1 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -55,6 +55,7 @@ pub(crate) const CONFIG: Config = Config { chip: None, adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on // the firmware's own blame. recovery_slot_keys: None, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 8be466f7842ab..cd686437e60d1 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -44,6 +44,7 @@ pub(crate) const CONFIG: Config = Config { chip: Some(&CHIP), adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G16, firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 02381c2aae344..0461fd6a6b8eb 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -54,6 +54,7 @@ pub(crate) const CONFIG: Config = Config { chip: None, adopt_root_entries: true, fabric_bridge: true, + sleep_restart: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 976c789930068..4b6af520956c1 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -80,6 +80,7 @@ const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) /// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. const G17_GATING_CONTROL: usize = 0xd06030; const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) +const G17_GATING_HOLD: u32 = 0x4; // BIT(2) /// These performance selectors must be sampled before firmware can gate the core registers. const G17_PERF_CONTROL: usize = 0xe0141c; @@ -281,6 +282,37 @@ impl CpuControl { } } } + + /// The provider, for a system sleep's last step to stop the CPU with interrupts off. + pub(crate) fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + + /// Settles the provider's books for a CPU that a system sleep's last step stopped. + pub(crate) fn stopped_elsewhere(&self) { + // SAFETY: The constructor established the live, role-checked provider. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_cpu_stopped(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!( + self.dev.as_ref(), + "Could not settle a stopped GPU coprocessor: {:?}\n", + error + ); + } + } + } + + /// The CPU's control and status words. + pub(crate) fn state(&self) -> Result<(u32, u32)> { + let (mut control, mut status) = (0, 0); + // SAFETY: The constructor established the live, role-checked provider, and both pointers + // are writable. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_cpu_state(self.mbox, &mut control, &mut status) + })?; + Ok((control, status)) + } } /// Device resources for this GPU instance. @@ -827,6 +859,25 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } + + /// Hold part of the gating off, for a domain about to go off. The firmware must be idle. + pub(crate) fn hold_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val | G17_GATING_HOLD, G17_GATING_CONTROL) + } + + /// Both bridge words and the gating word. + pub(crate) fn g17_fabric_words(&self) -> Result<[u32; 3]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + Ok([ + sgx.try_read32(G17_BRIDGE_CONTROL[0])?, + sgx.try_read32(G17_BRIDGE_CONTROL[1])?, + sgx.try_read32(G17_GATING_CONTROL)?, + ]) + } } /// Split the packed per-cluster core masks into one mask per cluster. From 087adb4fc3e34846de9ad7b344e0f235c5848e36 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:33 -0500 Subject: [PATCH 68/73] soc: apple: mtp: tell the MTP about a firmware-driven suspend An adopted MTP session refuses anything but suspend-to-idle, since nothing was known about what it needs. macOS leaves the MTP running across a sleep and sends it only the AP power state: idle 280 ms before the commit, on 73 ms after the wake, and no HELLO. Measured twice on T8140 under aura, with the same six mailbox messages each time. With the idle notice it also sets report 0xc1 on the DockChannel HID comm interface to 1, and to 2 after the on notice. So on a firmware-driven suspend rtkit-helper sends those notices and no longer refuses. If the idle notice isn't answered, the sleep is refused and nothing more is sent: macOS never sends an AP on to undo it, and one sent while the firmware may still be going idle may be what kept it from answering again. .complete looks whether a late answer came, and sends AP on only then. The idle notice goes out in .prepare, before any driver stops its own coprocessor, as macOS tells the MTP first of all: a sleep it refuses has stopped nothing yet. The on notice goes out in .resume, or in .complete for a sleep refused before this device suspended. dockchannel-hid sets the host state after the idle notice and back after the on notice, macOS's order, which the device link keeps; a firmware that doesn't answer only logs, and isn't told of the wake. Suspend-to-idle and every path without pm_suspend_via_firmware() are unchanged. With the debug host gone the DockChannel FIFO stays full, and each console character waits out the whole retry count. After one character times out, the console tries the next without waiting, until there's room again, so printing on the way into and out of a sleep doesn't hold anything up. Signed-off-by: Diljit Singh --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 42 +++++++-- drivers/soc/apple/rtkit-helper.c | 88 +++++++++++++++++++ .../tty/serial/apple_dockchannel_early_uart.c | 4 +- drivers/tty/serial/apple_dockchannel_uart.c | 4 +- tools/asahi/mtp-pm-tests.py | 8 +- 5 files changed, 138 insertions(+), 8 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index a752c224959de..68a31b88c23dc 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include "../hid-ids.h" @@ -86,6 +87,9 @@ struct dchid_init_hdr { #define CMD_REGISTER_RING 0x91 #define CMD_SEND_FIRMWARE 0x95 #define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_HOST_STATE 0xc1 +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 #define CMD_ACK_GPIO_CMD 0xa1 struct dchid_init_block_hdr { @@ -202,6 +206,8 @@ struct dockchannel_hid { bool inherited_mtp; /* Between the system sleep suspend and resume callbacks */ bool suspended; + /* The firmware was told of a sleep and has not been told of the wake */ + bool host_asleep; /* Held from probe on when the multi-touch interface uses power method 2 */ struct gpio_desc *afe_reset; @@ -447,6 +453,14 @@ static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); } +static int dchid_host_state(struct dockchannel_hid *dchid, u8 state) +{ + u8 msg[] = { CMD_HOST_STATE, state }; + int ret = dchid_comm_cmd(dchid, msg, sizeof(msg)); + + return ret < 0 ? ret : 0; +} + static int dchid_enable_interface(struct dchid_iface *iface) { u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; @@ -513,7 +527,7 @@ static int dchid_register_ring(struct dockchannel_hid *dchid) .size = DCHID_RING_SIZE, }; /* Observed in both macOS and standalone T8132 bringup before 0x91. */ - u8 prepare[] = { 0xc1, 0x02 }; + u8 prepare[] = { CMD_HOST_STATE, HOST_STATE_AWAKE }; int ret = 0; if (!dchid->use_ring) @@ -1768,10 +1782,13 @@ static void dockchannel_hid_remove(struct platform_device *pdev) } /* - * The coprocessor keeps running across suspend-to-idle: there is no known - * sleep command for it, and its RTKit session and the trackpad AFE power - * sequence are not restartable. Only mark the sleep window, in which input - * may wake the system and requests from the HID core fail fast. + * The coprocessor keeps running across suspend-to-idle, and its RTKit session + * and the trackpad AFE power sequence are not restartable. Mark the sleep + * window, in which input may wake the system and requests from the HID core + * fail fast. A firmware-driven suspend also sets the host state, after the idle + * notice the helper sends in its .prepare, and the wake sets it back after the + * helper's .resume has sent the on notice, the device link ordering the two. That + * is macOS's order. */ static int dockchannel_hid_suspend(struct device *dev) { @@ -1788,6 +1805,13 @@ static int dockchannel_hid_suspend(struct device *dev) if (ret) goto unwind; } + if (pm_suspend_via_firmware()) { + /* A firmware that doesn't answer isn't a reason to stay awake. */ + ret = dchid_host_state(dchid, HOST_STATE_SLEEP); + if (ret) + dev_warn(dev, "host state not set: %d\n", ret); + dchid->host_asleep = !ret; + } WRITE_ONCE(dchid->suspended, true); return 0; @@ -1807,6 +1831,14 @@ static int dockchannel_hid_resume(struct device *dev) struct dockchannel_hid *dchid = dev_get_drvdata(dev); int i, ret = 0; + if (dchid->host_asleep) { + dchid->host_asleep = false; + ret = dchid_host_state(dchid, HOST_STATE_AWAKE); + if (ret) + dev_warn(dev, "host state not restored: %d\n", ret); + ret = 0; + } + /* Allow feature requests again before notifying any HID driver. */ WRITE_ONCE(dchid->suspended, false); for (i = 0; i < MAX_INTERFACES; i++) { diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 07b5840da8fde..498d1413e14ef 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -39,6 +39,10 @@ struct apple_rtkit_helper { * it does not list must not replace storage the firmware still owns. */ bool strict_buffers; + /* The firmware was told the AP is going to sleep. */ + bool ap_idle; + /* It was told, gave no answer, and wasn't told otherwise. */ + bool idle_unanswered; void __iomem *asc_base; @@ -380,16 +384,97 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); } +static int apple_rtkit_helper_ap_power(struct apple_rtkit_helper *helper, bool on) +{ + int ret = apple_rtkit_ap_power(helper->rtk, on); + + if (ret) + dev_err(helper->dev, "AP power notice (%s) not answered: %d\n", + on ? "on" : "idle", ret); + /* + * Nothing is sent to undo an unanswered idle notice, which macOS never does: + * .complete looks whether a late answer came, and only then sends AP on. + */ + if (ret && !on) + helper->idle_unanswered = true; + return ret; +} + +static int apple_rtkit_helper_ap_idle(struct apple_rtkit_helper *helper) +{ + int ret; + + if (helper->ap_idle) + return 0; + ret = apple_rtkit_helper_ap_power(helper, false); + if (ret) + return ret; + helper->ap_idle = true; + dev_dbg(helper->dev, "AP idle notice answered for a sleep\n"); + return 0; +} + +static void apple_rtkit_helper_ap_on(struct apple_rtkit_helper *helper) +{ + if (!helper->ap_idle) + return; + helper->ap_idle = false; + if (!apple_rtkit_helper_ap_power(helper, true)) + dev_dbg(helper->dev, "AP on notice answered after a sleep\n"); +} + +/* + * macOS tells this coprocessor first of all. Told before any driver stops its own, + * an unanswered notice refuses a sleep that has stopped nothing yet. + */ +static int __maybe_unused apple_rtkit_helper_prepare(struct device *dev) +{ + if (!pm_suspend_via_firmware()) + return 0; + return apple_rtkit_helper_ap_idle(dev_get_drvdata(dev)); +} + +/* A sleep refused before this device's .suspend never calls its .resume. */ +static void __maybe_unused apple_rtkit_helper_complete(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + + if (helper->idle_unanswered) { + bool late = !apple_rtkit_is_running(helper->rtk); + + helper->idle_unanswered = false; + dev_info(dev, "after the unanswered idle notice: %s\n", + late ? "its answer came late, sending AP on" : + "still no answer, nothing sent"); + helper->ap_idle = late; + } + apple_rtkit_helper_ap_on(helper); +} + static int __maybe_unused apple_rtkit_helper_suspend(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + /* + * A firmware-driven suspend keeps this coprocessor running. It hears the + * AP's power state and nothing else, so an adopted session stays live. + * That is measured on T8140 and unverified on other SoCs. + */ + if (pm_suspend_via_firmware()) + return apple_rtkit_helper_ap_idle(helper); + /* Only suspend-to-idle has a verified live-session retention contract. */ if (helper->adopted && pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; return 0; } +static int __maybe_unused apple_rtkit_helper_resume(struct device *dev) +{ + apple_rtkit_helper_ap_on(dev_get_drvdata(dev)); + return 0; +} + static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); @@ -399,7 +484,10 @@ static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) } static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { + .prepare = pm_sleep_ptr(apple_rtkit_helper_prepare), + .complete = pm_sleep_ptr(apple_rtkit_helper_complete), .suspend = pm_sleep_ptr(apple_rtkit_helper_suspend), + .resume = pm_sleep_ptr(apple_rtkit_helper_resume), .freeze = pm_sleep_ptr(apple_rtkit_helper_freeze), .poweroff = pm_sleep_ptr(apple_rtkit_helper_freeze), }; diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 1048dde41d794..9842543843309 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -16,10 +16,12 @@ static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) { - unsigned int retries = DOCKCHANNEL_TX_RETRIES; + static bool no_reader; + unsigned int retries = no_reader ? 1 : DOCKCHANNEL_TX_RETRIES; while (readl(port->membase + DOCKCHANNEL_TX_FREE) == 0 && --retries) cpu_relax(); + no_reader = !retries; if (!retries) return; diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index c015d916ac432..1d890570bebf7 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -97,10 +97,12 @@ static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) static void dcuart_putc(struct uart_port *port, unsigned char c) { - unsigned int retries = TX_RETRIES; + static bool no_reader; + unsigned int retries = no_reader ? 1 : TX_RETRIES; while (readl(port->membase + DATA_TX_FREE) == 0 && --retries) cpu_relax(); + no_reader = !retries; if (!retries) return; diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py index 7d5b84d662ef0..a12aa7cef11dc 100644 --- a/tools/asahi/mtp-pm-tests.py +++ b/tools/asahi/mtp-pm-tests.py @@ -40,12 +40,18 @@ def function(source, name): struct magicmouse_sc { struct haptics *haptics; bool haptic_button_down, haptic_deep_down; }; struct hid_device { struct magicmouse_sc *data; int id; }; struct dchid_iface { struct hid_device *hid; }; -struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; +struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended, host_asleep; }; struct device { struct dockchannel_hid *data; }; static struct dockchannel_hid *transport; static unsigned suspends[16], resumes[16], key_releases; static int suspend_error_at = -1, resume_error_at = -1; static void *dev_get_drvdata(struct device *d) { return d->data; } +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 +#define dev_warn(...) ((void)0) +/* Not a firmware-driven suspend: the host state is never set here. */ +static bool pm_suspend_via_firmware(void) { return false; } +static int dchid_host_state(struct dockchannel_hid *d, int state) { (void)d; (void)state; assert(0); return 0; } static void *hid_get_drvdata(struct hid_device *d) { return d->data; } static void magicmouse_force_key(struct magicmouse_sc *m, bool down) { assert(!down); key_releases++; From e240fbe127cb24859951c330509d55c8fcb924da Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:33 -0500 Subject: [PATCH 69/73] accel/ane: H17: warn about ANE_SYS only if it really stayed on Every deep sleep logged that ANE_SYS was requested off and still on, read at the PM_SUSPEND_PREPARE notifier. That read comes too early to mean anything: the ANE's three DARTs share the domain and are still active, so genpd turns it off later, in the noirq phase, when the last of them suspends. The sleep that logged it had the domain off well before the commit. Drop the read at the put. The genpd notifier already records the real transition, so warn from that at the resume if the domain never went off, and keep its numbers at debug level otherwise. The wake also replayed about 30 lines of bring-up detail, which move to debug level, at boot as well. One line per sleep and one per wake remain. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 96 ++++++++++++++++----------------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 5ea7e8fc865b8..a9224c63326a1 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -218,8 +218,8 @@ static void ane_fw_snapshot_pmgr(struct ane_fw_backend *b) s->sys = ane_rd32(b, b->pmgr + 0x290); s->cpu = ane_rd32(b, b->pmgr + 0xc000); s->subblock = ane_rd32(b, b->pmgr + 0xc008); - dev_info(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", - s->sys, s->cpu, s->subblock); + dev_dbg(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", + s->sys, s->cpu, s->subblock); } /* Step 2: PWGATE, with ANE_SYS on and before the child power step. */ @@ -229,7 +229,7 @@ static void ane_fw_snapshot_gate(struct ane_fw_backend *b) s->gate_request = ane_rd32(b, b->gate + 0xc); s->gate_ack = ane_rd32(b, b->gate + 0x30); - dev_info(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); + dev_dbg(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); } /* Step 3: the engine and its DARTs, after the child power step. */ @@ -247,8 +247,8 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->pending = ane_rd32(b, b->engine + b->soc->irq_status); for (i = 0; i < 8; i++) s->scratch[i] = ane_rd32(b, b->engine + b->soc->scratch + i * 4); - dev_info(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", - s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); + dev_dbg(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", + s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); for (i = 0; i < ANE_FW_DARTS; i++) { s->dart[i].version = ane_rd32(b, b->dart[i] + 8); s->dart[i].protect = ane_rd32(b, b->dart[i] + 0x200); @@ -257,20 +257,20 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->dart[i].error = ane_rd32(b, b->dart[i] + 0x100); for (j = 0; j < 16; j++) s->dart[i].enabled[j] = ane_rd32(b, b->dart[i] + 0xc00 + 4 * j); - dev_info(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", - i, s->dart[i].version, s->dart[i].protect, - s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); + dev_dbg(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", + i, s->dart[i].version, s->dart[i].protect, + s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); for (j = 0; j < 16; j++) if (s->dart[i].enabled[j]) - dev_info(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, - s->dart[i].enabled[j]); + dev_dbg(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, + s->dart[i].enabled[j]); } for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { void __iomem *r = b->engine + 0x1810000 + i * 0x40; - dev_info(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", - i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), - ane_rd32(b, r + 0x20)); + dev_dbg(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", + i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), + ane_rd32(b, r + 0x20)); } } @@ -514,7 +514,6 @@ static int ane_fw_rvbar_ready(struct ane_fw_backend *b); /* Reverse of the proven power-up: sub-block, PWGATE, CPU, then ANE_SYS. */ static int ane_fw_power_down(struct ane_fw_backend *b) { - u32 sys; int ret; ane_wr32(b, ane_rd32(b, b->pmgr + 0xc008) & ~0xf, b->pmgr + 0xc008); @@ -532,20 +531,14 @@ static int ane_fw_power_down(struct ane_fw_backend *b) /* No engine or DART access from here until ANE_SYS is back on. */ b->powered = false; b->power_ref = false; - pm_runtime_put_sync(b->power); - /* - * PMGR reads are safe in any power state. ANE_SYS's actual state - * (bits 7:4) reads 0 once genpd has really turned the domain off. - */ - sys = ane_rd32(b, b->pmgr + 0x290); /* - * Here only requested: while other devices of the domain (the ANE - * DARTs) are active, genpd turns it off in the noirq phase of the - * system suspend; the genpd notifier records when it really does. + * This only drops our reference. The ANE DARTs share the domain and stay + * active, so genpd turns it off in the noirq phase of the system suspend. + * ane_fw_genpd_event() records that, and the resume reports a domain that + * stayed on. */ - dev_info(&b->pdev->dev, - "H17 ANE: powered down for system sleep; ANE_SYS requested off, %s at the put (%#x)\n", - sys & 0xf0 ? "still on" : "off", sys); + pm_runtime_put_sync(b->power); + dev_info(&b->pdev->dev, "H17 ANE: powered down for system sleep\n"); return 0; } @@ -679,10 +672,10 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) * although genpd really turned ANE_SYS off says whether they live in * an always-on block. */ - dev_info(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", - ane_fw_cold_engine_valid(&b->cold) ? "at its reset values" : - "with registers kept", b->sys_offs ? "really went off" : "stayed on", - b->cold.status, ane_rd32(b, b->engine + 0x1810000)); + dev_dbg(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", + ane_fw_cold_engine_valid(&b->cold) ? "at its reset values" : + "with registers kept", b->sys_offs ? "really went off" : "stayed on", + b->cold.status, ane_rd32(b, b->engine + 0x1810000)); if (!ane_fw_cold_engine_restartable(&b->cold)) return -EPERM; ret = ane_fw_rvbar_ready(b); @@ -719,8 +712,8 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) return ret; b->transport.allow_free = true; b->resume_ready_ns = ktime_get_ns(); - dev_info(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", - div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); + dev_dbg(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", + div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); /* Every open program back behind its handle, as the application left it. */ ane_h16_session_restart(&b->session); ret = ane_h16_session_replay(&b->session, &b->resume_replayed, &b->resume_first_ns); @@ -854,10 +847,15 @@ static void ane_fw_sleep_resume(struct ane_fw_backend *b) } ane_h16_session_sleep(&b->session, false); /* genpd's record of the domain across this sleep (boot time, so it counts s2idle). */ - dev_info(&b->pdev->dev, - "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", - b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), - b->sys_off_pmgr, b->sys_on_pmgr); + if (!b->sys_offs || (b->sys_off_pmgr & 0xf0)) + dev_warn(&b->pdev->dev, + "H17 ANE: ANE_SYS stayed on through the sleep: genpd off %u, on %u; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, b->sys_off_pmgr, b->sys_on_pmgr); + else + dev_dbg(&b->pdev->dev, + "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), + b->sys_off_pmgr, b->sys_on_pmgr); /* One line per wake; times from PM_POST_SUSPEND, which follows the thaw. */ ready = b->resume_ready_ns ? div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC) : -1; first = b->resume_first_ns ? div_u64(b->resume_first_ns - start, NSEC_PER_MSEC) : -1; @@ -991,7 +989,7 @@ static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface * ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); if (!ret) { b->next_dva += size; - dev_info(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); } return ret; } @@ -1061,8 +1059,8 @@ static void ane_fw_cold_shared_note(void *ctx, const char *event, u64 dva, u64 s { struct ane_fw_backend *b = ctx; - dev_info(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", - event, tag, size, dva); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", + event, tag, size, dva); } static const struct ane_fw_transport_ops ane_fw_ops = { @@ -1157,7 +1155,7 @@ static int ane_fw_power_up(struct ane_fw_backend *b, bool raw_boot, bool cold_en if ((ane_rd64(b, b->engine + 0x18a4010) & 3) != 3 || ane_rd64(b, b->engine + 0x18a4018)) return -EIO; - dev_info(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); + dev_dbg(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); return 0; } @@ -1367,7 +1365,7 @@ static int ane_fw_boot(struct ane_fw_backend *b) ret = ane_fw_scratch_wait(b, 7, 0x08042006); if (ret) return ret; - dev_info(&b->pdev->dev, "ANE boot: READY\n"); + dev_dbg(&b->pdev->dev, "ANE boot: READY\n"); if (ane_rd32(b, scratch) != 8 || ane_rd32(b, scratch + 4) != 0xc440 || ane_rd32(b, scratch + 12) != 0xf00000 || ane_rd32(b, scratch + 16)) return -EPROTO; @@ -1401,14 +1399,14 @@ static int ane_fw_boot(struct ane_fw_backend *b) ane_wr32(b, 0x08042006, scratch + 3 * 4); ret = ane_fw_scratch_wait(b, 3, 0); if (!ret) - dev_info(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); + dev_dbg(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); /* The firmware's table order, which the macOS channel selectors index. */ for (i = 0; !ret && i < ANE_FW_CHANNELS; i++) { const struct ane_fw_channel *c = &b->transport.channels[i]; - dev_info(&b->pdev->dev, - "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", - ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); + dev_dbg(&b->pdev->dev, + "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", + ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); } return ret; } @@ -1470,12 +1468,12 @@ static int ane_fw_startup(struct ane_fw_backend *b) return -EPROTO; if (commands[i].id == 3 && ane_fw_config_status(command, 16)) return -EPROTO; - dev_info(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); + dev_dbg(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); if (commands[i].id == 3) - dev_info(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", - get_unaligned_le32(command + 8)); + dev_dbg(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", + get_unaligned_le32(command + 8)); if (commands[i].id == 0x22) - dev_info(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); + dev_dbg(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); /* macOS order: START, then SET_SNE_PMU_BASE2, then CONFIG_GET. */ if (commands[i].id == 0 && b->soc->sleep_warm) { ret = ane_fw_pmu_base(b); From 37c379e182b93b952c93a300488970e1aa6788a1 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:33 -0500 Subject: [PATCH 70/73] arm64: dts: apple: t8140: describe what suspend-to-RAM needs - The sleep controller: the PMGR, watchdog, wake, AMCC, PMC, SPMI, DCS, DVFM, memory-controller and clock blocks it writes, from the ADT. Sizes cover only the words used. It gets the AP pin controller, whose pins it puts back after a wake, all but the four macOS leaves alone, and the PCIe host whose shared block ANS's link needs back. The wake sources left enabled are the ones a Mac wakes for: the PMU's RTC alarm, the USB host's, the SMC's two (the power button, the lid, AC, Wi-Fi and Bluetooth) and the MTP's two (the internal keyboard and trackpad). - The CPU clusters' power-gate enables in /arm-io/pmgr reg[22]: E at +0x0, P at +0x40. iBoot sets E's only. - The DCP's secure RTBuddy port: the ADT's dcp-exclave-mailbox, so far described only as a placeholder, which the DCP driver polls. Also the display power words the DCP driver writes around a warm restart, at the PMC word's address, and the GAPF slices of the DCP and DCPEXT DARTs. - The PMP DART's address filter from the ADT's dapf-instance-0, the ISP DART's from its own, the ISP DARTs' tunables from dart-tunables-instance-N, and the SEP DART's writes around a sleep, as macOS does them. - The names of the SIO's register blocks, so the sleep controller finds its ASC registers. The SIO stays disabled until the boot loader hands it over. Its domain stays on until the commit, which reads its run bit. - The shared PCIe block's registers that ANS's link needs back after a sleep, on the PCIe node, the PHY's IP block among them. The loader publishes their tunables there. APCIE_PHY_SW is APCIE_SYS_ST's parent and stays on, so only a real sleep lowers it, through the sleep controller. - The USB controller's power control block on the ATC PHY, for its hibernation, and the DRD config block on the controller, for the PIPE PHY's ready bit. - The GPU's PMP report entry, and GFX's domain as driver-owned for a sleep. Signed-off-by: Diljit Singh --- .../boot/dts/apple/t8140-peripherals.dtsi | 11 +- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 9 +- arch/arm64/boot/dts/apple/t8140.dtsi | 197 ++++++++++++++++-- 3 files changed, 198 insertions(+), 19 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi index 5198e90c72d52..9c4ef6734685a 100644 --- a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi @@ -233,12 +233,11 @@ /* ADT /arm-io/dcp-exclave-mailbox. */ dcp_secure_mbox: mailbox@452e7c000 { - compatible = "apple,t8140-secure-mailbox", "apple,secure-mailbox"; + compatible = "apple,t8140-secure-rtbuddy-proxy", + "apple,secure-rtbuddy-proxy"; reg = <0x4 0x52e7c000 0x0 0x4000>; - interrupt-parent = <&aic>; - interrupts = , - ; - iommus = <&dcp_dart 24>; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; status = "disabled"; }; @@ -481,12 +480,14 @@ status = "disabled"; }; + /* Enabled by the boot loader with its firmware, through its alias. */ sio: dma-controller@385e00000 { compatible = "apple,t8140-sio", "apple,sio"; /* Split the ASC bank around the separately owned mailbox window. */ reg = <0x3 0x85e00000 0x0 0x8000>, <0x3 0x85850000 0x0 0x4000>, <0x3 0x85e0c000 0x0 0x7c000>; + reg-names = "asc", "auxiliary", "asc-tail"; #dma-cells = <1>; dma-channels = <128>; mboxes = <&sio_mbox>; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index a2be4fc980916..c75f59faf2a36 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -61,6 +61,7 @@ * does not survive this state going off for long. */ apple,always-on; + apple,sleep-driver-owned; }; ps_venc0_sys: power-controller@240 { @@ -206,16 +207,18 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_sys_st"; - power-domains = <&ps_ans>, <&ps_apcie_st>; + power-domains = <&ps_ans>, <&ps_apcie_st>, <&ps_apcie_phy_sw>; }; + /* SYS_ST's parent, kept on: only a real sleep lowers it, through the sleep controller. */ ps_apcie_phy_sw: power-controller@3c0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_phy_sw"; - power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + power-domains = <&ps_apcie_sys_gp>; + apple,always-on; }; ps_pmp: power-controller@3d0 { @@ -851,6 +854,8 @@ #reset-cells = <0>; label = "sio_cpu"; status = "okay"; + /* The sleep reads the SIO's run bit at the commit, so it stays on until then. */ + apple,always-on; }; ps_fpwm0: power-controller@2b0 { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index dbe9bd0d4e59c..625d851be9b44 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -551,7 +551,42 @@ pmp_dart: iommu@300300000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x00300000 0x0 0xc000>; + reg = <0x3 0x00300000 0x0 0xc000>, + <0x3 0x00310000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* + * The loader's address filter, from the ADT's dapf-instance-0, + * written back after a sleep. + */ + apple,dapf-ranges = /bits/ 64 <0x220020000 0x220030000 + 0x220060000 0x220070000 + 0x2200a0000 0x2200b0000 + 0x2200e0000 0x2200f0000 + 0x220404000 0x220404070 + 0x220604000 0x220604070 + 0x220804000 0x220804070 + 0x30819d000 0x30819d00c + 0x210e40000 0x210e7ffff + 0x211e40000 0x211e7ffff + 0x40196c000 0x40197bfff + 0x480d14000 0x480e23fff + 0x210e78000 0x210ef0000 + 0x211e78000 0x211ef0000 + 0x401868000 0x40186df08 + 0x24f004000 0x24f004084 + 0x24f008000 0x24f00c140 + 0x250004000 0x250004084 + 0x250008000 0x25000c140 + 0x251004000 0x251004084 + 0x251008000 0x25100c140 + 0x22001c000 0x22001e000 + 0x22005c000 0x22005e000 + 0x22009c000 0x22009e000 + 0x2200dc000 0x2200de000 + 0x3082c8000 0x3082c8050 + 0x220104000 0x22011c000>; + apple,dapf-r20 = <0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x4 0x1 0x1 0x1 0x1 0x1 0x1 + 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1>; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -672,7 +707,9 @@ dcpext0_dart: iommu@416340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16340000 0x0 0xc000>; + reg = <0x4 0x16340000 0x0 0xc000>, + <0x3 0x0079c700 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -770,6 +807,13 @@ */ apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; apple,dma-offset = /bits/ 64 <0x0>; + /* What macOS writes around a sleep, as offset and value. */ + apple,sleep-prepare = <0x80 0xa102>, <0x704 0xffffffff>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x318 0x1>; + apple,sleep-restore-late = <0x80 0x8000>, <0x228 0x3f3f3f3f>, + <0x22c 0x1010101>; }; sep: sep@282600000 { @@ -825,6 +869,73 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + /* + * The CPU clusters' power-gate enables, in /arm-io/pmgr reg[22]: + * cluster-id 0 (E) at +0x0, 1 (P) at +0x40. iBoot sets E's only. + */ + cluster_pwrgate: power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + + /* + * Addresses from the ADT: /arm-io/pmgr reg[21], reg[22], + * reg[24] + 0x4000, reg[27], reg[28], reg[31] + 0x4000, /arm-io/wdt + * reg[0], /arm-io/pmgr reg[29] and reg[4]. The AMCC words were + * measured under aura. Sizes cover only the words used: others in + * these blocks are locked by the fabric. + */ + sleep: sleep-controller@300700000 { + compatible = "apple,t8140-sleep"; + reg = <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>, + <0x3 0x00734000 0x0 0x10>, + <0x3 0x00768000 0x0 0x40>, + <0x3 0x00780000 0x0 0xc>, + <0x3 0x082d4000 0x0 0x4>, + <0x3 0x082b0000 0x0 0x30>, + <0x3 0x082b8000 0x0 0x44>, + <0x3 0x08004000 0x0 0x48>, + <0x2 0x20025000 0x0 0x10>, + <0x2 0x20065000 0x0 0x10>, + <0x2 0x200a5000 0x0 0x10>, + <0x2 0x200e5000 0x0 0x10>, + <0x3 0x028a9c14 0x0 0x4>, + <0x2 0x10e70000 0x0 0x1000>, + <0x2 0x11e70000 0x0 0x1000>, + <0x3 0x08714000 0x0 0x84>, + <0x3 0x02800000 0x0 0x10000>, + <0x3 0x0005c000 0x0 0x4>, + <0x3 0x08280000 0x0 0x10000>, + <0x3 0x08908040 0x0 0x4>, + <0x3 0x00040034 0x0 0xc>, + <0x2 0x20400580 0x0 0x10>, + <0x2 0x20600580 0x0 0x10>, + <0x2 0x20800580 0x0 0x10>, + <0x3 0x003d0bc8 0x0 0x10>, + <0x3 0x00058000 0x0 0x10004>, + <0x3 0x00780010 0x0 0x4>; + reg-names = "ps", "pwrgate", "cpu-start", "cllt", "cluster", + "commit", "wdt", "aon-misc", "wake", + "amcc0", "amcc1", "amcc2", "amcc3", "pmc-dcp", + "dvfm-e", "dvfm-p", "spmi", "pmc-voters", "dcs", + "ps-aon", "spmi-a0", "clk-mux", "mcc0", "mcc1", "mcc2", + "ptd", "clk-perf", "cluster-dcs"; + apple,apcie = <&pcie0>; + /* All but the pins macOS leaves alone after a wake */ + apple,pinctrl = <&pinctrl_ap>; + apple,pinctrl-skip-pins = <49 50 51 134>; + /* + * NUB.SPMI0.SW3: the PMU's RTC alarm. ATC0_PMGR_USB_WAKEUP: the + * USB host's, which macOS enables at boot from the ADT and never + * masks. SMC.CPUWakeupAP and SMC.OutboxNotEmpty: the power button, + * the lid, AC, Wi-Fi and Bluetooth. MTP.DOCK.CHANNELS.AP0.IRQ and + * MTP.OUTBOX.N.EMPTY_IRQ: the internal keyboard and trackpad. + */ + apple,wake-enable = <131 70 145 165 80 99>; + }; + wdt: watchdog@3082b0000 { compatible = "apple,t8140-wdt", "apple,t8103-wdt"; reg = <0x3 0x082b0000 0x0 0x4000>; @@ -1190,9 +1301,15 @@ <0x3 0x92028000 0x0 0x8000>, <0x3 0x97020000 0x0 0x4000>, <0x3 0x97024000 0x0 0x4000>, - <0x3 0x97028000 0x0 0x4000>; + <0x3 0x97028000 0x0 0x4000>, + <0x3 0x96000000 0x0 0x1000000>, + <0x3 0x95046200 0x0 0x4000>, + <0x3 0x95044000 0x0 0x4000>, + <0x3 0x97000000 0x0 0x40000>, + <0x3 0x97040000 0x0 0x28000>; reg-names = "config", "rc", "port0", "port1", "port2", - "phy0", "phy1", "phy2"; + "phy0", "phy1", "phy2", + "axi2af", "cio3pll", "clkgen", "phy-block", "phy-ip"; interrupt-parent = <&aic>; interrupts = , , @@ -1239,8 +1356,9 @@ dwc3_0: usb@40a280000 { compatible = "apple,t8140-dwc3", "apple,t8103-dwc3"; reg = <0x4 0x0a280000 0x0 0xcd00>, - <0x4 0x0a28cd00 0x0 0x3200>; - reg-names = "dwc3-core", "dwc3-apple"; + <0x4 0x0a28cd00 0x0 0x3200>, + <0x4 0x0a200000 0x0 0x4000>; + reg-names = "dwc3-core", "dwc3-apple", "drd-cfg"; interrupt-parent = <&aic>; interrupts = ; dr_mode = "otg"; @@ -1283,9 +1401,10 @@ <0x4 0x08000000 0x0 0x8000>, <0x4 0x0aa90000 0x0 0x4000>, <0x4 0x0aa84000 0x0 0x4000>, - <0x4 0x0a300000 0x0 0x4000>; + <0x4 0x0a300000 0x0 0x4000>, + <0x4 0x0aa80000 0x0 0x4000>; reg-names = "core", "lpdptx", "axi2af", "usb2phy", - "pipehandler", "usb2phy-reg"; + "pipehandler", "usb2phy-reg", "pwrctl"; #phy-cells = <1>; #reset-cells = <0>; @@ -1308,6 +1427,7 @@ memory-region-names = "ttbs", "pagetables", "handoff"; operating-points-v2 = <&gpu_opp>; apple,clpc = <&clpc>; + apple,pmp-report-entry = <&pmp_report_gfx>; status = "disabled"; }; @@ -1363,7 +1483,9 @@ dcp_dart: iommu@412340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x12340000 0x0 0xc000>; + reg = <0x4 0x12340000 0x0 0xc000>, + <0x3 0x0079c880 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -1444,8 +1566,22 @@ #apple,bw-doorbell-cells = <2>; }; + /* + * Display power words the DCP driver writes around a warm restart. + * /arm-io/disp0 reg[5] + 0xa9c14, and /arm-io/pmgr reg[21] and reg[22]. + * The firmware owns DISP_CPU only while it runs. No driver binds this. + */ + dcp_power: display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + dcp: dcp@412e00000 { compatible = "apple,t8140-dcp", "apple,dcp"; + apple,display-power = <&dcp_power>; reg = <0x4 0x12e00000 0x0 0x4000>, <0x4 0x10000000 0x0 0xafc000>, <0x4 0x12240000 0x0 0x10000>, @@ -1455,8 +1591,8 @@ <0x4 0x12850000 0x0 0x4000>; reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3", "disp-4", "iop-vbar"; - mboxes = <&dcp_mbox>; - mbox-names = "mbox"; + mboxes = <&dcp_mbox>, <&dcp_secure_mbox>; + mbox-names = "mbox", "secure"; iommus = <&dcp_dart 23>; power-domains = <&ps_disp_cpu>; clocks = <&clk_disp0_154>, <&clk_disp0_194>; @@ -1840,6 +1976,17 @@ status = "disabled"; }; +#define T8140_ISP_DART_TUNABLES \ + apple,resume-tunable = <0x20c 0x80000521 0x80000521>, \ + <0x220 0x000f0f0f 0x000f0f0f>, \ + <0x224 0x00ffffff 0x00080808>, \ + <0x300 0x00001f31 0x00000001>, \ + <0x308 0x3ffffffc 0x10000000>, \ + <0x310 0x3ffffffc 0x3ffffffc>, \ + <0x318 0x00001f31 0x00000001>, \ + <0x320 0x3ffffffc 0x0ff00000>, \ + <0x328 0x3ffffffc 0x0ff017fc> + /* * ISP (live J700 ADT /arm-io/isp, sensor-type 0x70, H17 "ISP17a" * firmware, iBoot-loaded). "coproc" is the ISP block, the IRQ @@ -1851,7 +1998,28 @@ */ isp_dart0: iommu@3228c0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x228c0000 0x0 0xc000>; + reg = <0x3 0x228c0000 0x0 0xc000>, + <0x3 0x228d0000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* The ADT's dart-tunables-instance-0 */ + T8140_ISP_DART_TUNABLES; + /* + * The loader's address filter, from the ADT's dart-isp + * dapf-instance-0, written back after a sleep: the + * firmware's PMGR, PMP and DSID windows. + */ + apple,dapf-ranges = /bits/ 64 <0x300704000 0x300704063 + 0x300730000 0x300733fff + 0x300e3c000 0x300e3c003 + 0x31062c000 0x310648003 + 0x401660000 0x401660003 + 0x220004000 0x2200162fc + 0x220044000 0x2200562fc + 0x220084000 0x2200962fc + 0x2200c4000 0x2200d62fc + 0x220104000 0x2201162fc + 0x302824000 0x302824004 + 0x3003c0000 0x3003e4000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1872,6 +2040,8 @@ isp_dart1: iommu@3228e0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x228e0000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-1 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1892,6 +2062,8 @@ isp_dart2: iommu@322900000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x22900000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-2 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1937,6 +2109,7 @@ apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; }; +#undef T8140_ISP_DART_TUNABLES ane_dart0: iommu@401800000 { compatible = "apple,t8140-ane-dart", "apple,t8140-dart", From 048390ef8183b840d582d69d1cc7def517903a35 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:34 -0500 Subject: [PATCH 71/73] arm64: dts: apple: t8140-j700: describe what suspend-to-RAM needs - The DCP's secure port, and both display processors for the sleep controller. DCPEXT gets the display power words, and DISPEXT0's CPU domain is driver-owned for a sleep: the DCP driver raises it after a wake the way macOS does, not the power controller's resync. - The SIO's DART and mailbox, with the alias the loader finds the SIO by. - The debug port (hpm0) with its lock key, SSPS on both port controllers, and USB remote wake on both, as the root port's hub sits behind them. - The microphone array's power rail, SMC pcIO output 25 with the ADT's template (audio-hp-mic function-input_power_enable), off until the first capture as on macOS at rest. - No video encoder 0. The J700 has one encoder, the ADT's ave1 under VENC1_SYS; it has no node for the one t8140.dtsi describes at 0x285100000, and VENC0_SYS never takes its target. Its DARTs and domains deferred forever and logged a failure on every probe pass, at boot and at every wake. Leave the encoder and its DARTs disabled, as t8140.dtsi has them, and disable VENC0's six domains and its PMP request. The sleep controller still reads the VENC0_SYS word itself. Signed-off-by: Diljit Singh --- .../boot/dts/apple/t8140-j700-dcpext.dtsi | 8 +- arch/arm64/boot/dts/apple/t8140-j700.dts | 77 +++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 69d6eff9aa756..dd7580f5f13df 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -23,6 +23,8 @@ /* First mapper in the measured ADT iommu-parent list. */ iommus = <&dcpext0_dart 23>; power-domains = <&ps_dispext0_cpu>; + /* DISPEXT0's words and pmc-dcp, for the restart after a sleep */ + apple,display-power = <&dcp_power>; apple,connector-type = "USB-C"; status = "disabled"; @@ -65,7 +67,11 @@ }; }; -/* Retain the inherited firmware supply once the loader publishes it. */ +/* + * Retain the inherited firmware supply once the loader publishes it. After a sleep the DCP + * driver raises it, the way macOS does, not the PMGR resync. + */ &ps_dispext0_cpu { apple,always-on; + apple,sleep-driver-owned; }; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 5841470c3c28d..05232cd25ca4d 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -34,6 +34,8 @@ dcp = &dcp; disp0 = &display; disp0-piodma = &disp0_piodma; + /* and the SIO, with its firmware */ + sio = &sio; }; chosen { @@ -411,20 +413,53 @@ regulator-name = "jack-codec"; regulator-always-on; }; + + /* + * The mic array's rail: SMC pcIO output 25, the ADT's + * function-input_power_enable. macOS switches it around each capture. + */ + mic_array_supply: regulator-mic-array { + compatible = "regulator-fixed"; + regulator-name = "mic-array"; + gpio = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; }; #include "isp-imx558.dtsi" -&ave_dart0 { - status = "okay"; +/* + * The J700 has one video encoder, the ADT's ave1 at 0x289100000 under + * VENC1_SYS. Encoder 0, which t8140.dtsi describes, isn't there: the ADT has + * no node for it and VENC0_SYS refuses its target. Its DARTs and domains + * would wait forever on a domain that never rises. + */ +&pmp_report_ave0 { + status = "disabled"; }; -&ave_dart1 { - status = "okay"; +&ps_venc0_sys { + status = "disabled"; }; -&ave { - status = "okay"; +&ps_venc0_dma { + status = "disabled"; +}; + +&ps_venc0_pipe4 { + status = "disabled"; +}; + +&ps_venc0_pipe5 { + status = "disabled"; +}; + +&ps_venc0_me0 { + status = "disabled"; +}; + +&ps_venc0_me1 { + status = "disabled"; }; &serial0 { @@ -540,6 +575,19 @@ status = "okay"; }; +&dcp_secure_mbox { + status = "okay"; +}; + +/* The SIO itself is enabled by the loader that hands it over. */ +&sio_dart { + status = "okay"; +}; + +&sio_mbox { + status = "okay"; +}; + &dcp { /* 2408x1506 at 219 ppi; 525 nits peak from the backlight node */ panel: panel { @@ -634,6 +682,7 @@ &aop_audio { apple,chassis-name = "J700"; apple,machine-kind = "MacBook Neo"; + mic-supply = <&mic_array_supply>; }; &admac_leap_ns { @@ -692,7 +741,8 @@ }; &smc_gpio { - apple,pcio-templates = <10 0x800000 0xa0000>; + apple,pcio-templates = <10 0x800000 0xa0000>, + <25 0x0 0x190000>; }; &aop_spmi0 { @@ -1745,6 +1795,10 @@ }; }; +&sleep { + apple,debug-port = <&hpm0>; +}; + /* USB PD controllers, on the nub SPMI A0 bus */ &nub_spmi_a0 { status = "okay"; @@ -1752,6 +1806,8 @@ hpm1: usb-pd@a { compatible = "apple,sn201202x"; reg = <0xa SPMI_USID>; + apple,sleep-ssps; + apple,usb-remote-wake; interrupts = <37 IRQ_TYPE_EDGE_RISING>, <39 IRQ_TYPE_EDGE_RISING>, <43 IRQ_TYPE_EDGE_RISING>, @@ -1776,6 +1832,9 @@ hpm0: usb-pd@c { compatible = "apple,sn201202x"; reg = <0xc SPMI_USID>; + apple,debug-lock-key = "007J"; + apple,sleep-ssps; + apple,usb-remote-wake; interrupts = <11 IRQ_TYPE_EDGE_RISING>, <13 IRQ_TYPE_EDGE_RISING>, <17 IRQ_TYPE_EDGE_RISING>, @@ -1984,6 +2043,10 @@ /* The loader publishes external resources after validating their firmware. */ #include "t8140-j700-dcpext.dtsi" +&sleep { + apple,display-coprocessors = <&dcp>, <&dcpext>; +}; + &display { iommus = <&disp0_dart 0>, <&dispext0_dart 0>; }; From cd27afb6ccd0322997a586cc41f24ebbad47c0b6 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 7 Nov 2026 07:40:34 -0500 Subject: [PATCH 72/73] MAINTAINERS: list the files the T8140 sleep series adds The sleep controller, its header and its binding, and the display power words' binding, go under the T8140 section, the cluster power-gate binding and the AIC and ATC PHY headers under ARM/APPLE MACHINE SUPPORT, next to their drivers. Signed-off-by: Diljit Singh --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 78e5d506eff3a..410561cddf7d5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,6 +2093,7 @@ F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml @@ -2109,6 +2110,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml F: Documentation/devicetree/bindings/sound/apple,t6041-aop-audio.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -2130,8 +2132,10 @@ F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c +F: drivers/soc/apple/t8140-sleep.c F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h +F: include/linux/soc/apple/sleep.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/aop_t8140.rs @@ -2748,6 +2752,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -2811,7 +2816,9 @@ F: drivers/video/backlight/apple_dwi_bl.c F: drivers/watchdog/apple_wdt.c F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h +F: include/linux/irqchip/irq-apple-aic.h F: include/linux/mfd/macsmc.h +F: include/linux/phy/apple-atc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h From 1d22e0a346d5d6ec9ae2ca22dec80d3220c3215b Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 9 Nov 2026 13:21:58 -0500 Subject: [PATCH 73/73] tools/asahi: aop-bootargs-tests: take only the AOP's configs from aop.rs The test compiles struct AopHwConfig and the HW_CFG_* constants in a host harness, cutting everything from the struct to the OF table. The sleep series keeps the AOP's PM callbacks there, and they use kernel bindings, so with integration/aurora merged in, the cut no longer compiled (E0433, 13 errors). Take the struct and the constants with their doc comments, and leave the other items out. On integration's aop.rs the cut is the same as before. Signed-off-by: Diljit Singh --- tools/asahi/aop-bootargs-tests.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tools/asahi/aop-bootargs-tests.py b/tools/asahi/aop-bootargs-tests.py index d2e24ecb07eb2..fd2920a7a3353 100644 --- a/tools/asahi/aop-bootargs-tests.py +++ b/tools/asahi/aop-bootargs-tests.py @@ -90,9 +90,29 @@ def method(source): def configs(source): + """struct AopHwConfig and the HW_CFG_* constants, each with its doc comment. + + Other items between them (the sleep's PM callbacks) use kernel bindings the harness + doesn't have. + """ start = source.index('struct AopHwConfig {') end = source.index('kernel::of_device_table!', start) - return source[start:end] + out, pending, item, keep, depth = [], [], [], None, 0 + for line in source[start:end].splitlines(keepends=True): + if keep is None: + if not line.strip() or line.startswith(('//', '#[')): + pending.append(line) + continue + keep = line.startswith(('struct AopHwConfig', 'const HW_CFG_')) + item, pending, depth = pending + [line], [], 0 + else: + item.append(line) + depth += line.count('{') - line.count('}') + if depth == 0 and line.rstrip().endswith((';', '}')): + if keep: + out += item + keep = None + return ''.join(out + pending) def compile_run(rustc, p, binary, code):