diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 53948d60375dc..4908e46f58712 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -64,6 +64,19 @@ recovery remain refused. Supporting these operations requires a proven engine teardown/reset and endpoint recovery contract first. Renaming the provider does not qualify those operations. +Deep sleep keeps both functions bound and the radio powered, with its firmware +running. Their drivers stop for it, the link goes through L2 and PERST#, and the SoC +powers the engine off. On the way back the engine's whole register file must read as it +did before the first command, the state the bootstrap checks for; if anything differs, +the radios stay down. Only then does the retained state reopen for one more validated +read, on the same memory, which the retrained link needs before native ECAM. The +functions' configuration is then restored and their drivers resume, with no firmware +load. If a driver finds that its side didn't survive, the host removes both functions, +which resets the chip, and scans again, so that they probe and load their firmware as at +boot. If the link didn't come back, the radio's rail is cycled first. An abandoned sleep +never ran that power-off, so it replays nothing. This needs tunables +from a loader that publishes them, and is refused otherwise. + Suspend-to-idle is allowed only after radio admission, with no IRQ fault, idle command/FIFO checks and intact arena contents. Resume restores the configuration banks after genpd power restoration without submitting another diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index b45a737748180..e00258f50cdc8 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -78,11 +78,14 @@ properties: description: Optional host display-engine interrupt sources. mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 mbox-names: + minItems: 1 items: - const: mbox + - const: secure iommus: maxItems: 1 @@ -123,6 +126,13 @@ properties: items: pattern: '^(dp-phy|typec[0-9]+)$' + apple,display-power: + $ref: /schemas/types.yaml#/definitions/phandle + description: + A node whose ps, pwrgate and pmc-dcp registers hold the display power + words this coprocessor's driver writes around a firmware restart, such + as an apple,t8140-display-power node. + apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array items: diff --git a/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml new file mode 100644 index 0000000000000..41bd5586820f2 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,t8140-display-power.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 display power words + +maintainers: + - Diljit Singh + +description: + The words a display coprocessor's driver writes to power its domains off + and on around a restart of its firmware, as macOS does. A display + coprocessor names this node in its apple,display-power property. The words + belong to the power manager and the display block, so no driver binds it. + +properties: + compatible: + const: apple,t8140-display-power + + reg: + items: + - description: the display PMC word that counts the powered coprocessors + - description: the power manager's power state words + - description: the power manager's power gate words + + reg-names: + items: + - const: pmc-dcp + - const: ps + - const: pwrgate + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + }; diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index ad512ac8491df..b40cca489279f 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -82,6 +82,11 @@ allOf: reg: minItems: 3 maxItems: 3 + reg-names: + items: + - const: asc + - const: auxiliary + - const: asc-tail else: if: properties: @@ -137,6 +142,10 @@ properties: status: true + reg-names: + minItems: 1 + maxItems: 3 + '#dma-cells': const: 1 description: diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 202544c07b0bb..a33aaedfd90ef 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -95,6 +95,13 @@ properties: GFX supplier power cannot represent GPU work on firmware that keeps its coprocessor domain powered while the cores are idle. + apple,pmp-report-entry: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + PMP report entry that tells the PMP the GPU is in use. The driver withdraws + this request before it lowers the GPU's power state for a system sleep, and + makes it again after raising it. + apple,idle-off-delay-ms: minimum: 2 maximum: 1000 diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 38a24d0e27c36..459d491e07be9 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -79,13 +79,15 @@ properties: - description: DART registers - description: Address filter (DAPF) registers, which restrict the physical - addresses the DART lets its masters reach + addresses the DART lets its masters reach, or the slice of the + fabric filter (GAPF) that protects the power state of the + coprocessor CPU behind this DART, cleared to power that CPU down reg-names: minItems: 1 items: - const: dart - - enum: [dapf, auxiliary] + - enum: [dapf, gapf, auxiliary] interrupts: maxItems: 1 @@ -117,6 +119,54 @@ properties: may select a firmware-visible alias of the translation aperture. A consumer may further specify its own aperture in its binding. + apple,dapf-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-matrix + items: + items: + - description: First address of a range + - description: Last address of a range + description: + The address filter the loader programs into the dapf registers, one + entry per range. A sleep that reaches the hardware clears it, and the + DART writes it back before its consumer's firmware starts again. + + apple,dapf-r20: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The last word of each filter entry, one per range in apple,dapf-ranges. + 1 where absent. + + apple,sleep-prepare: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes before a consumer that gates itself is told about a + firmware-driven system suspend. Needs apple,sleep-restore. + + apple,sleep-restore: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after such a suspend, before the translations are put + back. With this, the consumer saves and restores the DART around the + suspend, as the order relative to its firmware's start matters. + + apple,sleep-restore-late: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after the translations are back and before the streams + are enabled. + apple,dma-offset: $ref: /schemas/types.yaml#/definitions/uint64 description: > @@ -158,6 +208,18 @@ properties: Tree by the bootloader and required to initialize dynamically powered DART instances. + apple,resume-tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + The same tuples for a DART whose bootloader has already applied them, + written back where a power-off lost them, before its consumer's + firmware starts again. + apple,bypass-sids: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index add726470307d..cb70ea20b8872 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -76,6 +76,14 @@ properties: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox + - description: + A coprocessor's secure RTBuddy port, the ADT's + iop,secure-rtbuddy-proxy. It is polled, with no interrupts described. + items: + - enum: + - apple,t8140-secure-rtbuddy-proxy + - const: apple,secure-rtbuddy-proxy + - description: An older ASC mailbox interface found on T2 and A11 that is also used for the NVMe coprocessor and the system management @@ -134,6 +142,16 @@ properties: maxItems: 2 allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,secure-rtbuddy-proxy + then: + required: + - interrupts + - interrupt-names - if: properties: compatible: @@ -212,8 +230,6 @@ allOf: required: - compatible - reg - - interrupts - - interrupt-names - "#mbox-cells" additionalProperties: false diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 11f3cfe160c6c..c7550daf40d80 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -175,6 +175,30 @@ properties: $ref: '#/$defs/tunable' description: Initialization tuples for the oe-fabric register region. + apple,tunable-axi2af: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's AXI bridge. + + apple,tunable-cio3pll: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's PLL. + + apple,tunable-clkgen: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's clock generator. + + apple,tunable-phy: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block. + + apple,tunable-phy-ip-pll: + $ref: '#/$defs/tunable' + description: PLL settings of the shared PHY, relative to its phy-ip registers. + + apple,tunable-phy-ip-auspma: + $ref: '#/$defs/tunable' + description: Lane settings of the shared PHY, relative to its phy-ip registers. + apple,piodma: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -420,8 +444,9 @@ allOf: properties: reg: minItems: 8 - maxItems: 8 + maxItems: 13 reg-names: + minItems: 8 items: - const: config - const: rc @@ -431,6 +456,11 @@ allOf: - const: phy0 - const: phy1 - const: phy2 + - const: axi2af + - const: cio3pll + - const: clkgen + - const: phy-block + - const: phy-ip interrupts: maxItems: 3 else: diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 5364f6dd8871b..8c84a8f3f369b 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -125,6 +125,10 @@ allOf: - description: USB2 PHY specific controls - description: USB3 PIPE interface controls - description: Secondary eUSB2 PHY register bank (T8140) + - description: + Power and hibernation control of the USB controller the PHY + serves (T8140). The PHY drives that controller's reset and + takes it through hibernation for a system sleep. reg-names: minItems: 5 items: @@ -134,6 +138,7 @@ allOf: - const: usb2phy - const: pipehandler - const: usb2phy-reg + - const: pwrctl required: - power-domains - if: diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 8281b883bd2ac..420f47e4522bc 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -23,6 +23,7 @@ allOf: properties: apple,firmware-owned: false apple,sleep-retained: false + apple,sleep-driver-owned: false apple,always-on: false apple,min-state: false apple,force-disable: false @@ -135,6 +136,13 @@ properties: system sleep. Runtime power management is unaffected. type: boolean + apple,sleep-driver-owned: + description: + The driver behind this domain lowers it before a system sleep and brings + it back afterwards, in an order its hardware needs. The power controller + leaves it alone after a wake. + type: boolean + apple,firmware-owned: description: The boot loader hands over active hardware whose power domain remains diff --git a/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml new file mode 100644 index 0000000000000..8d4e6ab0029db --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,cluster-pwrgate.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC CPU cluster power-gate enables + +maintainers: + - Diljit Singh + +description: | + One PMGR word per CPU cluster, whose top bit lets the cluster's cores power + down in WFI. A core that PMGR powers on comes up allowed to, but the boot CPU + was started by the boot ROM and only follows the word, which the boot loader + may leave clear. The words are set early in boot, in the order listed. + +properties: + compatible: + items: + - enum: + - apple,t8140-cluster-pwrgate + - const: apple,cluster-pwrgate + + reg: + minItems: 1 + maxItems: 8 + description: One word per cluster. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml new file mode 100644 index 0000000000000..adcd550f3d39a --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-sleep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 system sleep controls + +maintainers: + - Diljit Singh + +description: | + The registers a kernel running directly at EL2 on T8140 writes to enter and + return from suspend-to-RAM and to power its secondary CPUs off. They belong to + several hardware blocks, some of which other drivers also use, so this node + overlaps those drivers' register ranges and the driver does not claim them. + +properties: + compatible: + const: apple,t8140-sleep + + reg: + maxItems: 28 + + reg-names: + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: amcc2 + - const: amcc3 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: mcc2 + - const: ptd + - const: clk-perf + - const: cluster-dcs + + apple,wake-enable: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The wake source IDs left enabled across suspend-to-RAM. Every other + source is masked. + + apple,apcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PCIe host whose shared registers and tunables the storage controller's + link depends on. They are needed even with that host disabled. + + apple,pinctrl: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The pin controller whose pins a deep sleep resets. Its pins are saved + before a sleep and put back after it, before the storage controller + starts. + + apple,pinctrl-skip-pins: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Pins of that controller to leave alone across a sleep, neither saved nor + put back. + + apple,debug-port: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The USB-C port controller that carries the debug link, stopped for a sleep. + + apple,display-coprocessors: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The DCP and the external DCP, which must be off before a commit, in that + order. + +required: + - compatible + - reg + - reg-names + - apple,wake-enable + - apple,display-coprocessors + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index b023080e48c7f..7329ac57a538f 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -82,6 +82,10 @@ properties: Chassis identifier used to name the sound card, matching the apple,aop-audio convention. + mic-supply: + description: + The microphone array's power rail, switched on around each capture. + apple,machine-kind: $ref: /schemas/types.yaml#/definitions/string description: diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index 282d26ae0c6d2..9540e20101706 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -41,6 +41,24 @@ properties: wakeup-source: true + apple,sleep-ssps: + type: boolean + description: + Tell the controller the system's power state (SSPS) around a + firmware-driven suspend, S3 before it and S0 after. + + apple,debug-lock-key: + $ref: /schemas/types.yaml#/definitions/string + description: + The four-character key that unlocks the controller's debug mode, for + rearming the debug link after a sleep ended it. + + apple,usb-remote-wake: + type: boolean + description: + This controller serves the USB controller's root port, and watches its + USB2 lines for a remote wakeup during a firmware-driven system sleep. + connector: $ref: /schemas/connector/usb-connector.yaml# diff --git a/MAINTAINERS b/MAINTAINERS index b13c1507cf391..0cdac3657b043 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,6 +2093,7 @@ F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml @@ -2109,6 +2110,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml F: Documentation/devicetree/bindings/sound/apple,t6041-aop-audio.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -2130,8 +2132,10 @@ F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c +F: drivers/soc/apple/t8140-sleep.c F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h +F: include/linux/soc/apple/sleep.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/aop_t8140.rs @@ -2748,6 +2752,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -2811,7 +2816,9 @@ F: drivers/video/backlight/apple_dwi_bl.c F: drivers/watchdog/apple_wdt.c F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h +F: include/linux/irqchip/irq-apple-aic.h F: include/linux/mfd/macsmc.h +F: include/linux/phy/apple-atc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 69d6eff9aa756..dd7580f5f13df 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -23,6 +23,8 @@ /* First mapper in the measured ADT iommu-parent list. */ iommus = <&dcpext0_dart 23>; power-domains = <&ps_dispext0_cpu>; + /* DISPEXT0's words and pmc-dcp, for the restart after a sleep */ + apple,display-power = <&dcp_power>; apple,connector-type = "USB-C"; status = "disabled"; @@ -65,7 +67,11 @@ }; }; -/* Retain the inherited firmware supply once the loader publishes it. */ +/* + * Retain the inherited firmware supply once the loader publishes it. After a sleep the DCP + * driver raises it, the way macOS does, not the PMGR resync. + */ &ps_dispext0_cpu { apple,always-on; + apple,sleep-driver-owned; }; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 4b21c9b2401af..d0b405f71fa9e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -34,6 +34,8 @@ dcp = &dcp; disp0 = &display; disp0-piodma = &disp0_piodma; + /* and the SIO, with its firmware */ + sio = &sio; }; chosen { @@ -411,20 +413,53 @@ regulator-name = "jack-codec"; regulator-always-on; }; + + /* + * The mic array's rail: SMC pcIO output 25, the ADT's + * function-input_power_enable. macOS switches it around each capture. + */ + mic_array_supply: regulator-mic-array { + compatible = "regulator-fixed"; + regulator-name = "mic-array"; + gpio = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; }; #include "isp-imx558.dtsi" -&ave_dart0 { - status = "okay"; +/* + * The J700 has one video encoder, the ADT's ave1 at 0x289100000 under + * VENC1_SYS. Encoder 0, which t8140.dtsi describes, isn't there: the ADT has + * no node for it and VENC0_SYS refuses its target. Its DARTs and domains + * would wait forever on a domain that never rises. + */ +&pmp_report_ave0 { + status = "disabled"; }; -&ave_dart1 { - status = "okay"; +&ps_venc0_sys { + status = "disabled"; }; -&ave { - status = "okay"; +&ps_venc0_dma { + status = "disabled"; +}; + +&ps_venc0_pipe4 { + status = "disabled"; +}; + +&ps_venc0_pipe5 { + status = "disabled"; +}; + +&ps_venc0_me0 { + status = "disabled"; +}; + +&ps_venc0_me1 { + status = "disabled"; }; &serial0 { @@ -558,6 +593,19 @@ status = "okay"; }; +&dcp_secure_mbox { + status = "okay"; +}; + +/* The SIO itself is enabled by the loader that hands it over. */ +&sio_dart { + status = "okay"; +}; + +&sio_mbox { + status = "okay"; +}; + &dcp { /* 2408x1506 at 219 ppi; 525 nits peak from the backlight node */ panel: panel { @@ -652,6 +700,7 @@ &aop_audio { apple,chassis-name = "J700"; apple,machine-kind = "MacBook Neo"; + mic-supply = <&mic_array_supply>; }; &admac_leap_ns { @@ -739,7 +788,8 @@ }; &smc_gpio { - apple,pcio-templates = <10 0x800000 0xa0000>; + apple,pcio-templates = <10 0x800000 0xa0000>, + <25 0x0 0x190000>; }; &aop_spmi0 { @@ -1845,6 +1895,10 @@ }; }; +&sleep { + apple,debug-port = <&hpm0>; +}; + /* USB PD controllers, on the nub SPMI A0 bus */ &nub_spmi_a0 { status = "okay"; @@ -1852,6 +1906,8 @@ hpm1: usb-pd@a { compatible = "apple,sn201202x"; reg = <0xa SPMI_USID>; + apple,sleep-ssps; + apple,usb-remote-wake; interrupts = <37 IRQ_TYPE_EDGE_RISING>, <39 IRQ_TYPE_EDGE_RISING>, <43 IRQ_TYPE_EDGE_RISING>, @@ -1876,6 +1932,9 @@ hpm0: usb-pd@c { compatible = "apple,sn201202x"; reg = <0xc SPMI_USID>; + apple,debug-lock-key = "007J"; + apple,sleep-ssps; + apple,usb-remote-wake; interrupts = <11 IRQ_TYPE_EDGE_RISING>, <13 IRQ_TYPE_EDGE_RISING>, <17 IRQ_TYPE_EDGE_RISING>, @@ -2084,6 +2143,10 @@ /* The loader publishes external resources after validating their firmware. */ #include "t8140-j700-dcpext.dtsi" +&sleep { + apple,display-coprocessors = <&dcp>, <&dcpext>; +}; + &display { iommus = <&disp0_dart 0>, <&dispext0_dart 0>; }; diff --git a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi index f73a86c0f4326..65dc39b74fc24 100644 --- a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi @@ -228,12 +228,11 @@ /* ADT /arm-io/dcp-exclave-mailbox. */ dcp_secure_mbox: mailbox@452e7c000 { - compatible = "apple,t8140-secure-mailbox", "apple,secure-mailbox"; + compatible = "apple,t8140-secure-rtbuddy-proxy", + "apple,secure-rtbuddy-proxy"; reg = <0x4 0x52e7c000 0x0 0x4000>; - interrupt-parent = <&aic>; - interrupts = , - ; - iommus = <&dcp_dart 24>; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; status = "disabled"; }; @@ -475,12 +474,14 @@ status = "disabled"; }; + /* Enabled by the boot loader with its firmware, through its alias. */ sio: dma-controller@385e00000 { compatible = "apple,t8140-sio", "apple,sio"; /* Split the ASC bank around the separately owned mailbox window. */ reg = <0x3 0x85e00000 0x0 0x8000>, <0x3 0x85850000 0x0 0x4000>, <0x3 0x85e0c000 0x0 0x7c000>; + reg-names = "asc", "auxiliary", "asc-tail"; #dma-cells = <1>; dma-channels = <128>; mboxes = <&sio_mbox>; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 7123e79b3498a..eacb2caf324cc 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -61,6 +61,7 @@ * does not survive this state going off for long. */ apple,always-on; + apple,sleep-driver-owned; }; ps_venc0_sys: power-controller@240 { @@ -206,16 +207,18 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_sys_st"; - power-domains = <&ps_ans>, <&ps_apcie_st>; + power-domains = <&ps_ans>, <&ps_apcie_st>, <&ps_apcie_phy_sw>; }; + /* SYS_ST's parent, kept on: only a real sleep lowers it, through the sleep controller. */ ps_apcie_phy_sw: power-controller@3c0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_phy_sw"; - power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + power-domains = <&ps_apcie_sys_gp>; + apple,always-on; }; ps_pmp: power-controller@3d0 { @@ -866,6 +869,8 @@ #reset-cells = <0>; label = "sio_cpu"; status = "okay"; + /* The sleep reads the SIO's run bit at the commit, so it stays on until then. */ + apple,always-on; }; ps_fpwm0: power-controller@2b0 { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b781dd756a2d2..ead013f7a4927 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -551,7 +551,42 @@ pmp_dart: iommu@300300000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x00300000 0x0 0xc000>; + reg = <0x3 0x00300000 0x0 0xc000>, + <0x3 0x00310000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* + * The loader's address filter, from the ADT's dapf-instance-0, + * written back after a sleep. + */ + apple,dapf-ranges = /bits/ 64 <0x220020000 0x220030000 + 0x220060000 0x220070000 + 0x2200a0000 0x2200b0000 + 0x2200e0000 0x2200f0000 + 0x220404000 0x220404070 + 0x220604000 0x220604070 + 0x220804000 0x220804070 + 0x30819d000 0x30819d00c + 0x210e40000 0x210e7ffff + 0x211e40000 0x211e7ffff + 0x40196c000 0x40197bfff + 0x480d14000 0x480e23fff + 0x210e78000 0x210ef0000 + 0x211e78000 0x211ef0000 + 0x401868000 0x40186df08 + 0x24f004000 0x24f004084 + 0x24f008000 0x24f00c140 + 0x250004000 0x250004084 + 0x250008000 0x25000c140 + 0x251004000 0x251004084 + 0x251008000 0x25100c140 + 0x22001c000 0x22001e000 + 0x22005c000 0x22005e000 + 0x22009c000 0x22009e000 + 0x2200dc000 0x2200de000 + 0x3082c8000 0x3082c8050 + 0x220104000 0x22011c000>; + apple,dapf-r20 = <0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x4 0x1 0x1 0x1 0x1 0x1 0x1 + 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1>; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -672,7 +707,9 @@ dcpext0_dart: iommu@416340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16340000 0x0 0xc000>; + reg = <0x4 0x16340000 0x0 0xc000>, + <0x3 0x0079c700 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -770,6 +807,13 @@ */ apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; apple,dma-offset = /bits/ 64 <0x0>; + /* What macOS writes around a sleep, as offset and value. */ + apple,sleep-prepare = <0x80 0xa102>, <0x704 0xffffffff>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x318 0x1>; + apple,sleep-restore-late = <0x80 0x8000>, <0x228 0x3f3f3f3f>, + <0x22c 0x1010101>; }; sep: sep@282600000 { @@ -825,6 +869,73 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + /* + * The CPU clusters' power-gate enables, in /arm-io/pmgr reg[22]: + * cluster-id 0 (E) at +0x0, 1 (P) at +0x40. iBoot sets E's only. + */ + cluster_pwrgate: power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + + /* + * Addresses from the ADT: /arm-io/pmgr reg[21], reg[22], + * reg[24] + 0x4000, reg[27], reg[28], reg[31] + 0x4000, /arm-io/wdt + * reg[0], /arm-io/pmgr reg[29] and reg[4]. The AMCC words were + * measured under aura. Sizes cover only the words used: others in + * these blocks are locked by the fabric. + */ + sleep: sleep-controller@300700000 { + compatible = "apple,t8140-sleep"; + reg = <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>, + <0x3 0x00734000 0x0 0x10>, + <0x3 0x00768000 0x0 0x40>, + <0x3 0x00780000 0x0 0xc>, + <0x3 0x082d4000 0x0 0x4>, + <0x3 0x082b0000 0x0 0x30>, + <0x3 0x082b8000 0x0 0x44>, + <0x3 0x08004000 0x0 0x48>, + <0x2 0x20025000 0x0 0x10>, + <0x2 0x20065000 0x0 0x10>, + <0x2 0x200a5000 0x0 0x10>, + <0x2 0x200e5000 0x0 0x10>, + <0x3 0x028a9c14 0x0 0x4>, + <0x2 0x10e70000 0x0 0x1000>, + <0x2 0x11e70000 0x0 0x1000>, + <0x3 0x08714000 0x0 0x84>, + <0x3 0x02800000 0x0 0x10000>, + <0x3 0x0005c000 0x0 0x4>, + <0x3 0x08280000 0x0 0x10000>, + <0x3 0x08908040 0x0 0x4>, + <0x3 0x00040034 0x0 0xc>, + <0x2 0x20400580 0x0 0x10>, + <0x2 0x20600580 0x0 0x10>, + <0x2 0x20800580 0x0 0x10>, + <0x3 0x003d0bc8 0x0 0x10>, + <0x3 0x00058000 0x0 0x10004>, + <0x3 0x00780010 0x0 0x4>; + reg-names = "ps", "pwrgate", "cpu-start", "cllt", "cluster", + "commit", "wdt", "aon-misc", "wake", + "amcc0", "amcc1", "amcc2", "amcc3", "pmc-dcp", + "dvfm-e", "dvfm-p", "spmi", "pmc-voters", "dcs", + "ps-aon", "spmi-a0", "clk-mux", "mcc0", "mcc1", "mcc2", + "ptd", "clk-perf", "cluster-dcs"; + apple,apcie = <&pcie0>; + /* All but the pins macOS leaves alone after a wake */ + apple,pinctrl = <&pinctrl_ap>; + apple,pinctrl-skip-pins = <49 50 51 134>; + /* + * NUB.SPMI0.SW3: the PMU's RTC alarm. ATC0_PMGR_USB_WAKEUP: the + * USB host's, which macOS enables at boot from the ADT and never + * masks. SMC.CPUWakeupAP and SMC.OutboxNotEmpty: the power button, + * the lid, AC, Wi-Fi and Bluetooth. MTP.DOCK.CHANNELS.AP0.IRQ and + * MTP.OUTBOX.N.EMPTY_IRQ: the internal keyboard and trackpad. + */ + apple,wake-enable = <131 70 145 165 80 99>; + }; + wdt: watchdog@3082b0000 { compatible = "apple,t8140-wdt", "apple,t8103-wdt"; reg = <0x3 0x082b0000 0x0 0x4000>; @@ -1190,9 +1301,15 @@ <0x3 0x92028000 0x0 0x8000>, <0x3 0x97020000 0x0 0x4000>, <0x3 0x97024000 0x0 0x4000>, - <0x3 0x97028000 0x0 0x4000>; + <0x3 0x97028000 0x0 0x4000>, + <0x3 0x96000000 0x0 0x1000000>, + <0x3 0x95046200 0x0 0x4000>, + <0x3 0x95044000 0x0 0x4000>, + <0x3 0x97000000 0x0 0x40000>, + <0x3 0x97040000 0x0 0x28000>; reg-names = "config", "rc", "port0", "port1", "port2", - "phy0", "phy1", "phy2"; + "phy0", "phy1", "phy2", + "axi2af", "cio3pll", "clkgen", "phy-block", "phy-ip"; interrupt-parent = <&aic>; interrupts = , , @@ -1239,8 +1356,9 @@ dwc3_0: usb@40a280000 { compatible = "apple,t8140-dwc3", "apple,t8103-dwc3"; reg = <0x4 0x0a280000 0x0 0xcd00>, - <0x4 0x0a28cd00 0x0 0x3200>; - reg-names = "dwc3-core", "dwc3-apple"; + <0x4 0x0a28cd00 0x0 0x3200>, + <0x4 0x0a200000 0x0 0x4000>; + reg-names = "dwc3-core", "dwc3-apple", "drd-cfg"; interrupt-parent = <&aic>; interrupts = ; dr_mode = "otg"; @@ -1283,9 +1401,10 @@ <0x4 0x08000000 0x0 0x8000>, <0x4 0x0aa90000 0x0 0x4000>, <0x4 0x0aa84000 0x0 0x4000>, - <0x4 0x0a300000 0x0 0x4000>; + <0x4 0x0a300000 0x0 0x4000>, + <0x4 0x0aa80000 0x0 0x4000>; reg-names = "core", "lpdptx", "axi2af", "usb2phy", - "pipehandler", "usb2phy-reg"; + "pipehandler", "usb2phy-reg", "pwrctl"; #phy-cells = <1>; #reset-cells = <0>; @@ -1308,6 +1427,7 @@ memory-region-names = "ttbs", "pagetables", "handoff"; operating-points-v2 = <&gpu_opp>; apple,clpc = <&clpc>; + apple,pmp-report-entry = <&pmp_report_gfx>; status = "disabled"; }; @@ -1363,7 +1483,9 @@ dcp_dart: iommu@412340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x12340000 0x0 0xc000>; + reg = <0x4 0x12340000 0x0 0xc000>, + <0x3 0x0079c880 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -1444,8 +1566,22 @@ #apple,bw-doorbell-cells = <2>; }; + /* + * Display power words the DCP driver writes around a warm restart. + * /arm-io/disp0 reg[5] + 0xa9c14, and /arm-io/pmgr reg[21] and reg[22]. + * The firmware owns DISP_CPU only while it runs. No driver binds this. + */ + dcp_power: display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + dcp: dcp@412e00000 { compatible = "apple,t8140-dcp", "apple,dcp"; + apple,display-power = <&dcp_power>; reg = <0x4 0x12e00000 0x0 0x4000>, <0x4 0x10000000 0x0 0xafc000>, <0x4 0x12240000 0x0 0x10000>, @@ -1455,8 +1591,8 @@ <0x4 0x12850000 0x0 0x4000>; reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3", "disp-4", "iop-vbar"; - mboxes = <&dcp_mbox>; - mbox-names = "mbox"; + mboxes = <&dcp_mbox>, <&dcp_secure_mbox>; + mbox-names = "mbox", "secure"; iommus = <&dcp_dart 23>; power-domains = <&ps_disp_cpu>; clocks = <&clk_disp0_154>, <&clk_disp0_194>; @@ -1840,6 +1976,17 @@ status = "disabled"; }; +#define T8140_ISP_DART_TUNABLES \ + apple,resume-tunable = <0x20c 0x80000521 0x80000521>, \ + <0x220 0x000f0f0f 0x000f0f0f>, \ + <0x224 0x00ffffff 0x00080808>, \ + <0x300 0x00001f31 0x00000001>, \ + <0x308 0x3ffffffc 0x10000000>, \ + <0x310 0x3ffffffc 0x3ffffffc>, \ + <0x318 0x00001f31 0x00000001>, \ + <0x320 0x3ffffffc 0x0ff00000>, \ + <0x328 0x3ffffffc 0x0ff017fc> + /* * ISP (live J700 ADT /arm-io/isp, sensor-type 0x70, H17 "ISP17a" * firmware, iBoot-loaded). "coproc" is the ISP block, the IRQ @@ -1851,7 +1998,28 @@ */ isp_dart0: iommu@3228c0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x228c0000 0x0 0xc000>; + reg = <0x3 0x228c0000 0x0 0xc000>, + <0x3 0x228d0000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* The ADT's dart-tunables-instance-0 */ + T8140_ISP_DART_TUNABLES; + /* + * The loader's address filter, from the ADT's dart-isp + * dapf-instance-0, written back after a sleep: the + * firmware's PMGR, PMP and DSID windows. + */ + apple,dapf-ranges = /bits/ 64 <0x300704000 0x300704063 + 0x300730000 0x300733fff + 0x300e3c000 0x300e3c003 + 0x31062c000 0x310648003 + 0x401660000 0x401660003 + 0x220004000 0x2200162fc + 0x220044000 0x2200562fc + 0x220084000 0x2200962fc + 0x2200c4000 0x2200d62fc + 0x220104000 0x2201162fc + 0x302824000 0x302824004 + 0x3003c0000 0x3003e4000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1872,6 +2040,8 @@ isp_dart1: iommu@3228e0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x228e0000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-1 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1892,6 +2062,8 @@ isp_dart2: iommu@322900000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x22900000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-2 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1937,6 +2109,7 @@ apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; }; +#undef T8140_ISP_DART_TUNABLES ane_dart0: iommu@401800000 { compatible = "apple,t8140-ane-dart", "apple,t8140-dart", diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 126f60eee6c74..fd318f9071928 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -160,3 +160,4 @@ CONFIG_EDAC_APPLE_SOC=m # unless the bootloader supplies a window (/chosen/aurora,ctxr-exec-window) or # the ctxr_exec_window= command-line option is given. CONFIG_ARM64_CTXR_EXECMEM=y +CONFIG_APPLE_T8140_SLEEP=y diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index a444c8915e886..4ed50e0dc0b5c 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -7,6 +7,7 @@ #include #include +#include /** * struct cpu_operations - Callback operations for hotplugging CPUs. @@ -46,6 +47,58 @@ struct cpu_operations { #endif }; +/** + * struct spin_table_power_ops - Platform control of parking spin-table CPUs + * (smp_spin_table.c). + * + * @cpu_stop: Tell the hardware the calling CPU is stopping. The CPU keeps + * running, and stays coherent, until @cpu_off. + * @cpu_off: Power off the calling CPU, which must not return. + * @cpu_on: Start a CPU that @cpu_off powered off, at its reset vector. + * + * Each takes the CPU's MPIDR hardware ID. + */ +struct spin_table_power_ops { + void (*cpu_stop)(u64 mpidr); + void (*cpu_off)(u64 mpidr); + int (*cpu_on)(u64 mpidr); +}; + +#ifdef CONFIG_HOTPLUG_CPU +int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +int spin_table_hold_parked(void); +void spin_table_unhold_parked(void); +int spin_table_stop_parked(void); +void spin_table_release_stopped(void); +void spin_table_stop_finished(void); +bool spin_table_cpus_stuck(void); +#else +static inline int spin_table_hold_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_unhold_parked(void) {} + +static inline int spin_table_stop_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_release_stopped(void) {} +static inline void spin_table_stop_finished(void) {} + +static inline int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return -EOPNOTSUPP; +} + +static inline bool spin_table_cpus_stuck(void) +{ + return false; +} +#endif + int __init init_cpu_ops(int cpu); extern const struct cpu_operations *get_cpu_ops(int cpu); diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 0a421a88b3012..65982bd36a11f 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -1089,6 +1089,7 @@ static inline bool cpu_has_lpa2(void) } void __init init_cpucap_indirect_list_impdef(void); +void arm64_impdef_cpu_reset_restore(void); void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index e133011f64b5b..7e88e4b6ca501 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -15,6 +15,9 @@ #include extern const struct cpu_operations smp_spin_table_ops; +#ifdef CONFIG_HOTPLUG_CPU +extern const struct cpu_operations smp_rvbar_spin_table_ops; +#endif #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL extern const struct cpu_operations acpi_parking_protocol_ops; #endif @@ -24,6 +27,9 @@ static const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; static const struct cpu_operations *const dt_supported_cpu_ops[] __initconst = { &smp_spin_table_ops, +#ifdef CONFIG_HOTPLUG_CPU + &smp_rvbar_spin_table_ops, +#endif &cpu_psci_ops, NULL, }; @@ -109,6 +115,19 @@ int __init init_cpu_ops(int cpu) return -EOPNOTSUPP; } +#ifdef CONFIG_HOTPLUG_CPU + /* + * A boot object that sends a reset CPU back through its release address + * says so with /chosen/aurora,s2r-mailbox. Its spin-table CPUs can then + * leave the kernel, through the parking spin table. + */ + if (cpu_ops[cpu] == &smp_spin_table_ops && + of_property_present(of_chosen, "aurora,s2r-mailbox")) { + cpu_ops[cpu] = &smp_rvbar_spin_table_ops; + pr_info_once("spin-table CPUs can park: the boot object brings them back\n"); + } +#endif + return 0; } diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c index 4078f95146c32..f08e210bcb813 100644 --- a/arch/arm64/kernel/cpufeature_impdef.c +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -363,6 +364,23 @@ static const struct arm64_cpu_capabilities arm64_impdef_features[] = { {}, }; +/* + * Restore what these features keep per CPU, on a CPU that was reset without + * going through CPU hotplug: its enable callbacks do not run again, and a + * context switch only writes ACTLR when the thread's value changes. + */ +void arm64_impdef_cpu_reset_restore(void) +{ +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + write_sysreg(current->thread.actlr, actlr_el1); +#endif +#ifdef CONFIG_ARM64_APPLE_AMX + if (cpus_have_cap(ARM64_HAS_AMX_APPLE)) + cpu_disable_el0_amx(NULL); +#endif +} + void __init init_cpucap_indirect_list_impdef(void) { init_cpucap_indirect_list_from_array(arm64_impdef_features); diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 9c4a407222ea1..4fb4e7bb1e39e 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -64,7 +64,8 @@ int machine_kexec_prepare(struct kimage *kimage) return -EOPNOTSUPP; } - if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { + if (kimage->type != KEXEC_TYPE_CRASH && + (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck())) { pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); return -EBUSY; } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 49029eace3ad9..2d8819416a489 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include extern void secondary_holding_pen(void); @@ -125,3 +127,288 @@ const struct cpu_operations smp_spin_table_ops = { .cpu_prepare = smp_spin_table_cpu_prepare, .cpu_boot = smp_spin_table_cpu_boot, }; + +/* + * A spin table whose CPUs can leave the kernel. + * + * Plain spin-table has no way out of the kernel, so disabling the non-boot + * CPUs for system sleep fails with -EOPNOTSUPP. Some boot monitors leave a + * CPU's reset vector locked to their own image and, after a reset, send the + * CPU back through the same release address. Such a monitor publishes + * /chosen/aurora,s2r-mailbox, and its "spin-table" CPUs then get these ops + * (cpu_ops.c), so the device tree keeps the method every kernel knows. One can + * also name "aurora,rvbar-spin-table" as a CPU's method. + * + * A CPU that really powers off can stop the rest of the system on such a chip, + * so a CPU leaving the kernel parks here instead: still coherent, interrupts + * masked. From there it goes back through secondary_holding_pen, or, when the + * platform suspends the system, it stops and takes its last WFI after the commit. + * It is then started again at its reset vector. Stopping and starting a CPU is + * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug + * fails cleanly, as on plain spin-table. + * + * A CPU left awake at the commit can hang the machine, and the commit can't be + * undone. So each parked CPU first answers a hold, while the suspend can still + * refuse, and only a held CPU stops. + */ +#ifdef CONFIG_HOTPLUG_CPU +static const struct spin_table_power_ops *spin_table_power __read_mostly; +static struct cpumask spin_table_cpus_parked; +static struct cpumask spin_table_cpus_held; +static struct cpumask spin_table_cpus_unpark; +static struct cpumask spin_table_cpus_off; +static bool spin_table_hold_requested; +static bool spin_table_stop_requested; +static bool spin_table_stop_released; + +#define SPIN_TABLE_WAIT_US 100000 + +int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; +} + +/* A refused suspend: the held CPUs go back to the park. */ +void spin_table_unhold_parked(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + dsb(ish); + sev(); +} + +/* + * Every CPU not online is held or off. One that is neither, parked or not (a start that + * failed, a CPU never started), may be awake anywhere. + */ +static bool spin_table_all_held(bool report) +{ + unsigned int cpu; + bool all = true; + + for_each_present_cpu(cpu) { + if (cpu_online(cpu) || cpumask_test_cpu(cpu, &spin_table_cpus_held) || + cpumask_test_cpu(cpu, &spin_table_cpus_off)) + continue; + if (!report) + return false; + pr_err("CPU%u did not answer the hold for a suspend\n", cpu); + all = false; + } + + return all; +} + +/* + * Before a suspend's last refusal, interrupts off, other CPUs offline: every parked CPU + * answers and waits, still parked, for spin_table_stop_parked() or + * spin_table_unhold_parked(). Nothing is written to the hardware. Returns -ETIMEDOUT, + * with the hold undone, if any CPU not online is neither held nor off. + */ +int spin_table_hold_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_hold_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (spin_table_all_held(false)) + return 0; + udelay(1); + } + + if (spin_table_all_held(true)) + return 0; + spin_table_unhold_parked(); + + return -ETIMEDOUT; +} + +/* + * At the commit, after spin_table_hold_parked(): the held CPUs stop and wait for + * spin_table_release_stopped(). Returns -ETIMEDOUT if any didn't stop. The request + * stays up until the wake, so a late one still stops. + */ +int spin_table_stop_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_stop_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (cpumask_empty(&spin_table_cpus_parked)) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +/* After the commit: the stopped CPUs take their last WFI. */ +void spin_table_release_stopped(void) +{ + WRITE_ONCE(spin_table_stop_released, true); + dsb(ish); + sev(); +} + +/* + * Back from a suspend, before starting any CPU. Every CPU lost its power with the + * SoC, so one that never saw the stop is started at its reset vector too. + */ +void spin_table_stop_finished(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + WRITE_ONCE(spin_table_stop_requested, false); + WRITE_ONCE(spin_table_stop_released, false); + cpumask_or(&spin_table_cpus_off, &spin_table_cpus_off, &spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_held); + cpumask_clear(&spin_table_cpus_unpark); +} + +static int smp_rvbar_spin_table_cpu_boot(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + int ret; + + /* The CPU may run straight into the pen: release it before it starts. */ + write_pen_release(mpidr); + + if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { + ret = spin_table_power->cpu_on(mpidr); + if (ret) { + cpumask_set_cpu(cpu, &spin_table_cpus_off); + return ret; + } + } else if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) { + cpumask_set_cpu(cpu, &spin_table_cpus_unpark); + } + + dsb(ish); + sev(); + + return 0; +} + +static bool smp_rvbar_spin_table_cpu_can_disable(unsigned int cpu) +{ + return cpu != 0; +} + +static int smp_rvbar_spin_table_cpu_disable(unsigned int cpu) +{ + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + return -EOPNOTSUPP; + + return READ_ONCE(spin_table_power) ? 0 : -EOPNOTSUPP; +} + +#ifdef CONFIG_KEXEC_CORE +static void smp_rvbar_spin_table_restart(void) +{ + typeof(cpu_soft_restart) *restart; + + cpu_install_idmap(); + restart = (void *)__pa_symbol(cpu_soft_restart); + restart(0, __pa_symbol(secondary_holding_pen), 0, 0, 0); +} +#else +/* Never reached: cpu_die() doesn't leave its loop without KEXEC_CORE. */ +static void smp_rvbar_spin_table_restart(void) +{ +} +#endif + +/* Held until the commit's stop, or until a refusal sends it back to the park. */ +static void smp_rvbar_spin_table_hold(unsigned int cpu, u64 mpidr) +{ + cpumask_set_cpu(cpu, &spin_table_cpus_held); + + while (READ_ONCE(spin_table_hold_requested)) { + if (READ_ONCE(spin_table_stop_requested)) { + spin_table_power->cpu_stop(mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_off); + cpumask_clear_cpu(cpu, &spin_table_cpus_held); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + while (!READ_ONCE(spin_table_stop_released)) + wfe(); + spin_table_power->cpu_off(mpidr); + } + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_held); +} + +static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + + /* The early-boot and crash paths get here without hotplug support. */ + if (!READ_ONCE(spin_table_power) || !IS_ENABLED(CONFIG_KEXEC_CORE)) { + while (1) + wfe(); + } + + cpumask_set_cpu(cpu, &spin_table_cpus_parked); + + while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { + if (READ_ONCE(spin_table_hold_requested)) + smp_rvbar_spin_table_hold(cpu, mpidr); + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_unpark); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + smp_rvbar_spin_table_restart(); +} + +static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) +{ + int us; + + if (!READ_ONCE(spin_table_power)) + return 0; + + for (us = 0; us < SPIN_TABLE_WAIT_US; us += 10) { + if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) + return 0; + udelay(10); + } + + return -ETIMEDOUT; +} + +const struct cpu_operations smp_rvbar_spin_table_ops = { + .name = "aurora,rvbar-spin-table", + .cpu_init = smp_spin_table_cpu_init, + .cpu_prepare = smp_spin_table_cpu_prepare, + .cpu_boot = smp_rvbar_spin_table_cpu_boot, + .cpu_can_disable = smp_rvbar_spin_table_cpu_can_disable, + .cpu_disable = smp_rvbar_spin_table_cpu_disable, + .cpu_die = smp_rvbar_spin_table_cpu_die, + .cpu_kill = smp_rvbar_spin_table_cpu_kill, +}; + +/* + * Parking CPUs leave the kernel only through a platform's power ops. Until one + * registers, kexec has to take them as stuck, as on plain spin-table. + */ +bool spin_table_cpus_stuck(void) +{ + unsigned int cpu; + + if (READ_ONCE(spin_table_power)) + return false; + for_each_possible_cpu(cpu) + if (get_cpu_ops(cpu) == &smp_rvbar_spin_table_ops) + return true; + return false; +} +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index c41724a40b756..2a69c79b4ed91 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,13 @@ void notrace __cpu_suspend_exit(void) /* Restore additional feature-specific configuration */ ptrauth_suspend_exit(); + + /* + * A firmware-driven system suspend really reset this CPU. Nothing else on + * this path restores implementation-defined state. + */ + if (pm_resume_via_firmware()) + arm64_impdef_cpu_reset_restore(); } /* diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 5ea7e8fc865b8..a9224c63326a1 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -218,8 +218,8 @@ static void ane_fw_snapshot_pmgr(struct ane_fw_backend *b) s->sys = ane_rd32(b, b->pmgr + 0x290); s->cpu = ane_rd32(b, b->pmgr + 0xc000); s->subblock = ane_rd32(b, b->pmgr + 0xc008); - dev_info(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", - s->sys, s->cpu, s->subblock); + dev_dbg(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", + s->sys, s->cpu, s->subblock); } /* Step 2: PWGATE, with ANE_SYS on and before the child power step. */ @@ -229,7 +229,7 @@ static void ane_fw_snapshot_gate(struct ane_fw_backend *b) s->gate_request = ane_rd32(b, b->gate + 0xc); s->gate_ack = ane_rd32(b, b->gate + 0x30); - dev_info(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); + dev_dbg(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); } /* Step 3: the engine and its DARTs, after the child power step. */ @@ -247,8 +247,8 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->pending = ane_rd32(b, b->engine + b->soc->irq_status); for (i = 0; i < 8; i++) s->scratch[i] = ane_rd32(b, b->engine + b->soc->scratch + i * 4); - dev_info(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", - s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); + dev_dbg(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", + s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); for (i = 0; i < ANE_FW_DARTS; i++) { s->dart[i].version = ane_rd32(b, b->dart[i] + 8); s->dart[i].protect = ane_rd32(b, b->dart[i] + 0x200); @@ -257,20 +257,20 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->dart[i].error = ane_rd32(b, b->dart[i] + 0x100); for (j = 0; j < 16; j++) s->dart[i].enabled[j] = ane_rd32(b, b->dart[i] + 0xc00 + 4 * j); - dev_info(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", - i, s->dart[i].version, s->dart[i].protect, - s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); + dev_dbg(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", + i, s->dart[i].version, s->dart[i].protect, + s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); for (j = 0; j < 16; j++) if (s->dart[i].enabled[j]) - dev_info(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, - s->dart[i].enabled[j]); + dev_dbg(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, + s->dart[i].enabled[j]); } for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { void __iomem *r = b->engine + 0x1810000 + i * 0x40; - dev_info(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", - i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), - ane_rd32(b, r + 0x20)); + dev_dbg(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", + i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), + ane_rd32(b, r + 0x20)); } } @@ -514,7 +514,6 @@ static int ane_fw_rvbar_ready(struct ane_fw_backend *b); /* Reverse of the proven power-up: sub-block, PWGATE, CPU, then ANE_SYS. */ static int ane_fw_power_down(struct ane_fw_backend *b) { - u32 sys; int ret; ane_wr32(b, ane_rd32(b, b->pmgr + 0xc008) & ~0xf, b->pmgr + 0xc008); @@ -532,20 +531,14 @@ static int ane_fw_power_down(struct ane_fw_backend *b) /* No engine or DART access from here until ANE_SYS is back on. */ b->powered = false; b->power_ref = false; - pm_runtime_put_sync(b->power); - /* - * PMGR reads are safe in any power state. ANE_SYS's actual state - * (bits 7:4) reads 0 once genpd has really turned the domain off. - */ - sys = ane_rd32(b, b->pmgr + 0x290); /* - * Here only requested: while other devices of the domain (the ANE - * DARTs) are active, genpd turns it off in the noirq phase of the - * system suspend; the genpd notifier records when it really does. + * This only drops our reference. The ANE DARTs share the domain and stay + * active, so genpd turns it off in the noirq phase of the system suspend. + * ane_fw_genpd_event() records that, and the resume reports a domain that + * stayed on. */ - dev_info(&b->pdev->dev, - "H17 ANE: powered down for system sleep; ANE_SYS requested off, %s at the put (%#x)\n", - sys & 0xf0 ? "still on" : "off", sys); + pm_runtime_put_sync(b->power); + dev_info(&b->pdev->dev, "H17 ANE: powered down for system sleep\n"); return 0; } @@ -679,10 +672,10 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) * although genpd really turned ANE_SYS off says whether they live in * an always-on block. */ - dev_info(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", - ane_fw_cold_engine_valid(&b->cold) ? "at its reset values" : - "with registers kept", b->sys_offs ? "really went off" : "stayed on", - b->cold.status, ane_rd32(b, b->engine + 0x1810000)); + dev_dbg(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", + ane_fw_cold_engine_valid(&b->cold) ? "at its reset values" : + "with registers kept", b->sys_offs ? "really went off" : "stayed on", + b->cold.status, ane_rd32(b, b->engine + 0x1810000)); if (!ane_fw_cold_engine_restartable(&b->cold)) return -EPERM; ret = ane_fw_rvbar_ready(b); @@ -719,8 +712,8 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) return ret; b->transport.allow_free = true; b->resume_ready_ns = ktime_get_ns(); - dev_info(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", - div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); + dev_dbg(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", + div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); /* Every open program back behind its handle, as the application left it. */ ane_h16_session_restart(&b->session); ret = ane_h16_session_replay(&b->session, &b->resume_replayed, &b->resume_first_ns); @@ -854,10 +847,15 @@ static void ane_fw_sleep_resume(struct ane_fw_backend *b) } ane_h16_session_sleep(&b->session, false); /* genpd's record of the domain across this sleep (boot time, so it counts s2idle). */ - dev_info(&b->pdev->dev, - "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", - b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), - b->sys_off_pmgr, b->sys_on_pmgr); + if (!b->sys_offs || (b->sys_off_pmgr & 0xf0)) + dev_warn(&b->pdev->dev, + "H17 ANE: ANE_SYS stayed on through the sleep: genpd off %u, on %u; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, b->sys_off_pmgr, b->sys_on_pmgr); + else + dev_dbg(&b->pdev->dev, + "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), + b->sys_off_pmgr, b->sys_on_pmgr); /* One line per wake; times from PM_POST_SUSPEND, which follows the thaw. */ ready = b->resume_ready_ns ? div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC) : -1; first = b->resume_first_ns ? div_u64(b->resume_first_ns - start, NSEC_PER_MSEC) : -1; @@ -991,7 +989,7 @@ static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface * ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); if (!ret) { b->next_dva += size; - dev_info(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); } return ret; } @@ -1061,8 +1059,8 @@ static void ane_fw_cold_shared_note(void *ctx, const char *event, u64 dva, u64 s { struct ane_fw_backend *b = ctx; - dev_info(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", - event, tag, size, dva); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", + event, tag, size, dva); } static const struct ane_fw_transport_ops ane_fw_ops = { @@ -1157,7 +1155,7 @@ static int ane_fw_power_up(struct ane_fw_backend *b, bool raw_boot, bool cold_en if ((ane_rd64(b, b->engine + 0x18a4010) & 3) != 3 || ane_rd64(b, b->engine + 0x18a4018)) return -EIO; - dev_info(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); + dev_dbg(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); return 0; } @@ -1367,7 +1365,7 @@ static int ane_fw_boot(struct ane_fw_backend *b) ret = ane_fw_scratch_wait(b, 7, 0x08042006); if (ret) return ret; - dev_info(&b->pdev->dev, "ANE boot: READY\n"); + dev_dbg(&b->pdev->dev, "ANE boot: READY\n"); if (ane_rd32(b, scratch) != 8 || ane_rd32(b, scratch + 4) != 0xc440 || ane_rd32(b, scratch + 12) != 0xf00000 || ane_rd32(b, scratch + 16)) return -EPROTO; @@ -1401,14 +1399,14 @@ static int ane_fw_boot(struct ane_fw_backend *b) ane_wr32(b, 0x08042006, scratch + 3 * 4); ret = ane_fw_scratch_wait(b, 3, 0); if (!ret) - dev_info(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); + dev_dbg(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); /* The firmware's table order, which the macOS channel selectors index. */ for (i = 0; !ret && i < ANE_FW_CHANNELS; i++) { const struct ane_fw_channel *c = &b->transport.channels[i]; - dev_info(&b->pdev->dev, - "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", - ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); + dev_dbg(&b->pdev->dev, + "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", + ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); } return ret; } @@ -1470,12 +1468,12 @@ static int ane_fw_startup(struct ane_fw_backend *b) return -EPROTO; if (commands[i].id == 3 && ane_fw_config_status(command, 16)) return -EPROTO; - dev_info(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); + dev_dbg(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); if (commands[i].id == 3) - dev_info(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", - get_unaligned_le32(command + 8)); + dev_dbg(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", + get_unaligned_le32(command + 8)); if (commands[i].id == 0x22) - dev_info(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); + dev_dbg(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); /* macOS order: START, then SET_SNE_PMU_BASE2, then CONFIG_GET. */ if (commands[i].id == 0 && b->soc->sleep_warm) { ret = ane_fw_pmu_base(b); diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 72910ef976f8e..d60458d69c590 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -65,6 +65,8 @@ /* Vendor command telling the controller that the host goes to sleep. */ #define BT7932_OP_HOST_SLEEP 0xfc6f #define BT7932_HOST_SLEEP_PLEN 7 +/* How long a wake reason may take to arrive once the transport is back. */ +#define BT7932_WAKE_REASON_MS 200 #define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" #define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" @@ -153,6 +155,8 @@ struct bt7932 { bool tx_blocked; /* nothing is sent until resume */ bool irq_held; /* IRQ off until the PCI state is restored */ bool pci_saved; /* saved PCI state not yet restored */ + bool wake_quiesced; /* in host sleep with HCI set up to wake, deep sleep */ + bool wake_expected; /* the next wake report is what woke the host */ enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; }; @@ -300,6 +304,70 @@ static bool bt7932_wake_report(struct bt7932 *bt, const u8 *data, size_t len) } } +/* + * Names a wake report (see bt7932_wake_event()) for the log, and points @addr + * at the remote address it carries, or NULL. + */ +static const char *bt7932_wake_name(const u8 *data, size_t len, const u8 **addr) +{ + /* Where the address starts: after status and handle, or LE report fields. */ + size_t offset = 0; + const char *name = "event"; + + switch (data[0]) { + case HCI_EV_CONN_REQUEST: + name = "connection request"; + offset = HCI_EVENT_HDR_SIZE; + break; + case HCI_EV_CONN_COMPLETE: + name = "connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_SYNC_CONN_COMPLETE: + name = "synchronous connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_HARDWARE_ERROR: + name = "hardware error"; + break; + case HCI_EV_LE_META: + if (len < HCI_EVENT_HDR_SIZE + 1) + break; + /* Subevent, report count, event type (two bytes if extended), address type. */ + switch (data[HCI_EVENT_HDR_SIZE]) { + case HCI_EV_LE_ADVERTISING_REPORT: + name = "LE advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_DIRECT_ADV_REPORT: + name = "LE directed advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_EXT_ADV_REPORT: + name = "LE extended advertising report"; + offset = HCI_EVENT_HDR_SIZE + 5; + break; + } + break; + } + *addr = offset && len >= offset + sizeof(bdaddr_t) ? data + offset : NULL; + return name; +} + +/* Caller owns lock and expects the wake reason. */ +static void bt7932_log_wake(struct bt7932 *bt, const u8 *data, size_t len) +{ + const u8 *addr; + const char *name = bt7932_wake_name(data, len, &addr); + + if (addr) + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s from %pMR\n", name, addr); + else + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s\n", name); + bt->wake_expected = false; + wake_up_all(&bt->wake_wait); +} + static int bt7932_validate_ptx(const u8 *data, size_t size) { static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; @@ -520,11 +588,13 @@ static int bt7932_sleep_locked(struct bt7932 *bt) } if (value > 1) return -EPROTO; - /* A previous host-sleep notification must not cancel control 3. */ - if (bt->pm_state == BT7932_QUIESCING) + /* + * A previous host-sleep notification must not cancel control 3. Once + * quiesced, only the host ends the host sleep, so leave the rest to + * bt7932_restore_transport(), as macOS ignores it then too. + */ + if (bt->pm_state == BT7932_QUIESCING || bt->pm_state == BT7932_QUIESCED) return 0; - if (bt->pm_state == BT7932_QUIESCED) - return -EPROTO; if (bt->pm_state == BT7932_RESTORING) { if (value) { if (!bt->waking) { @@ -725,8 +795,11 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) * remember that it arrived. */ if (geo->packet == HCI_EVENT_PKT && bt->wake_tracking && - bt7932_wake_report(bt, descriptor + 16, length)) + bt7932_wake_report(bt, descriptor + 16, length)) { + if (bt->wake_expected) + bt7932_log_wake(bt, descriptor + 16, length); pm_wakeup_dev_event(&bt->pdev->dev, 0, true); + } hci_recv_frame(bt->hdev, skb); } slot = (slot + 1) & BT7932_MASK; @@ -743,9 +816,9 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) return 0; } -static irqreturn_t bt7932_irq_thread(int irq, void *data) +/* Takes what the firmware published: sleep notifications, completions, packets. */ +static void bt7932_service(struct bt7932 *bt) { - struct bt7932 *bt = data; u16 producer; int channel, ret; @@ -785,6 +858,11 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) queue_work(bt->tx_wq, &bt->tx_work); out: mutex_unlock(&bt->lock); +} + +static irqreturn_t bt7932_irq_thread(int irq, void *data) +{ + bt7932_service(data); return IRQ_HANDLED; } @@ -1125,7 +1203,7 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) * Follows the wakeup setting of the PCI function, see bt7932_probe(). * Only while it is on does hci_suspend_sync() leave page scan and LE passive * scan set up for the devices flagged for remote wakeup, and only then does - * bt7932_suspend() keep the transport running with its interrupt armed. + * bt7932_suspend() leave the controller able to wake the system. */ static bool bt7932_wakeup(struct hci_dev *hdev) { @@ -1487,29 +1565,33 @@ static bool bt7932_hci_pm(struct bt7932 *bt) } /* - * Suspend-to-idle can keep the transport running so that the controller can - * wake the system: with wakeup enabled on the PCI function, the controller - * up and HCI suspended by this driver. + * The controller can wake the system with wakeup enabled on the PCI function, the controller + * up and HCI suspended by this driver. In suspend-to-idle the transport stays up and the wake + * is its MSI. A deep sleep takes the link down: the controller goes into host sleep, which is + * the quiesce, and wakes the system through the SMC, as macOS does. */ static bool bt7932_may_wake(struct bt7932 *bt) { - return device_may_wakeup(&bt->pdev->dev) && pm_suspend_no_platform() && - bt7932_hci_pm(bt) && test_bit(HCI_UP, &bt->hdev->flags); + return device_may_wakeup(&bt->pdev->dev) && bt7932_hci_pm(bt) && + test_bit(HCI_UP, &bt->hdev->flags); } static void bt7932_track_wake(struct bt7932 *bt, bool on) { mutex_lock(&bt->lock); bt->wake_tracking = on; + if (!on) + bt->wake_expected = false; mutex_unlock(&bt->lock); } /* - * Before system sleep, the host tells the controller that it is going to - * sleep with a vendor command whose parameters are all ones; nothing is sent - * when the host is back. It is answered by a vendor event, if at all, so - * place it on the command ring directly rather than through the HCI command - * flow, and wait until the controller has taken it. + * For suspend-to-idle with the transport kept running: a vendor command whose + * parameters are all ones. macOS uses the same bytes, but only as a request to + * its own transport, which turns them into the host sleep of a deep sleep (see + * bt7932_suspend()) and never sends them on. It is answered by a vendor event, + * if at all, so place it on the command ring directly rather than through the + * HCI command flow, and wait until the controller has taken it. * * Returns -EAGAIN if nothing was published (the controller could not be * reached without a fault, or the ring was full), -EBUSY if the command was @@ -1583,6 +1665,7 @@ static int bt7932_arm_wakeup(struct bt7932 *bt) ret = bt->fault ? -EIO : 0; if (!ret) { bt->wake_armed = true; + bt->wake_expected = true; /* A failed wake handshake would fault the transport. */ bt->tx_blocked = true; } @@ -1607,7 +1690,6 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) mutex_lock(&bt->lock); bt->wake_armed = false; bt->tx_blocked = false; - bt->wake_tracking = false; bt->pci_saved = false; mutex_unlock(&bt->lock); wake_up_all(&bt->wake_wait); @@ -1617,6 +1699,28 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) } } +/* + * After a sleep set up for Bluetooth wake, the controller delivers what woke + * the host once the transport runs again. Take it while HCI is still + * suspended, so that the core records the wake reason too, and log it. + * Ends wake tracking. + */ +static void bt7932_take_wake_reason(struct bt7932 *bt) +{ + /* The interrupt thread leaves the rings alone until the restore ends. */ + bt7932_service(bt); + wait_event_timeout(bt->wake_wait, + !READ_ONCE(bt->wake_expected) || READ_ONCE(bt->fault), + msecs_to_jiffies(BT7932_WAKE_REASON_MS)); + mutex_lock(&bt->lock); + if (bt->wake_expected && !bt->fault) + dev_dbg(&bt->pdev->dev, "Bluetooth did not wake the host\n"); + mutex_unlock(&bt->lock); + bt7932_track_wake(bt, false); + /* The core takes a wake reason only until hci_resume_dev(). */ + flush_work(&bt->hdev->rx_work); +} + /* * With wakeup enabled, the transport stays up through suspend-to-idle * instead of being quiesced: @@ -1637,6 +1741,12 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) * The PCIe link and host bridge stay up in suspend-to-idle anyway, as the * Wi-Fi function on the same link is never stopped, and ASPM stays off on * this link as the transport was admitted. + * + * A deep sleep with wakeup enabled quiesces the transport after HCI has set + * up its wake scans. The quiesce (sleep control 3, notification 3) is the + * host sleep that macOS enters at S2R. The controller keeps scanning and + * wakes the system through the SMC. bt7932_resume() ends the host sleep + * with control 0, as macOS does, and then takes the wake reason. */ static int bt7932_suspend(struct device *dev) { @@ -1644,8 +1754,9 @@ static int bt7932_suspend(struct device *dev) bool wake; int ret; + /* A transport that's down has nothing to stop, and it can't cost a deep sleep. */ if (READ_ONCE(bt->fault)) - return -EIO; + return pm_suspend_no_platform() ? -EIO : 0; if (!bt->registered) return 0; wake = bt7932_may_wake(bt); @@ -1656,16 +1767,19 @@ static int bt7932_suspend(struct device *dev) ret = hci_suspend_dev(bt->hdev); if (ret) { bt7932_track_wake(bt, false); - return ret; + if (pm_suspend_no_platform()) + return ret; + dev_warn(dev, "HCI did not suspend: %d\n", ret); } } - if (wake) { + if (wake && pm_suspend_no_platform()) { ret = bt7932_arm_wakeup(bt); if (!ret) { - dev_dbg(dev, "system sleep with wakeup armed\n"); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } bt7932_track_wake(bt, false); + wake = false; if (ret == -EBUSY) { /* * The host sleep command may still take effect, so the @@ -1676,8 +1790,7 @@ static int bt7932_suspend(struct device *dev) hci_resume_dev(bt->hdev); bt7932_reset(bt->hdev); return ret; - } - if (ret != -EAGAIN) { + } else if (ret != -EAGAIN) { dev_err(dev, "transport failed while arming wakeup\n"); hci_resume_dev(bt->hdev); return ret; @@ -1686,9 +1799,28 @@ static int bt7932_suspend(struct device *dev) ret = bt7932_quiesce(bt); if (!ret) { dev_dbg(dev, "system sleep quiesce acknowledged\n"); + if (!wake) + return 0; + /* hci_suspend_dev() may have stopped short of the wake setup. */ + if (!bt->hdev->suspended || + bt->hdev->suspend_state != BT_SUSPEND_CONFIGURE_WAKE) { + bt7932_track_wake(bt, false); + dev_warn(dev, "no Bluetooth wake for this sleep\n"); + return 0; + } + mutex_lock(&bt->lock); + bt->wake_quiesced = true; + bt->wake_expected = true; + mutex_unlock(&bt->lock); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } + if (wake) + bt7932_track_wake(bt, false); dev_warn(dev, "system sleep quiesce failed: %d\n", ret); + /* The wake finds out what's left. A deep sleep takes the power regardless. */ + if (!pm_suspend_no_platform()) + return 0; /* Undo our request before the PM core rolls back the rest of the bus. */ if (!READ_ONCE(bt->fault)) bt7932_restore_transport(bt); @@ -1732,23 +1864,43 @@ static int bt7932_resume_noirq(struct device *dev) static int bt7932_resume(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + bool wake_quiesced = bt->wake_quiesced; int ret; + bt->wake_quiesced = false; + /* A deep sleep retrains the link. With the chip gone, nothing here may touch it. */ + if (!pci_device_is_present(bt->pdev)) { + bt7932_track_wake(bt, false); + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "radio gone after the sleep"); + mutex_unlock(&bt->lock); + apple_piodma_radio_recover(); + return -ENODEV; + } + if (bt->wake_armed) { bt7932_disarm_wakeup(bt); + bt7932_take_wake_reason(bt); if (READ_ONCE(bt->fault)) return -EIO; return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } - if (READ_ONCE(bt->fault)) + if (READ_ONCE(bt->fault)) { + bt7932_track_wake(bt, false); return -EIO; + } if (!bt->registered) return 0; if (READ_ONCE(bt->pm_state) != BT7932_RUNNING) { ret = bt7932_restore_transport(bt); - if (ret) + if (ret) { + bt7932_track_wake(bt, false); + apple_piodma_radio_recover(); return ret; + } } + if (wake_quiesced) + bt7932_take_wake_reason(bt); dev_dbg(dev, "transport restored after system sleep\n"); return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } @@ -1778,9 +1930,15 @@ static void bt7932_remove(struct pci_dev *pdev) u16 command = U16_MAX; bool quiesced, drained; - /* Closing HCI may send Reset: keep the transport and IRQ alive for it. */ - if (bt->registered) + /* + * The HCI core stops its command and event work before it closes a device it + * unregisters, so a Reset on that close goes unsent and waits out its 2 s timeout. + * The quiesce below stops the firmware. Close still runs with the transport alive. + */ + if (bt->registered) { + hci_clear_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_unregister_dev(bt->hdev); + } quiesced = bt->ipc_ready && !READ_ONCE(bt->fault) && !bt7932_quiesce(bt); mutex_lock(&bt->lock); WRITE_ONCE(bt->stopping, true); diff --git a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c index 61e93ce1f31db..8a57f9f30ca1a 100644 --- a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c +++ b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c @@ -65,11 +65,57 @@ static void bt7932_wake_event_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, bt7932_wake_event(error, sizeof(error)), BT7932_WAKE_ERROR); } +/* The wake log names the event and the remote address it carries. */ +static void bt7932_wake_name_bredr_test(struct kunit *test) +{ + static const u8 request[] = { HCI_EV_CONN_REQUEST, 10, + 1, 2, 3, 4, 5, 6, 0x40, 0x25, 0x00, 0x01 }; + static const u8 complete[] = { HCI_EV_CONN_COMPLETE, 11, 0x00, 0x01, 0x00, + 1, 2, 3, 4, 5, 6, 0x01, 0x00 }; + static const u8 sync_short[] = { HCI_EV_SYNC_CONN_COMPLETE, 1, 0x00 }; + static const u8 error[] = { HCI_EV_HARDWARE_ERROR, 1, 0x00 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(request, sizeof(request), &addr), + "connection request"); + KUNIT_EXPECT_PTR_EQ(test, addr, &request[2]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(complete, sizeof(complete), &addr), + "connection complete"); + KUNIT_EXPECT_PTR_EQ(test, addr, &complete[5]); + bt7932_wake_name(sync_short, sizeof(sync_short), &addr); + KUNIT_EXPECT_NULL(test, addr); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(error, sizeof(error), &addr), "hardware error"); + KUNIT_EXPECT_NULL(test, addr); +} + +static void bt7932_wake_name_le_test(struct kunit *test) +{ + /* Subevent, one report, event type, address type, address, data length, RSSI. */ + static const u8 adv[] = { HCI_EV_LE_META, 12, HCI_EV_LE_ADVERTISING_REPORT, 1, + 0x00, 0x01, 1, 2, 3, 4, 5, 6, 0x00, 0xc0 }; + /* The extended report's event type takes two bytes. */ + static const u8 ext[] = { HCI_EV_LE_META, 11, HCI_EV_LE_EXT_ADV_REPORT, 1, + 0x13, 0x00, 0x01, 1, 2, 3, 4, 5, 6 }; + static const u8 adv_short[] = { HCI_EV_LE_META, 2, HCI_EV_LE_ADVERTISING_REPORT, 1 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(adv, sizeof(adv), &addr), + "LE advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &adv[6]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(ext, sizeof(ext), &addr), + "LE extended advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &ext[7]); + bt7932_wake_name(adv_short, sizeof(adv_short), &addr); + KUNIT_EXPECT_NULL(test, addr); +} + static struct kunit_case bt7932_wake_test_cases[] = { KUNIT_CASE(bt7932_wake_event_bredr_test), KUNIT_CASE(bt7932_wake_event_le_test), KUNIT_CASE(bt7932_wake_event_replies_test), KUNIT_CASE(bt7932_wake_event_error_test), + KUNIT_CASE(bt7932_wake_name_bredr_test), + KUNIT_CASE(bt7932_wake_name_le_test), {} }; diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index fb87ae6ae137a..0603a40d865b6 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -856,6 +857,37 @@ static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) return cpufreq_generic_suspend(policy); } +/* + * A firmware-driven sleep powers the cluster down, and it comes back in a performance state + * of the hardware's choosing, not the one asked for at suspend. Take what it reads as the + * expected state again, before the governor starts, rather than stopping every request as if + * someone else owned it. + */ +static int apple_soc_cpufreq_resume(struct cpufreq_policy *policy) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned long flags; + u64 reg; + + if (!priv->info->verify_transition || !pm_resume_via_firmware()) + return 0; + + raw_spin_lock_irqsave(&priv->lock, flags); + reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + if (!(reg & APPLE_DVFS_CMD_BUSY)) { + unsigned int now = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); + + if (now != priv->expected_pstate) + dev_info(priv->cpu_dev, "DVFS state %u after the sleep, %u before\n", + now, priv->expected_pstate); + priv->expected_pstate = now; + priv->transition_failed = false; + } + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { @@ -1571,6 +1603,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, .suspend = apple_soc_cpufreq_suspend, + .resume = apple_soc_cpufreq_resume, }; static int __init apple_soc_cpufreq_module_init(void) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index b7336f97ee9fc..ee660cc38992f 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dmaengine.h" @@ -139,6 +140,12 @@ struct admac_chan { struct list_head issued; struct list_head to_free; + + /* The client's last configuration, set up again after a system sleep. */ + struct dma_slave_config cfg; + bool cfg_valid; + /* A system sleep took the channel's registers; set it up before a transfer. */ + bool lost; }; struct admac_sram { @@ -629,11 +636,30 @@ static void admac_reset_rings(struct admac_chan *adchan) writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, adchan->no)); } +static int admac_device_config(struct dma_chan *chan, struct dma_slave_config *config); + +/* Registers can't be reached before a client powers the fabric, so this waits for it. */ +static void admac_restore_chan(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + + if (!adchan->lost) + return; + adchan->lost = false; + + if (ad->split_channels) + writel_relaxed(0x18080001, ad->base + REG_CTL); + writel_relaxed(adchan->carveout, ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); + if (adchan->cfg_valid) + admac_device_config(&adchan->chan, &adchan->cfg); +} + static void admac_start_current_tx(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; int ch = adchan->no; + admac_restore_chan(adchan); admac_reset_rings(adchan); writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, ch)); if (ad->split_channels) { @@ -686,10 +712,15 @@ static int admac_resume(struct dma_chan *chan) int ret = 0; spin_lock_irqsave(&adchan->lock, flags); - if (adchan->current_tx && adchan->current_tx->error) + if (adchan->current_tx && adchan->current_tx->error) { ret = -EIO; - else + } else if (adchan->lost && adchan->current_tx) { + /* The rings went with the registers: start from the top. */ + adchan->nperiod_acks = 0; + admac_start_current_tx(adchan); + } else { admac_start_chan(adchan); + } spin_unlock_irqrestore(&adchan->lock, flags); return ret; @@ -1052,6 +1083,9 @@ static int admac_device_config(struct dma_chan *chan, | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), ad->base + REG_CHAN_FIFOCTL(ad, adchan->no)); + adchan->cfg = *config; + adchan->cfg_valid = true; + return 0; } @@ -1081,8 +1115,36 @@ static int admac_runtime_resume(struct device *dev) return 0; } +/* + * A system sleep that reached the hardware takes the registers of a channel a + * client still holds, and its client won't configure it again: only a new + * hw_params does that. + */ +static int __maybe_unused admac_resume_early(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + unsigned long flags; + int i; + + if (!pm_resume_via_firmware()) + return 0; + + mutex_lock(&ad->cache_alloc_lock); + ad->ctl_programmed = false; + mutex_unlock(&ad->cache_alloc_lock); + + for (i = 0; i < ad->nchannels; i++) { + spin_lock_irqsave(&ad->channels[i].lock, flags); + ad->channels[i].lost = true; + spin_unlock_irqrestore(&ad->channels[i].lock, flags); + } + + return 0; +} + /* Allocated channels must stay powered, including paused cyclic transfers. */ static const struct dev_pm_ops admac_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, admac_resume_early) RUNTIME_PM_OPS(admac_runtime_suspend, admac_runtime_resume, NULL) }; diff --git a/drivers/dma/apple-sio-boot-test.c b/drivers/dma/apple-sio-boot-test.c index 3d8f93d11d8bb..5994518f619b0 100644 --- a/drivers/dma/apple-sio-boot-test.c +++ b/drivers/dma/apple-sio-boot-test.c @@ -187,12 +187,36 @@ static void sio_boot_dma_arrays_test(struct kunit *test) } } +/* Only a channel with nothing on it may pass through a firmware restart. */ +static void sio_sleep_busy_test(struct kunit *test) +{ + struct sio_chan *chan = kunit_kzalloc(test, sizeof(*chan), GFP_KERNEL); + struct sio_tx *tx = kunit_kzalloc(test, sizeof(*tx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, chan); + KUNIT_ASSERT_NOT_NULL(test, tx); + spin_lock_init(&chan->vc.lock); + + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); + chan->current_tx = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->current_tx = NULL; + chan->idle = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->idle = NULL; + chan->ring = true; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->ring = false; + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); +} + static struct kunit_case sio_boot_test_cases[] = { KUNIT_CASE(sio_boot_order_test), KUNIT_CASE(sio_boot_optional_tables_test), KUNIT_CASE(sio_boot_bad_params_test), KUNIT_CASE(sio_boot_control_ack_failure_test), KUNIT_CASE(sio_boot_dma_arrays_test), + KUNIT_CASE(sio_sleep_busy_test), {} }; diff --git a/drivers/dma/apple-sio-probe-test.c b/drivers/dma/apple-sio-probe-test.c index c486c5072f07c..9b9c5a0774320 100644 --- a/drivers/dma/apple-sio-probe-test.c +++ b/drivers/dma/apple-sio-probe-test.c @@ -72,6 +72,8 @@ static void sio_match_data_test(struct kunit *test) KUNIT_EXPECT_EQ(test, hw->fifo_threshold, 0x800U); KUNIT_EXPECT_FALSE(test, hw->clear_period_desc); KUNIT_EXPECT_FALSE(test, hw->check_boot_tables); + KUNIT_EXPECT_FALSE(test, hw->firmware_only); + KUNIT_EXPECT_FALSE(test, sio_hw_t8140.firmware_only); } static struct kunit_case sio_probe_test_cases[] = { diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index e1a8ec545b8c6..1ecbec8298cdf 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include "dmaengine.h" @@ -160,6 +162,11 @@ struct sio_hw_data { u32 cfg_word4; /* configuration word 4 */ bool cfg_six_words; /* the configuration ends in a zero sixth word */ bool clear_period_desc; + /* + * Only run the firmware, so that it can be stopped for a system sleep. Nothing + * uses its DMA here, and its service parameters haven't been traced. + */ + bool firmware_only; /* DMA arrays owned by the driver, if not protocol 9's */ const struct sio_dma_arrays *arrays; int (*check_boot_tables)(struct device *dev); @@ -173,6 +180,8 @@ struct sio_data { struct dma_device dma; struct device *dev; struct apple_rtkit *rtk; + /* The firmware is stopped for a system sleep. */ + bool asleep; u32 protocol; void *shmem; dma_addr_t shmem_iova; @@ -2014,6 +2023,11 @@ static int sio_probe(struct platform_device *pdev) if (err) return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + if (hw->firmware_only) { + dev_info(&pdev->dev, "firmware running, no DMA service\n"); + goto rpm_put; + } + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); if (err) return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); @@ -2057,6 +2071,9 @@ static void sio_remove(struct platform_device *pdev) { struct sio_data *sio = platform_get_drvdata(pdev); + if (sio->hw->firmware_only) + return; + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&sio->dma); } @@ -2074,7 +2091,93 @@ static __maybe_unused int sio_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); +/* A channel with a transfer can't be carried through a firmware restart. */ +static bool sio_chan_busy(struct sio_chan *siochan) +{ + unsigned long flags; + bool busy; + + spin_lock_irqsave(&siochan->vc.lock, flags); + busy = siochan->current_tx || siochan->idle || siochan->ring; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return busy; +} + +static __maybe_unused int sio_suspend_late(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int i, ret; + + if (!pm_suspend_via_firmware() || sio->asleep) + return 0; + + for (i = 0; i < sio->nchannels; i++) { + if (sio_chan_busy(&sio->channels[i])) { + dev_err(dev, "channel %d is busy, refusing the sleep\n", i); + return -EBUSY; + } + } + + ret = apple_rtkit_sleep_stop(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) { + dev_err(dev, "firmware did not stop for sleep: %d\n", ret); + if (apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) + dev_err(dev, "firmware did not come back\n"); + return ret; + } + sio->asleep = true; + + return 0; +} + +static __maybe_unused int sio_resume_early(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int ret; + + if (!sio->asleep) + return 0; + /* + * A refused sleep: an idled firmware doesn't wake with its power kept, and SIO_CPU shares + * SIO's switch, so it can't be power-cycled alone. The sleep engine's own PMGR reset of it + * restarts it (S); if that fails it stays stopped until the next real sleep. + */ + if (!pm_resume_via_firmware()) { + if (!apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) { + sio->asleep = false; + return 0; + } + ret = apple_sleep_engine_reset(dev); + if (!ret) + ret = apple_rtkit_power_cycled_return(sio->rtk, + sio->base + REG_CPU_CONTROL); + if (ret) { + dev_warn(dev, "sleep refused: firmware not back after a reset (%d), stopped until a real sleep\n", + ret); + return 0; + } + sio->asleep = false; + dev_info(dev, "sleep refused: firmware back after a reset\n"); + return 0; + } + sio->asleep = false; + + /* Endpoints and parameters stay with the firmware: nothing is sent again. */ + ret = apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + else + dev_dbg(dev, "firmware back from sleep\n"); + + return 0; +} + +static const struct dev_pm_ops sio_pm_ops = { + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + LATE_SYSTEM_SLEEP_PM_OPS(sio_suspend_late, sio_resume_early) + RUNTIME_PM_OPS(sio_suspend, sio_resume, NULL) +}; static struct platform_driver apple_sio_driver = { .driver = { diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b35ed086b4710..a11f021a71d56 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -47,6 +47,7 @@ #include #include #include +#include #include "apple_boot_display.h" #include "apple_resume.h" @@ -67,8 +68,15 @@ struct apple_drm_private { * the screen, the DRM device stays unregistered and its DCPs running. */ bool aborted; + /* This suspend found every firmware already stopped. */ + bool s2r_skipped; + /* A dark wake left the outputs as the suspend did, for the next resume. */ + bool dark_held; + struct notifier_block pm_nb; }; +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data); + /* Component readiness is later than platform probe/bind. */ static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; @@ -1749,6 +1757,9 @@ static int apple_drm_init(struct device *dev) goto err_unbind; } + apple->pm_nb.notifier_call = apple_drm_pm_notify; + register_pm_notifier(&apple->pm_nb); + return 0; err_unlock: @@ -1772,6 +1783,8 @@ static void apple_drm_uninit(struct device *dev) if (apple->aborted) return; + unregister_pm_notifier(&apple->pm_nb); + /* Before anything of the DRM device is torn down. */ t8132_ext_drm_fini(&apple->drm); apple_drm_quiesce_connectors(&apple->drm); @@ -1890,12 +1903,38 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); #ifdef CONFIG_PM_SLEEP +/* + * Left stopped by a suspend that never reached the commit. Nothing can commit to + * them, and a refused sleep has to be able to try again. + */ +static bool apple_drm_firmware_stopped(struct apple_drm_private *apple) +{ + struct drm_crtc *crtc; + bool stopped = false; + + drm_for_each_crtc(crtc, &apple->drm) { + if (t8132_ext_crtc(crtc) || + !dcp_s2r_firmware_stopped(to_apple_crtc(crtc)->dcp)) + return false; + stopped = true; + } + + return stopped; +} + static int apple_platform_suspend(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple && !apple->aborted) - return drm_mode_config_helper_suspend(&apple->drm); + /* Still suspended from before a dark wake. */ + if (apple && apple->dark_held) + return 0; + + if (apple && !apple->aborted) { + apple->s2r_skipped = apple_drm_firmware_stopped(apple); + if (!apple->s2r_skipped) + return drm_mode_config_helper_suspend(&apple->drm); + } return 0; } @@ -1917,29 +1956,71 @@ static bool apple_drm_hold_resume_events(struct apple_drm_private *apple, u32 cr return held; } +static void apple_drm_resume_outputs(struct apple_drm_private *apple) +{ + u32 dropped = 0; + + apple->dark_held = false; + if (apple->drm.mode_config.suspend_state) + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped && apple_drm_hold_resume_events(apple, dropped)) + dropped = 0; + drm_mode_config_helper_resume(&apple->drm); + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - u32 dropped = 0; - if (apple && !apple->aborted) { - if (apple->drm.mode_config.suspend_state) - dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); - /* Have userspace set the mode of a CRTC resumed disabled again. */ - if (dropped && apple_drm_hold_resume_events(apple, dropped)) - dropped = 0; - drm_mode_config_helper_resume(&apple->drm); - if (dropped) - drm_kms_helper_hotplug_event(&apple->drm); + if (!apple || apple->aborted || apple->s2r_skipped) + return 0; + + /* A dark wake goes back to sleep with user space frozen: nothing is lit for it. */ + if (pm_dark_wake()) { + apple->dark_held = true; + dev_info(dev, "dark wake: outputs stay off\n"); + return 0; } + apple_drm_resume_outputs(apple); return 0; } +/* The suspend after a dark wake failed before it suspended this device. */ +static void apple_platform_complete(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple && apple->dark_held && !pm_dark_wake()) + apple_drm_resume_outputs(apple); +} + +/* It failed before ->prepare() reached this device, so ->complete() didn't run either. */ +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_drm_private *apple = container_of(nb, struct apple_drm_private, pm_nb); + + if (action == PM_POST_SUSPEND && apple->dark_held) { + dev_warn(apple->drm.dev, "dark wake over without a resume: outputs back now\n"); + apple_drm_resume_outputs(apple); + } + + return NOTIFY_DONE; +} + static const struct dev_pm_ops apple_platform_pm_ops = { .suspend = apple_platform_suspend, .resume = apple_platform_resume, + .complete = apple_platform_complete, }; +#else +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + return NOTIFY_DONE; +} #endif static struct platform_driver apple_platform_driver = { diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index de8fc565a11dd..92ff5ce615c85 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1692,7 +1693,23 @@ static __maybe_unused int dcpaud_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(dcpaud_pm_ops, dcpaud_suspend, dcpaud_resume, NULL); +/* The SIO channel and the sink's link both end in a sleep that reaches the hardware. */ +static __maybe_unused int dcpaud_system_suspend(struct device *dev) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + + if (pm_suspend_via_firmware() && dcpaud->substream && dcpaud->substream->ref_count) { + dev_err(dev, "a stream is open, refusing the sleep\n"); + return -EBUSY; + } + + return pm_runtime_force_suspend(dev); +} + +static const struct dev_pm_ops dcpaud_pm_ops = { + SYSTEM_SLEEP_PM_OPS(dcpaud_system_suspend, pm_runtime_force_resume) + RUNTIME_PM_OPS(dcpaud_suspend, dcpaud_resume, NULL) +}; static const struct of_device_id dcpaud_of_match[] = { { .compatible = "apple,dpaudio" }, diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 3dc3128b600ae..9f29f28b21f87 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -592,6 +592,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, scoped_guard(mutex, &dcp->tb_lock) { route->deactivate_xbar_done = false; route->deactivate_error = 0; + route->sleep_parked = false; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -639,7 +640,10 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return route->deactivate_error; if (!route->deactivate_xbar_done) { dcp_typec_route_phy_off(route); - if (!route->tunnel || route->xbar_up) + /* Parked for a sleep, its output is already down and not held. */ + if (route->sleep_parked) + ret = 0; + else if (!route->tunnel || route->xbar_up) ret = mux_control_deselect(route->active_xbar ?: route->xbar); else ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); @@ -653,6 +657,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) ret = 0; /* Deselect always unlocks: never consume that selection twice. */ route->deactivate_xbar_done = true; + route->sleep_parked = false; route->xbar_up = false; if (ret) { /* Idle is unproven; retain the owner and the real error. */ @@ -4289,6 +4294,53 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) return dcp_fabric_run_probe(!!routes, dcp_register_typec_routes_present, &ctx); } +/* + * A sleep that reaches its commit powers the crossbar off. Take the output of a direct + * route down while it still has power, as a released display's is, and select it again + * after the wake. The route stays, and with it the display on the port. + */ +void dcp_typec_route_park(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route || route->tunnel || !route->selected) + return; + guard(mutex)(&dcp->tb_lock); + if (route->deactivate_xbar_done || route->sleep_parked) + return; + /* Deselect always unlocks. */ + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + route->sleep_parked = true; + dev_dbg(dcp->dev, "display route parked for the sleep: crossbar output down: %d\n", ret); +} + +void dcp_typec_route_unpark(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route) + return; + guard(mutex)(&dcp->tb_lock); + if (!route->sleep_parked) + return; + route->sleep_parked = false; + ret = mux_control_try_select(route->active_xbar ?: route->xbar, route->mux_index); + if (ret) { + /* Nothing is held, so the route's release has nothing to give back. */ + route->deactivate_xbar_done = true; + dev_err(dcp->dev, "display route not back after the sleep: crossbar output: %d\n", + ret); + return; + } + dev_dbg(dcp->dev, "display route back after the sleep: crossbar output up\n"); +} + /* * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY * (see dcp_typec_route_deactivate()). Point it at the HDMI output again diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index f68b91e4d18f8..f46211ec3a8e1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -51,6 +51,8 @@ struct apple_dcp_typec_route { /* Lower mux release is consumed even when idle programming fails. */ bool deactivate_xbar_done; int deactivate_error; + /* Its crossbar output is down for a sleep, and not held. */ + bool sleep_parked; }; int dcp_typec_phy_on(struct apple_dcp *dcp); @@ -111,6 +113,8 @@ bool dcp_is_t8132_ext(const struct apple_dcp *dcp); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); +void dcp_typec_route_park(struct apple_dcp *dcp); +void dcp_typec_route_unpark(struct apple_dcp *dcp); void dcp_typec_routes_disable(struct apple_dcp *dcp); void dcp_typec_routes_disable_dev(struct device *dev); void dcp_typec_retrain_work(struct work_struct *work); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index adb31d035c9be..2f135e7419e95 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -362,6 +363,8 @@ const struct drm_edid *dcp_panel_edid_snapshot(struct apple_dcp *dcp, unsigned int count, u64 *generation); bool dcp_panel_metadata_current(struct apple_dcp *dcp, u64 generation); +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on); + struct apple_dcp { /* Generation-specific wire/transaction state, sharing the common KMS owner. */ struct iomfb_m4 *v26_6; @@ -638,6 +641,38 @@ struct apple_dcp { /* Messages from the coprocessor are held in the mailbox for sleep. */ bool rx_held; + /* T8140 suspend-to-RAM stops the firmware, see dcp_s2r_suspend(). */ + bool s2r_active; + bool s2r_stopped; + /* DPMS off stopped the firmware and lowered the domains; DPMS on brings them back. */ + bool dpms_powered_off; + /* Refused its idle with the display off: the firmware still runs, the display is dark. */ + bool s2r_refused_off; + /* The firmware's secure mailbox, where it has one. */ + struct apple_rtkit *secure_rtk; + bool secure_answered; + bool secure_polling; + /* Polled every jiffy until then, see dcp_secure_poll(). */ + unsigned long secure_fast_until; + int secure_map_ret; + s64 secure_map_us; + struct delayed_work secure_poll_wq; + /* Stopped and not restarted, which outlives a suspend that gets refused. */ + bool s2r_fw_down; + /* DCPEXT kept down through a dark wake, see dcp_s2r_dark_finish(). */ + bool s2r_dark_held; + struct notifier_block s2r_pm_nb; + /* This driver raised the display domains, and nothing has lowered them since. */ + bool s2r_domains_up; + /* Counted in pmc-dcp, the number of powered DCPs */ + bool pmc_up; + bool s2r_power_on; + bool s2r_power_oob; + u8 s2r_step; + struct completion s2r_dart_done; + struct device *s2r_disp_dart; + struct device *s2r_dcp_dart; + /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 20da2fa4df2ee..65655dfcdc1df 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -18,13 +18,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -144,6 +147,135 @@ static void dcp_release_context(void *data) kfree(dcp); } +#define DCP_SECURE_EP 0x30 +#define DCP_SECURE_TIMEOUT_MS 200 + +/* + * The firmware's secure mailbox. No other software answers it under Linux, so the + * driver does, as macOS's secure side does: 3 is answered with 2, and 4 needs nothing. + * What these messages mean isn't known. + */ +static void dcp_secure_recv(void *cookie, u8 ep, u64 msg) +{ + struct apple_dcp *dcp = cookie; + int ret; + + dev_dbg(dcp->dev, "secure port: ep %#x msg %#llx\n", ep, msg); + if (ep != DCP_SECURE_EP || (msg >> 60) != 3) + return; + + ret = apple_rtkit_send_message(dcp->secure_rtk, ep, 2ULL << 60, NULL, false); + if (ret) + dev_warn(dcp->dev, "secure port: no reply sent: %d\n", ret); +} + +static const struct apple_rtkit_ops dcp_secure_rtkit_ops = { + .recv_message = dcp_secure_recv, +}; + +/* + * The port's interrupts aren't in the DT. Its handshakes, idle and restart poll it themselves; + * this answers the rest. Every jiffy for a second after its start or a message, as more tend to + * follow, otherwise every 100 ms: woken every jiffy, an idle CPU never reaches deep idle. + */ +#define DCP_SECURE_POLL_FAST_MS 1000 +#define DCP_SECURE_POLL_SLOW_MS 100 + +static void dcp_secure_poll(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + secure_poll_wq); + bool fast = time_before(jiffies, dcp->secure_fast_until); + + if (!READ_ONCE(dcp->secure_polling)) + return; + + if (apple_rtkit_poll(dcp->secure_rtk) > 0) { + if (!fast) + dev_info_once(dcp->dev, "secure port: a message while idle\n"); + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + fast = true; + } + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, + fast ? 1 : msecs_to_jiffies(DCP_SECURE_POLL_SLOW_MS)); +} + +static void dcp_secure_poll_start(struct apple_dcp *dcp) +{ + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + WRITE_ONCE(dcp->secure_polling, true); + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, 0); +} + +static void dcp_secure_poll_stop(struct apple_dcp *dcp) +{ + WRITE_ONCE(dcp->secure_polling, false); + cancel_delayed_work_sync(&dcp->secure_poll_wq); +} + +/* After the firmware's HELLO on its main mailbox. */ +static void dcp_secure_port_start(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->secure_rtk) + return; + + ret = apple_rtkit_port_handshake(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) { + dev_warn(dcp->dev, "secure port handshake not finished: %d\n", ret); + return; + } + dcp->secure_answered = true; + dcp_secure_poll_start(dcp); + dev_info(dcp->dev, "secure port up\n"); +} + +/* Between the main mailbox's AP idle and its IOP idle. */ +static void dcp_secure_port_idle(void *data) +{ + struct apple_dcp *dcp = data; + int ret; + + if (!dcp->secure_rtk || !dcp->secure_answered) + return; + + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_idle(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP idle not acknowledged: %d\n", ret); +} + +static void dcp_release_secure_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + dcp_secure_poll_stop(dcp); + apple_rtkit_free(dcp->secure_rtk); + dcp->secure_rtk = NULL; +} + +/* Nothing is read from the port until its handshake is wanted. */ +static int dcp_secure_port_init(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + + if (of_property_match_string(dev->of_node, "mbox-names", "secure") < 0) + return 0; + + dcp->secure_rtk = apple_rtkit_init(dev, dcp, "secure", 0, &dcp_secure_rtkit_ops); + if (IS_ERR(dcp->secure_rtk)) { + int ret = PTR_ERR(dcp->secure_rtk); + + dcp->secure_rtk = NULL; + return dev_err_probe(dev, ret, "Failed to initialize the secure port\n"); + } + + INIT_DELAYED_WORK(&dcp->secure_poll_wq, dcp_secure_poll); + + return devm_add_action_or_reset(dev, dcp_release_secure_rtkit, dcp); +} + static void dcp_release_rtkit(void *data) { struct apple_dcp *dcp = data; @@ -1580,11 +1712,16 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) return iomfb_v14_7_atomic_check(dcp, crtc, commit); - if (dcp->crashed) - return -EINVAL; - crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + /* + * A crashed DCP can't be lit, but a commit that leaves its CRTC off has to pass: + * the system suspend's disable-all includes every CRTC, and a stopped external DCP + * would refuse every sleep after the first. + */ + if (dcp->crashed) + return crtc_state && crtc_state->enable ? -EINVAL : 0; + /* External H17P does not program adaptive sync. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && @@ -3872,6 +4009,7 @@ int dcp_start(struct platform_device *pdev) /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ init_completion(&dcp->start_done); + init_completion(&dcp->s2r_dart_done); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) return iomfb_v26_6_start(dcp); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) @@ -4175,6 +4313,9 @@ static bool dcp_h17g_panel(struct apple_dcp *dcp) dcp_has_panel(dcp); } +static void dcp_dpms_power_off(struct apple_dcp *dcp); +static void dcp_dpms_power_on(struct apple_dcp *dcp); + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { /* The 14.7 panel has its own known-level blank and restore hooks. */ @@ -4244,6 +4385,7 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { + dcp_dpms_power_on(dcp); if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { @@ -4390,11 +4532,15 @@ void dcp_poweroff(struct platform_device *pdev) cancel_delayed_work(&dcp->swap_watchdog_wq); - /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + /* + * Internal H17P DPMS is a brightness present, never a pipe stop. On T8140 the + * display then powers off as well, after the backlight. + */ if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + dcp_dpms_power_off(dcp); return; } @@ -5481,6 +5627,13 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; + /* After a warm restart its log counters need echoing, or it refuses its next idle. */ + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcp") || + of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) + apple_rtkit_set_oslog_echo(dcp->rtk, true); + ret = dcp_secure_port_init(dcp); if (ret) goto err_piodma; @@ -5499,6 +5652,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } + dcp_secure_port_start(dcp); enable_work(&dcp->dimensions_wq); schedule_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); @@ -5861,6 +6015,8 @@ static void dcp_t8132_ext_routes(struct apple_dcp *dcp) dcp->nr_typec_routes); } +static int dcp_s2r_pm_register(struct apple_dcp *dcp); + static int dcp_platform_probe_pipe(struct platform_device *pdev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); @@ -6038,6 +6194,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) } platform_set_drvdata(pdev, dcp); + ret = dcp_s2r_pm_register(dcp); + if (ret) + return ret; if (iomfb_is_m4(dcp)) { dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; @@ -6719,10 +6878,13 @@ static int dcp_platform_prepare(struct device *dev) return 0; } +static void dcp_s2r_dark_finish(struct apple_dcp *dcp); + static void dcp_platform_complete(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_s2r_dark_finish(dcp); if (!dcp->external) return; mutex_lock(&dcp->hpd_mutex); @@ -6730,105 +6892,1234 @@ static void dcp_platform_complete(struct device *dev) mutex_unlock(&dcp->hpd_mutex); } -static int dcp_platform_suspend(struct device *dev) +/* + * T8140 suspend-to-RAM: the sleep engine waits for the DCP's domain to be off, so + * the firmware has to stop first. Measured from macOS, in order: + * + * 1. A414(0), A478 x2, A476(1), A478, A412, A473(0). + * 2. D572(0) from the firmware: the display DART's streams go off. + * 3. A500, unnamed, then D107s and D575(0) from the firmware. + * 4. RTKit idle, then the ASC's run bit off. + * 5. The DCP DART's streams off. That's the coprocessor's own shutdown, apart + * from step 2. The sleep driver then lowers the domains. + * + * macOS also restarts the firmware 893 ms before the commit, calls A478, A473(0), + * A115, A500, and stops it again. Why isn't known, and that's not done here. + * + * The firmware takes IOP idle only with its secure mailbox answered, see + * dcp_s2r_answer_secure_port(). + */ +#define DCP_S2R_DISP_STREAMS (BIT(0) | BIT(3) | BIT(4) | BIT(16)) +#define DCP_S2R_DCP_STREAMS (BIT(23) | BIT(24)) +#define DCP_S2R_D572_TIMEOUT_MS 1000 /* macOS: about 170 ms after A473(0) */ +#define DCP_PS_MIN GENMASK(19, 16) + +static struct device *dcp_s2r_find_dart(struct device_node *np) { - struct apple_dcp *dcp = dev_get_drvdata(dev); + struct device_node *dart; + struct platform_device *pdev; - /* Its power sequences may crash the DCP and blank the panel. */ - if (dcp_takeover_was_aborted(dcp)) - return -EBUSY; + dart = of_parse_phandle(np, "iommus", 0); + if (!dart) + return NULL; + pdev = of_find_device_by_node(dart); + of_node_put(dart); - /* - * T8132: setPowerState powers the display down; the coprocessor keeps - * running and its power domains stay on, as below, unless the internal - * display is blanked with appledrm.power_gate_ms set: then the - * coprocessor and its domains are powered down too (iomfb_m4_suspend()). - */ - if (iomfb_is_m4(dcp)) { - int ret = iomfb_m4_suspend(dcp); + return pdev ? &pdev->dev : NULL; +} - if (!ret) - device_set_wakeup_path(dev); - return ret; +static void dcp_s2r_put_darts(struct apple_dcp *dcp) +{ + if (dcp->s2r_dcp_dart) + put_device(dcp->s2r_dcp_dart); + if (dcp->s2r_disp_dart) + put_device(dcp->s2r_disp_dart); + dcp->s2r_dcp_dart = NULL; + dcp->s2r_disp_dart = NULL; +} + +static void dcp_s2r_find_darts(struct apple_dcp *dcp) +{ + struct device_node *piodma; + + /* A suspend that failed after an earlier lookup still holds those. */ + dcp_s2r_put_darts(dcp); + dcp->s2r_dcp_dart = dcp_s2r_find_dart(dcp->dev->of_node); + piodma = of_get_child_by_name(dcp->dev->of_node, "piodma"); + if (piodma) { + dcp->s2r_disp_dart = dcp_s2r_find_dart(piodma); + of_node_put(piodma); } - if (READ_ONCE(dcp->ext_dead)) - return 0; - iomfb_v14_7_backlight_suspend(dcp); +} - /* Serialize PM against explicit bring-up, including queued startup work. - * Before a startup attempt the device remains suspendable. After any - * attempt we cannot prove DMA/firmware quiescence, even on failure. - */ - if (dcp->external) { - const char *why; +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on) +{ + if (!READ_ONCE(dcp->s2r_active)) + return; - mutex_lock(&dcp->hpd_mutex); - why = dcp_external_sleep_vetoed(dcp); - if (why) { - mutex_unlock(&dcp->hpd_mutex); - if (dcp->external_native) - dev_warn(dev, "suspend refused: %s\n", why); - else - dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); - return -EBUSY; - } - WRITE_ONCE(dcp->external_suspended, true); - mutex_unlock(&dcp->hpd_mutex); + if (dcp->s2r_disp_dart) + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, on, false); + complete(&dcp->s2r_dart_done); +} + +static bool dcp_s2r_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcp"); +} + +static void dcp_s2r_cpu_stop(struct apple_dcp *dcp) +{ + /* macOS writes it twice. */ + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); +} + +/* Up from bind, unless its handshake failed then. A bound firmware needs it to take IOP idle. */ +static void dcp_s2r_answer_secure_port(struct apple_dcp *dcp) +{ + if (!dcp->secure_answered) + dcp_secure_port_start(dcp); +} + +/* After a refused idle: AP on to both mailboxes, as the GPU and the AOP send it. */ +static void dcp_s2r_idle_undo(struct apple_dcp *dcp) +{ + int ret; + + ret = apple_rtkit_ap_power(dcp->rtk, true); + if (ret) + dev_warn(dcp->dev, "AP on after the refused idle: %d\n", ret); + if (!dcp->secure_rtk || !dcp->secure_answered || READ_ONCE(dcp->secure_polling)) + return; + ret = apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP on after the refused idle: %d\n", ret); + dcp_secure_poll_start(dcp); +} + +/* + * macOS's order up to RTKit idle. On success the firmware is idle and its run bit + * is still set. + */ +static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) +{ + ktime_t start = ktime_get(); + /* Off since a refused idle: no second set_power_state(0), whose D572 may never come. */ + bool display_off = dcp->s2r_refused_off; + bool idle_sent = false; + s64 power_ms; + bool idle; + int ret; + + reinit_completion(&dcp->s2r_dart_done); + dcp_s2r_find_darts(dcp); + WRITE_ONCE(dcp->s2r_active, true); + + if (display_off) { + dev_info(dcp->dev, "display still off from a refused idle\n"); + goto display_is_off; } - if (dcp_is_external_h17p(dcp)) { - int ret; + /* In band, 3 of 3 on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, false, true, false); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(0): %d\n", ret); + goto err; + } + power_ms = ktime_ms_delta(ktime_get(), start); - dcp_disable_typec_work(dcp, false); - ret = dcp_external_disconnect(dcp, false); - if (ret) { - dcp_enable_typec_work(dcp); - return ret; - } - device_set_wakeup_path(dev); - return 0; + /* Without D572 the display isn't known to be off, and nothing below can undo this. */ + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) { + dev_err(dcp->dev, "no D572 %lld ms after set_power_state(0)\n", + ktime_ms_delta(ktime_get(), start)); + ret = -ETIMEDOUT; + goto err; } + dev_dbg(dcp->dev, "display off: set_power_state(0) %lld ms, D572 %lld ms\n", power_ms, + ktime_ms_delta(ktime_get(), start)); + display_off = true; - dcp_drain_for_sleep(dcp); - /* - * The Type-C route reports cable removal through - * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays - * connected, and resume powers the CRTC back up through dcp_poweron() - * as after DPMS off. - */ - dcp_disable_typec_work(dcp, false); - cancel_delayed_work_sync(&dcp->swap_watchdog_wq); - if (dcp->external_native) - disable_work_sync(&dcp->external_audio_work); +display_is_off: + /* D572 itself is still being answered, and a busy channel can't be sent on. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before A500\n"); - if (dcp->avep) - av_service_disconnect(dcp); + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + goto err; + } + dev_dbg(dcp->dev, "A500 replied after %lld ms\n", ktime_ms_delta(ktime_get(), start)); - if (dcp->hdmi_hpd_irq) { - disable_irq(dcp->hdmi_hpd_irq); - if (!dcp->active_typec_route) { - disconnected_hpd_event(dcp->connector); - dcp_dptx_disconnect(dcp, 0); - } + /* The firmware's callbacks after A500 have to be answered first. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before idle\n"); + + idle_sent = true; + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); + if (ret) { + /* Answer what it still has queued, its log counters among them, and ask again. */ + dev_warn(dcp->dev, "firmware refused idle after %lld ms: %d, control %#x status %#x: retrying once\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + apple_rtkit_poll(dcp->rtk); + msleep(30); + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); } + if (ret) { + dev_err(dcp->dev, "firmware did not go idle after %lld ms: %d, control %#x status %#x\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + goto err; + } + + /* Both set once the firmware's last message has been read; macOS waits for them. */ + if (readl_poll_timeout(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS, *status, + (*status & (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) == + (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED), 100, 5000)) + dev_warn(dcp->dev, "idle status not set: %#x\n", *status); + dev_dbg(dcp->dev, "firmware idle after %lld ms, status %#x\n", + ktime_ms_delta(ktime_get(), start), *status); + + WRITE_ONCE(dcp->s2r_active, false); + dcp->s2r_refused_off = false; + + return 0; + +err: + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); /* - * Set the device as a wakeup device, which forces its power - * domains to stay on. We need this as we do not support full - * shutdown properly yet. + * Refused with the display off: leave the firmware running, never stop it under its + * own feet. DPMS on lights the display again on the running firmware. */ - device_set_wakeup_path(dev); - /* Last: the steps above still talk to the coprocessor. */ - dcp_hold_for_sleep(dcp); + if (display_off) { + if (idle_sent) + dcp_s2r_idle_undo(dcp); + dcp->s2r_refused_off = true; + dev_warn(dcp->dev, "display left off with the firmware running\n"); + return ret; + } + /* The display may be off and the firmware half idle. Nothing here brings it back. */ + WRITE_ONCE(dcp->crashed, true); - return 0; + return ret; } -static int dcp_platform_resume(struct device *dev) +/* Its power state can only be lowered once this has finished. */ +static int dcp_s2r_stop_streams(struct apple_dcp *dcp) { - struct apple_dcp *dcp = dev_get_drvdata(dev); + if (!dcp->s2r_dcp_dart) + return -ENODEV; - if (dcp_takeover_was_aborted(dcp)) - return 0; + return apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, false, true); +} + +static int dcp_s2r_stop_firmware(struct apple_dcp *dcp) +{ + u32 status; + int ret; + + ret = dcp_s2r_quiesce(dcp, &status); + if (ret) + return ret; + + apple_rtkit_mailbox_off(dcp->rtk); + dcp_s2r_cpu_stop(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + return dcp_s2r_stop_streams(dcp); +} + +/* + * macOS stops this firmware with one call the size of A500, then about 4 ms later AP + * then IOP idle and the run bit off twice. Nothing has shown it needing a secure + * port. + * + * An attached display goes first, as macOS's DP port releases its display at system + * sleep: the pipe off, HPD low and the display released, as DPMS off does, then the + * crossbar output. The route and the connector stay, and after the wake the link + * connects again, as macOS replays the port's HPD. + */ +static bool dcp_s2r_ext_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); +} + +/* + * Taken as stopped, but a start since raised its domains and failed after the run bit, + * as one a signal ended. Left running, it refuses every sleep; stopped, it comes back + * from the sleep's power cycle like any other. + */ +static void dcp_s2r_stop_failed_start(struct apple_dcp *dcp) +{ + if (!dcp->s2r_domains_up || + !(readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL) & + APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return; + dev_warn(dcp->dev, "a failed start left the firmware running: stopping it for the sleep\n"); + dcp_s2r_cpu_stop(dcp); +} + +/* + * pmc-dcp counts the powered DCPs, as macOS keeps it: 0, 1, 2, 1, 0 across a wake. A DCP + * counts from when this driver powers it up until it powers it off or a sleep stops it. + */ +static DEFINE_MUTEX(dcp_pmc_lock); +static unsigned int dcp_pmc_count; + +static void dcp_pmc_set(struct apple_dcp *dcp, void __iomem *pmc, bool up) +{ + guard(mutex)(&dcp_pmc_lock); + if (up != dcp->pmc_up) { + dcp->pmc_up = up; + if (up) + dcp_pmc_count++; + else if (dcp_pmc_count) + dcp_pmc_count--; + } + writel(dcp_pmc_count, pmc); +} + +/* The sleep controller lowers the domains and pmc-dcp itself. */ +static void dcp_pmc_forget(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_pmc_lock); + if (dcp->pmc_up && dcp_pmc_count) + dcp_pmc_count--; + dcp->pmc_up = false; +} + +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp); + +/* + * Crashed firmware takes no stop sequence, and refusing would hold every later sleep off until + * reboot. Its CPU stops cold, as for a failed start, and the wake starts it again, as macOS + * recovers a crashed DCP. A system sleep only: DPMS off still refuses it. + */ +static bool dcp_s2r_stops_crashed(struct apple_dcp *dcp) +{ + return READ_ONCE(dcp->crashed) && !IS_ERR_OR_NULL(dcp->rtk) && + pm_suspend_target_state != PM_SUSPEND_ON; +} + +static int dcp_s2r_stop_crashed(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + dev_warn(dcp->dev, "firmware crashed: stopped cold for the sleep, started again after it\n"); + apple_rtkit_mailbox_off(dcp->rtk); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) +{ + int ret; + + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + dcp_typec_route_park(dcp); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + return 0; + } + + /* No A500 and no disconnect: the link and its crossbar output just go down. */ + if (dcp_s2r_stops_crashed(dcp)) { + dcp_disable_typec_work(dcp, false); + dcp_typec_route_park(dcp); + return dcp_s2r_stop_crashed(dcp); + } + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) + dev_dbg(dcp->dev, "a display is attached: released for the sleep, connected again after it\n"); + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dcp_enable_typec_work(dcp); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* The display's link went above: its crossbar output follows, as on macOS. */ + dcp_typec_route_park(dcp); + + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + usleep_range(4000, 5000); + + ret = apple_rtkit_sleep_stop(dcp->rtk, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (ret) { + dev_err(dcp->dev, "firmware did not go idle: %d, control %#x status %#x\n", ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +/* A failure before the firmware has stopped leaves it running, and refuses cleanly. */ +static int dcp_s2r_suspend(struct apple_dcp *dcp) +{ + bool was_off; + int ret; + + /* Still stopped from a suspend that never reached the commit. */ + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + ret = dcp_s2r_stop_streams(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + return 0; + } + + if (dcp_s2r_stops_crashed(dcp)) + return dcp_s2r_stop_crashed(dcp); + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + dcp_s2r_answer_secure_port(dcp); + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* Presentation stays paused if this fails: see dcp_s2r_quiesce(). */ + was_off = dcp->s2r_refused_off; + ret = dcp_s2r_stop_firmware(dcp); + if (ret) { + if (READ_ONCE(dcp->s2r_fw_down)) + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + /* Dark since a refused idle, as after DPMS off: it stays so until DPMS on. */ + if (was_off) + dcp->dpms_powered_off = true; + + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static void dcp_s2r_ps_lower(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & BIT(28)) { + val &= ~(BIT(28) | GENMASK(9, 8)); + writel(val, reg); + } + writel((val & ~GENMASK(3, 0)) | GENMASK(9, 8), reg); +} + +/* macOS's power-off, measured 3 of 3. The firmware is stopped and its DARTs are off. */ +static int dcp_s2r_domains_off(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + dcp->s2r_domains_up = false; + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + dcp_pmc_set(dcp, pmc, false); + val = readl(ps + 0x10000); + if (val & BIT(28)) + writel(val & ~(BIT(28) | GENMASK(9, 8)), ps + 0x10000); + writel(3, pwrgate + 0xc8c); + writel(0, pwrgate + 0xcd0); + dcp_s2r_ps_lower(ps + 0x10000); + dcp_s2r_ps_lower(ps + 0x318); + dcp_s2r_ps_lower(ps + 0x280); + + ret = readl_poll_timeout(ps + 0x10000, val, !FIELD_GET(GENMASK(7, 4), val), 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power off: %#x\n", val); + dev_dbg(dcp->dev, "display domains off: CPU %#x FE %#x SYS %#x\n", readl(ps + 0x10000), + readl(ps + 0x318), readl(ps + 0x280)); +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * Exact mirror of the writes in macOS's power-off, measured 3 of 3. DISP_SYS and + * DISP_FE first, then DISP_CPU with AUTO_ENABLE last. + */ +static int dcp_s2r_domains_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + val = readl(ps + 0x280) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x280); + writel(val | 0xf | BIT(28), ps + 0x280); + val = readl(ps + 0x318) & ~(GENMASK(9, 8) | GENMASK(3, 0)); + writel(val | 0xf, ps + 0x318); + val = readl(ps + 0x10000) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x10000); + writel(0, pwrgate + 0xc8c); + writel(1, pwrgate + 0xcd0); + writel(val | 0xf | BIT(28), ps + 0x10000); + dcp_pmc_set(dcp, pmc, true); + + ret = readl_poll_timeout(ps + 0x10000, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power on: %#x\n", val); + else + dcp->s2r_domains_up = true; +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * DISPEXT0 up as macOS does it after a wake: SYS and FE on, then the CPU with PS_MIN 4 first, + * since the wake reset it, then on, and pmc-dcp up once it reads on. + */ +static int dcp_s2r_ext_domains_on(struct apple_dcp *dcp) +{ + static const u16 sys_fe[] = { 0x288, 0x320 }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + u32 val; + int i, ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + for (i = 0; i < ARRAY_SIZE(sys_fe); i++) { + val = readl(ps + sys_fe[i]) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + sys_fe[i]); + writel(val | 0xf | BIT(28), ps + sys_fe[i]); + } + val = readl(ps + 0x328) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28) | DCP_PS_MIN); + val |= FIELD_PREP(DCP_PS_MIN, 4); + writel(val, ps + 0x328); + writel(val | 0xf, ps + 0x328); + writel(val | 0xf | BIT(28), ps + 0x328); + + ret = readl_poll_timeout(ps + 0x328, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) { + dev_err(dcp->dev, "DISPEXT0_CPU did not power on: %#x\n", val); + } else { + dcp_pmc_set(dcp, pmc, true); + dcp->s2r_domains_up = true; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * DCPEXT back after a real sleep, the same way in every macOS wake measured: its domains, + * its DART's streams, and a warm restart, with no secure port and no display-on call. After + * its own power cycle the start is the same, with its display DART lit again. + */ +static int dcp_s2r_ext_start(struct apple_dcp *dcp, bool cycled) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (!ret) + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + /* After the domains, whose writes it would reject, the DART protects them again. */ + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + if (!cycled) + return apple_rtkit_sleep_return(dcp->rtk, control); + + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, false); + return apple_rtkit_power_cycled_return(dcp->rtk, control); +} + +/* DISPEXT0 off in its own order, CPU first. Only the CPU has to power-gate to start again. */ +static int dcp_s2r_ext_domains_off(struct apple_dcp *dcp) +{ + static const struct { + u16 offset; + u32 first, second; + bool member; + } steps[] = { + { 0x328, 0xf0400ff, 0xf0403f0 }, { 0x320, 0, 0x23f0, true }, { 0x288, 0, 0x23f0 }, + }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + int i, ret = -ENODEV; + u32 val; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + /* + * Its display DART goes off with DISPEXT0, and every framebuffer is mapped into it: a + * map during the hold would reach its registers. Dark until the domains are back. + */ + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, true); + dcp->s2r_domains_up = false; + dcp_pmc_set(dcp, pmc, false); + ret = 0; + for (i = 0; i < ARRAY_SIZE(steps); i++) { + int gated; + + if (steps[i].first) + writel(steps[i].first, ps + steps[i].offset); + writel(steps[i].second, ps + steps[i].offset); + gated = readl_poll_timeout(ps + steps[i].offset, val, + !(val & GENMASK(7, 4)) && !(val & BIT(11)) && + (steps[i].member || (val & BIT(8))), 100, 50000); + dev_dbg(dcp->dev, "DISPEXT0 power word %#x reads %#x: %d\n", steps[i].offset, val, + gated); + if (i == 0 && gated) + ret = gated; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +#define DCP_S2R_EXT_CYCLE_HOLD_MS 1000 + +static bool dcp_s2r_ext_cpu_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps; + bool on; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return false; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + of_node_put(np); + if (!ps) + return false; + on = FIELD_GET(GENMASK(7, 4), readl(ps + 0x328)); + iounmap(ps); + + return on; +} + +/* + * After a refused sleep DCPEXT has idled with its power kept, and an idled firmware doesn't + * wake again: power-cycle it through DISPEXT0's own sequence, then start it as after a sleep. + * The sleep driver lowers DISPEXT0 before refusals of its own, and with it off the CPU and the + * display DART don't answer, so it's raised first. + */ +static int dcp_s2r_ext_cycle(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (ret) + return ret; + if (!dcp_s2r_ext_cpu_on(dcp)) { + dev_info(dcp->dev, "DISPEXT0 lowered by the refused sleep: raised for the cycle\n"); + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + } + + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + ret = dcp_s2r_ext_domains_off(dcp); + if (!ret) { + msleep(DCP_S2R_EXT_CYCLE_HOLD_MS); + ret = dcp_s2r_ext_start(dcp, true); + } + + return ret; +} + +/* Between the main mailbox's IOP report and its AP on, measured 2 of 2. */ +static void dcp_secure_port_map(void *data) +{ + struct apple_dcp *dcp = data; + ktime_t start = ktime_get(); + + dcp->secure_map_ret = apple_rtkit_port_map(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + dcp->secure_map_us = ktime_us_delta(ktime_get(), start); +} + +static int dcp_s2r_warm_restart(struct apple_dcp *dcp) +{ + int ret; + + ret = dcp_s2r_domains_on(dcp); + if (ret) + return ret; + + /* After the domains, whose writes it would reject, the DART protects them again. */ + if (dcp->s2r_dcp_dart) + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + + if (dcp->secure_rtk) { + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_restart(dcp->secure_rtk); + if (ret) + return ret; + } + + ret = apple_rtkit_warm_restart(dcp->rtk); + if (ret) + return ret; + + writel(APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + ret = apple_rtkit_boot_with(dcp->rtk, dcp->secure_rtk ? dcp_secure_port_map : NULL, dcp); + if (ret) + return ret; + + if (dcp->secure_rtk) { + ret = dcp->secure_map_ret ?: + apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port handshake after restart: %d\n", ret); + else + dev_dbg(dcp->dev, "secure port up after restart, map took %lld us\n", + dcp->secure_map_us); + dcp_secure_poll_start(dcp); + } + + return 0; +} + +/* + * Display on after a sleep, and after DPMS off: the warm restart, then the named calls + * macOS sends first, A414(0), A478 x2, A476(1), A478, A412, then A473(1). + * + * Not sent: A439, which macOS overlaps with A473(1), A438 around swaps, A461 per frame + * and A352. They stay unnamed, and A439 and A438 hold macOS kernel pointers. Linux + * already lights the panel without A438 at boot. + */ +static int dcp_s2r_display_on(struct apple_dcp *dcp) +{ + int ret; + + ret = dcp_s2r_warm_restart(dcp); + if (ret) { + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + return ret; + } + + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + + /* Out of band; either channel answered on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(1): %d\n", ret); + goto out; + } + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); +out: + WRITE_ONCE(dcp->s2r_active, false); + + return ret; +} + +/* Power kept: lowered as DPMS off does. A failed flush left the GAPF slice faulting that. */ +static int dcp_s2r_refused_domains_off(struct apple_dcp *dcp) +{ + int ret = dcp->s2r_dcp_dart ? apple_dart_gapf_ready(dcp->s2r_dcp_dart) : -ENODEV; + + return ret ?: dcp_s2r_domains_off(dcp); +} + +static void dcp_s2r_restart_done(struct apple_dcp *dcp, int ret) +{ + bool ext = dcp_s2r_ext_capable(dcp); + + dcp_s2r_put_darts(dcp); + /* An attached display's crossbar output; its link reconnects from the Type-C work. */ + if (ext) + dcp_typec_route_unpark(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); + return; + } + /* Back from a full restart, even after a failed start had marked it crashed. */ + WRITE_ONCE(dcp->crashed, false); + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + if (ext && !READ_ONCE(dcp->typec_work_enabled)) + dcp_enable_typec_work(dcp); +} + +/* + * If the display doesn't come back, the panel stays dark until reboot, and the driver + * must not talk to the stopped firmware, so treat it as a crashed DCP. Nothing else in + * the system depends on it. + */ +static int dcp_s2r_restart(struct apple_dcp *dcp, bool real) +{ + int ret = -EAGAIN; + + /* + * After a refused sleep the firmware has idled with its power kept, and an idled firmware + * doesn't wake again: the internal DCP comes back as DPMS on brings it, from a power + * cycle, and DCPEXT through its own. + */ + if (dcp_s2r_capable(dcp)) { + ret = real ? 0 : dcp_s2r_refused_domains_off(dcp); + if (!ret) + ret = dcp_s2r_display_on(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware restarted as for DPMS: %d\n", + ret); + } else if (dcp_s2r_ext_capable(dcp)) { + ret = real ? dcp_s2r_ext_start(dcp, false) : dcp_s2r_ext_cycle(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware %s after a power cycle: %d\n", + ret ? "not back" : "back", ret); + else if (ret) + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + } + dcp_s2r_restart_done(dcp, ret); + + return ret; +} + +static int dcp_s2r_resume(struct apple_dcp *dcp) +{ + bool real = pm_resume_via_firmware(); + + dcp->s2r_stopped = false; + /* A real sleep took the domains' power. */ + if (real) + dcp->s2r_domains_up = false; + /* A dark wake sleeps again with the panel never lit, as if DPMS had turned it off. */ + if (pm_dark_wake() && dcp_s2r_capable(dcp) && !dcp->dpms_powered_off) { + dcp->dpms_powered_off = true; + dev_info(dcp->dev, "dark wake: firmware and panel stay off\n"); + } + /* DCPEXT too, its crossbar output parked: dcp_s2r_dark_finish() if no resume follows. */ + if (pm_dark_wake() && dcp_s2r_ext_capable(dcp)) { + dcp->s2r_dark_held = true; + dev_info(dcp->dev, "dark wake: firmware and output stay off\n"); + return 0; + } + dcp->s2r_dark_held = false; + /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ + if (dcp->dpms_powered_off) { + /* Off after a refused idle, its power may be on: DPMS on needs it lowered. */ + if (!real) + dcp_s2r_refused_domains_off(dcp); + dcp_s2r_put_darts(dcp); + return 0; + } + dcp_s2r_restart(dcp, real); + + return 0; +} + +/* + * The suspend after a dark wake failed before suspending DCPEXT, or before preparing it. It's + * still down from the real sleep, whose wake reset its CPU: start it as after one. + */ +static void dcp_s2r_dark_finish(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->s2r_dark_held || pm_dark_wake() || IS_ERR_OR_NULL(dcp->rtk)) + return; + dcp->s2r_dark_held = false; + dcp->s2r_stopped = false; + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_ext_start(dcp, true); + dev_info(dcp->dev, "dark wake over without a resume: firmware %s: %d\n", + ret ? "not back" : "back", ret); + dcp_s2r_restart_done(dcp, ret); +} + +static int dcp_s2r_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_dcp *dcp = container_of(nb, struct apple_dcp, s2r_pm_nb); + + if (action == PM_POST_SUSPEND) + dcp_s2r_dark_finish(dcp); + + return NOTIFY_DONE; +} + +static void dcp_s2r_pm_unregister(void *data) +{ + struct apple_dcp *dcp = data; + + unregister_pm_notifier(&dcp->s2r_pm_nb); +} + +static int dcp_s2r_pm_register(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp_s2r_ext_capable(dcp)) + return 0; + dcp->s2r_pm_nb.notifier_call = dcp_s2r_pm_notify; + ret = register_pm_notifier(&dcp->s2r_pm_nb); + if (ret) + return ret; + + return devm_add_action_or_reset(dcp->dev, dcp_s2r_pm_unregister, dcp); +} + +/* + * Its own suspend failed with the CPU stopped, and the PM core won't call .resume: start it + * again as a refused sleep does, or the display stays dark until a later sleep gets through. + */ +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp) +{ + dev_warn(dcp->dev, "stop failed with the CPU stopped: starting the firmware again\n"); + dcp_s2r_restart(dcp, false); + dcp->s2r_stopped = false; +} + +/* + * DPMS on the internal T8140 panel as macOS does a display sleep: the backlight to 0, + * then the firmware stopped and the display domains off, and the reverse on wake. Not + * during a system sleep: that path stops the firmware itself. + */ +static bool dcp_dpms_power_cycles(struct apple_dcp *dcp) +{ + return dcp_s2r_capable(dcp) && pm_suspend_target_state == PM_SUSPEND_ON; +} + +static void dcp_dpms_power_off(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (!dcp_dpms_power_cycles(dcp) || READ_ONCE(dcp->s2r_fw_down)) + return; + + ret = dcp_s2r_suspend(dcp); + if (ret && !READ_ONCE(dcp->s2r_fw_down)) { + dev_warn(dcp->dev, "DPMS: firmware stop refused, backlight only: %d\n", ret); + return; + } + dcp->s2r_stopped = false; + if (!ret) + ret = dcp_s2r_domains_off(dcp); + dcp_s2r_put_darts(dcp); + dcp->dpms_powered_off = true; + dev_dbg(dcp->dev, "DPMS: display powered off in %lld ms: %d\n", + ktime_ms_delta(ktime_get(), start), ret); +} + +/* After a refused idle the firmware still runs with the display off: light it again there. */ +static void dcp_s2r_light_running(struct apple_dcp *dcp) +{ + int ret; + + dcp_s2r_find_darts(dcp); + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (!ret && !wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "display not back on the running firmware: %d\n", ret); + return; + } + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "display back on the running firmware\n"); +} + +static void dcp_dpms_power_on(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (dcp->s2r_refused_off) { + dcp->s2r_refused_off = false; + dcp_s2r_light_running(dcp); + return; + } + if (!dcp->dpms_powered_off) + return; + dcp->dpms_powered_off = false; + if (!READ_ONCE(dcp->s2r_fw_down)) + return; + + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_display_on(dcp); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "DPMS: display power-on failed, it stays off: %d\n", ret); + return; + } + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "DPMS: display powered on in %lld ms\n", + ktime_ms_delta(ktime_get(), start)); +} + +bool dcp_s2r_firmware_stopped(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return READ_ONCE(dcp->s2r_fw_down); +} + +static int dcp_platform_suspend(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + /* Its power sequences may crash the DCP and blank the panel. */ + if (dcp_takeover_was_aborted(dcp)) + return -EBUSY; + + if (pm_suspend_via_firmware() && dcp_s2r_capable(dcp)) + return dcp_s2r_suspend(dcp); + if (pm_suspend_via_firmware() && dcp_s2r_ext_capable(dcp)) + return dcp_s2r_ext_suspend(dcp); + + /* + * T8132: setPowerState powers the display down; the coprocessor keeps + * running and its power domains stay on, as below, unless the internal + * display is blanked with appledrm.power_gate_ms set: then the + * coprocessor and its domains are powered down too (iomfb_m4_suspend()). + */ + if (iomfb_is_m4(dcp)) { + int ret = iomfb_m4_suspend(dcp); + + if (!ret) + device_set_wakeup_path(dev); + return ret; + } + if (READ_ONCE(dcp->ext_dead)) + return 0; + iomfb_v14_7_backlight_suspend(dcp); + + /* Serialize PM against explicit bring-up, including queued startup work. + * Before a startup attempt the device remains suspendable. After any + * attempt we cannot prove DMA/firmware quiescence, even on failure. + */ + if (dcp->external) { + const char *why; + + mutex_lock(&dcp->hpd_mutex); + why = dcp_external_sleep_vetoed(dcp); + if (why) { + mutex_unlock(&dcp->hpd_mutex); + if (dcp->external_native) + dev_warn(dev, "suspend refused: %s\n", why); + else + dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); + return -EBUSY; + } + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); + } + + if (dcp_is_external_h17p(dcp)) { + int ret; + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + device_set_wakeup_path(dev); + return 0; + } + + dcp_drain_for_sleep(dcp); + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + if (dcp->external_native) + disable_work_sync(&dcp->external_audio_work); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->hdmi_hpd_irq) { + disable_irq(dcp->hdmi_hpd_irq); + if (!dcp->active_typec_route) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); + /* Last: the steps above still talk to the coprocessor. */ + dcp_hold_for_sleep(dcp); + + return 0; +} + +static int dcp_platform_resume(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp_takeover_was_aborted(dcp)) + return 0; + + if (dcp->s2r_stopped) + return dcp_s2r_resume(dcp); if (iomfb_is_m4(dcp)) return iomfb_m4_resume(dcp); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index aaaf2918c4b75..2046fb4a6f033 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -121,6 +121,7 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp); int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); +bool dcp_s2r_firmware_stopped(struct platform_device *pdev); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8bca8d2cebeca..fc88a2b1f8d5c 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -357,7 +357,7 @@ static bool iomfb_uses_queue(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } -static bool iomfb_channels_idle(struct apple_dcp *dcp) +bool iomfb_channels_idle(struct apple_dcp *dcp) { return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && @@ -2931,6 +2931,10 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct platform_device *pdev = apple_crtc->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + /* Can't be lit: a resume then restores its CRTC disabled, not failing every output. */ + if (READ_ONCE(dcp->crashed)) + return false; + if (iomfb_is_m4(dcp)) { struct drm_display_mode fixed; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 2ae15b13b8d43..ee7096c54ff83 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -190,6 +190,7 @@ enum dcpep_method { dcpep_pipe_query_478, dcpep_pipe_query_474, dcpep_pipe_set_476, + dcpep_unnamed_500, iomfbep_set_brightness_correction, dcpep_num_methods }; @@ -210,6 +211,7 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_channels_idle(struct apple_dcp *dcp); bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, u32 out_len); /* Desktop 26.6: does the record hold what the tag's handler reads and writes? */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 1e24d0e617a79..eb3e7e0e84c05 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -241,6 +241,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A500", dcpep_unnamed_500), /* (0x4, 0) */ IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ IOMFB_METHOD("A428", iomfbep_set_brightness_correction), /* (0x4, 0x4) 0x10000 */ IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ @@ -312,6 +313,21 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +/* + * D572: the firmware controls the display DART's power stage with this. The + * argument is 0 when it's done with the display and 1 when it wants it back. + */ +static bool trampoline_dart_power_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_packet_header *header = in - sizeof(*header); + + if (header->in_len >= sizeof(u32)) + dcp_s2r_dart_power(dcp, get_unaligned_le32(in)); + + return trampoline_zero(dcp, tag, out, in); +} + static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -691,7 +707,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [563] = trampoline_true, [565] = trampoline_true, [567] = trampoline_true, - [572] = trampoline_zero, /* powerUpDART */ + [572] = trampoline_dart_power_h17p, /* powerUpDART */ [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 80d341e7eb2e1..6c64f915bf7ea 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -504,6 +504,7 @@ DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_IN(dcp_unnamed_500, dcpep_unnamed_500, u32); #endif DCP_THUNK_INOUT(dcp_set_brightness_correction, iomfbep_set_brightness_correction, u32, u32); @@ -2297,6 +2298,120 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } } +#if DCP_FW_H17(DCP_FW_VER) +/* + * set_power_state around a firmware-driven suspend. What macOS sends before it in + * either direction: A414(0), A478 twice, A476(1), A478, A412. All are named in + * this firmware's table, and their arguments are measured from macOS. + * + * For display on, macOS also sends A439 at the same moment. Its name isn't known, + * it carries a macOS kernel pointer and its reply wasn't captured, so it isn't + * sent here. + */ +enum dcp_s2r_step { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A412, DCP_S2R_A473, +}; + +static const u8 dcp_s2r_seq[] = { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A478, + DCP_S2R_A412, DCP_S2R_A473, +}; + +static void dcp_s2r_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned int step = dcp->s2r_step++; + + switch (dcp_s2r_seq[step]) { + case DCP_S2R_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A478: + dcp_pipe_query_478(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A476: { + u32 val = 1; + + dcp_pipe_set_476(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A412: + dcp_is_main_display(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A473: { + /* Measured: the state, then 00 00 00 00 00 aa aa aa. */ + struct DCP_FW_NAME(dcp_set_power_state_req) req = { + .unklong = dcp->s2r_power_on, + .padding = { 0xaa, 0xaa, 0xaa }, + }; + + dcp_set_power_state(dcp, dcp->s2r_power_oob, &req, + dcp->s2r_power_on ? dcp_on_final : complete_set_powerstate, + cookie); + break; + } + } +} + +static void dcp_s2r_500_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); +} + +/* Unnamed. macOS sends it after D572 on every display off, with these bytes. */ +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + u32 val = 0xaaaa0000; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_unnamed_500(dcp, false, &val, dcp_s2r_500_done, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} + +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->s2r_power_on = on; + dcp->s2r_power_oob = oob; + dcp->s2r_step = named_calls ? 0 : ARRAY_SIZE(dcp_s2r_seq) - 1; + dcp_s2r_run(dcp, NULL, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} +#endif + static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { /* H17P does not confirm that scanout has stopped. */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 3bd2b0d2c214b..93e8895e9fb48 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -389,5 +389,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); +#if DCP_FW_H17(DCP_FW_VER) +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp); +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob); +#endif #endif diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index db13d06336b3c..c32b1bf5f055d 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/sleep.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index f3707ec05066a..19cbc189c1074 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -231,12 +231,80 @@ asahi_of_device_table!( #[cfg(CONFIG_DRM_ASAHI_G15)] asahi_of_device_table!(); +struct SleepOps(bindings::dev_pm_ops); + +// SAFETY: Immutable: the callbacks are plain functions. +unsafe impl Sync for SleepOps {} + +/// # Safety +/// +/// `dev` must be a device this driver is bound to, as for a PM core callback. +unsafe fn sleep_data<'a>(dev: *mut bindings::device) -> &'a AsahiPlatformDriverData<'a> { + // SAFETY: Probe stored a `Pin>` here, and the PM core keeps the + // device bound for the duration of a callback. + unsafe { &*bindings::dev_get_drvdata(dev).cast() } +} + +/// Runs before any device suspends, so the GPU goes first and work still in flight can finish. +unsafe extern "C" fn sleep_prepare(dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: Called by the PM core. + let data = unsafe { sleep_data(dev) }; + match data.g17.as_ref() { + Some(gpu) => kernel::error::from_result(|| gpu.sleep_prepare().map(|_| 0)), + None => EBUSY.to_errno(), + } +} + +/// Runs after every device resumed. Does nothing unless [`sleep_prepare`] stopped the firmware. +unsafe extern "C" fn sleep_complete(dev: *mut bindings::device) { + // SAFETY: Called by the PM core. + if let Some(gpu) = unsafe { sleep_data(dev) }.g17.as_ref() { + gpu.sleep_complete(); + } +} + +static SLEEP_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_prepare), + complete: Some(sleep_complete), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// For a driver with no firmware restart: a sleep that powers the SoC down would lose its firmware. +unsafe extern "C" fn sleep_refuse(_dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if unsafe { bindings::pm_suspend_via_firmware() } { + EBUSY.to_errno() + } else { + 0 + } +} + +static SLEEP_REFUSE_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_refuse), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// The callbacks of a driver that refuses a sleep that powers the SoC down. +pub(crate) fn sleep_refuse_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_REFUSE_OPS.0) +} + /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { type IdInfo = ProbeConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_OPS.0) + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { // SAFETY: Takes no arguments, process context. Clears the T6001 AGX request in the PMP report if the pmp_report.agx_vote // parameter set it; a no-op everywhere else. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 9ffd858598b1c..ee9708c88cee5 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -179,6 +179,10 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(driver) = &this._driver { driver.close_binding(); diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 4cf9057f04fb0..8ab2c88244fbf 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -145,6 +145,10 @@ impl platform::Driver for Driver { type Data<'bound> = Option>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index df5a31f621a3c..96d02e0d3c658 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -57,6 +57,7 @@ mod recovery; mod replay; mod reports; mod runtime; +mod sleep; pub(crate) mod stats; pub(crate) mod status; mod teardown; @@ -191,6 +192,9 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, + /// The firmware's warm restart across a system sleep is known for this SoC. Otherwise the + /// driver refuses a sleep that powers the SoC down. + pub(crate) sleep_restart: bool, /// The SoC's values of region C. pub(crate) region_c: &'static fw::initdata::RegionCConfig, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, @@ -531,7 +535,11 @@ impl Coprocessor { ) -> Result> { for attempt in 0..RECEIVER_ATTEMPTS { match rtkit::RtKit::::new(dev.as_ref().as_ref(), None, index, state.clone()) { - Ok(receiver) => return Ok(receiver), + Ok(receiver) => { + // Its OSLOG counters stay unanswered until a soak shows echoing them is safe. + receiver.set_oslog_echo(false); + return Ok(receiver); + } Err(error) if error == EINVAL || error == ETIMEDOUT => { if attempt + 1 < RECEIVER_ATTEMPTS { fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); @@ -573,6 +581,15 @@ impl Coprocessor { self.rtkit.send(EP_DOORBELL, msg) } + /// Starts the CPU of firmware that lost power with its state intact, and waits for RTKit to + /// come up. No endpoint starts and no initialization: it already has both. + fn restart(&mut self) -> Result { + self.rtkit.with(|rtkit| rtkit.warm_restart())?; + self.cpu.start()?; + self.running = true; + self.rtkit.with(|rtkit| rtkit.boot()) + } + /// Stops the CPU. The firmware keeps its state, but no longer runs. fn stop(&mut self) { if self.running { @@ -580,6 +597,14 @@ impl Coprocessor { self.running = false; } } + + /// The books for a CPU that the sleep's commit stopped, with no register access. + fn stopped_at_commit(&mut self) { + if self.running { + self.cpu.stopped_elsewhere(); + self.running = false; + } + } } impl Drop for Coprocessor { @@ -612,6 +637,44 @@ impl Bridge { } } + /// The bridge across a power-off of the GPU's domain. Nothing on a target that runs without + /// it. Teardown still clears it afterwards. + fn set(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + return self.regs.enable_g17_bridge(); + } + self.regs.disable_g17_bridge(); + Ok(()) + } + + /// Core gating across a power-off of the GPU's domain. Nothing on a target that runs without + /// the bridge. + fn set_gating(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + self.regs.enable_g17_gating() + } else { + self.regs.hold_g17_gating() + } + } + + /// Both bridge words and the gating word, on a target that has them. + fn words(&self) -> Option<[u32; 3]> { + if !self.enabled { + return None; + } + self.regs.g17_fabric_words().ok() + } + + fn forget(&mut self) { + self.enabled = false; + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -686,7 +749,10 @@ impl Firmware { /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { use recovery::Memory; - if self.recovery.pending() || self.init.recovery_state()? != 0 { + if self.primary.state.shared.sleeping() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + { return Err(EAGAIN); } let target = channel::Rings::publish_control(&self.init, record)?; @@ -738,7 +804,7 @@ impl Firmware { /// Observe a telemetry epoch boundary without changing GPU power policy. fn observe_statistics_power(&mut self) { - if self.rings.statistics_suspended() { + if self.rings.statistics_suspended() || self.primary.state.shared.sleeping() { return; } let roles = self.roles(); @@ -874,6 +940,14 @@ struct Shared { compute_blocks: usize, descriptor_flags: [u32; 2], crashed: AtomicBool, + /// The firmware is stopped, or about to be: nothing may read or publish to it. + sleeping: AtomicBool, + /// A sleep holds preparations closed, so a recovery settling meanwhile doesn't reopen them. + sleep_gate: AtomicBool, + /// The power state word as it was before the firmware stopped. + sleep_ps: AtomicU32, + /// The stop the sleep's commit makes, from `.prepare` to `.complete`; null otherwise. + sleep_commit: core::sync::atomic::AtomicPtr, feed: Arc, #[pin] feed_work: Work, @@ -943,6 +1017,9 @@ impl_has_work! { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } if let Err(error) = this.grow_render() { if this.state.lock().is_some() { dev_err!( @@ -966,6 +1043,9 @@ impl WorkItem for Shared { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } let result = { let mut state = this.state.lock(); match (*state).as_deref_mut() { @@ -1258,6 +1338,10 @@ impl Gpu { compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), + sleeping: AtomicBool::new(false), + sleep_gate: AtomicBool::new(false), + sleep_ps: AtomicU32::new(0), + sleep_commit: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 425eb6f517b71..e155fa40f5bfb 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -528,6 +528,11 @@ impl Port { self.demand.set_active(full); } + /// The next statistics drain and reader discard what the decoder holds. + pub(super) fn statistics_epoch_bump(&self) { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } + pub(super) fn statistics_epoch(&self) -> u64 { self.statistics_epoch.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 81948e738769c..c9090678657e5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2240,6 +2240,10 @@ impl Pair { } } } + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } + /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. /// The steady state in which a command may publish without the device diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c..9d8c108b3cab3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -257,6 +257,12 @@ impl ParkedPair { } } +impl ParkedPair { + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } +} + impl Prepared { /// Called after the registry revalidates its preparation epoch and parked pair identity. pub(crate) fn finish_install( diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 543612cd211a6..bbbb73d63dc5a 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -104,7 +104,7 @@ fn write_power( firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>, ) -> kernel::fmt::Result { - if firmware.rings.statistics_suspended() { + if firmware.rings.statistics_suspended() || firmware.primary.state.shared.sleeping() { return writeln!(f, "power_management=firmware suspended=1"); } let core_power = firmware.bridge.regs.g17_core_power(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4af2704be0b44..0ebdb0ca972c9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -301,6 +301,57 @@ impl Firmware { Ok(()) } + /// A step of the event pass is waiting on the firmware with no event to wake it. + fn runtime_polling(&self, now: u64) -> bool { + self.render_control_backpressured() + || self.queues.replay_outer_wait.is_some() + || self.queues.render.replay_polling() + || self.queues.teardown.polling(now) + || self.compute_exit_polling(now) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()) + } + + /// Nothing is published that hasn't retired, nothing is waiting on the firmware, its rings + /// are empty, and the idle check has taken the effort and power holds down. + pub(in crate::g17) fn sleep_quiet(&self, now: u64) -> Result { + use crate::g17::recovery::Memory; + let compute = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + if compute + || self.queues.render_in_flight() + || !self.queues.render_settled() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + || self.runtime_polling(now) + || self.hold.active() + || self.registration_power_release_pending + || self.effort.applied() != 0 + { + return Ok(false); + } + let [consumer, unknown, producer] = self.init.control_counters()?; + if consumer != producer || unknown != producer { + return Ok(false); + } + for &role in self.roles() { + if channel::Events::pending(&self.init, role)? { + return Ok(false); + } + } + Ok(true) + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -377,6 +428,10 @@ impl crate::g17::Shared { /// callbacks are finished. Shutdown can detach Firmware during an off-lock /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { + // The way back queues a pass, which also handles a crash noticed meanwhile. + if self.sleeping() { + return; + } if self.crashed.load(core::sync::atomic::Ordering::Acquire) { self.stop_runtime(); return; @@ -442,7 +497,9 @@ impl crate::g17::Shared { } if recover { firmware.service_runtime_recovery(deferred)?; - if !firmware.recovery.pending() { + if !firmware.recovery.pending() + && !self.sleep_gate.load(core::sync::atomic::Ordering::Acquire) + { if !self.preparations.reopen() { return Ok(()); } @@ -518,19 +575,7 @@ impl crate::g17::Shared { self.queue_reclaims(); } } - let backpressured = firmware.render_control_backpressured(); - let polling = backpressured - || firmware.queues.replay_outer_wait.is_some() - || firmware.queues.render.replay_polling() - || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now) - || firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); + let polling = firmware.runtime_polling(now); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index fc2e01be761ae..8d99333b86e25 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -1038,6 +1038,28 @@ impl Registry { } Ok(()) } + /// No graph is being built or grown. + pub(in crate::g17) fn render_settled(&self) -> bool { + self.render + .entries + .iter() + .flatten() + .all(|entry| entry.preparing.is_none() && !entry.growing) + } + + /// The firmware restarted with no power on the cores: nothing it was told about a pair through + /// their registers can be relied on, so the next command installs it again. + pub(in crate::g17) fn forget_render_registrations(&mut self) { + for entry in self.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + pair.forget_registration(); + } + if let Some(parked) = entry.parked.as_mut() { + parked.forget_registration(); + } + } + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c new file mode 100644 index 0000000000000..57af9cc3dfef0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The GPU's power state word across a system sleep. The domain is always on as far + * as genpd knows, so nothing else lowers it. The word and its parent come from the + * device tree. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sleep.h" + +#define PS_AUTO_ENABLE BIT(28) +#define PS_AUTO GENMASK(27, 24) +#define PS_BUSY BIT(11) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTIVE 0xf +#define PS_POLL_US 100 +#define PS_TIMEOUT_US 50000 + +/* In the "sgx" range, as regs.rs has them. */ +#define SGX_GATING 0xd06030 +#define SGX_GATING_HOLD BIT(2) +#define SGX_BRIDGE 0x1000104 +#define SGX_BRIDGE_WORDS 2 +#define SGX_BRIDGE_ENABLE BIT(0) + +#define ASC_CPU_STATUS 0x4 + +/* The PMP's request for the GPU, where the device tree names one. */ +int asahi_g17_pmp_request(struct device *dev, bool enable) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (!np) + return 0; + ret = apple_pmp_report_entry_request(np, enable); + of_node_put(np); + + return ret; +} + +static int asahi_g17_ps(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -EINVAL; + ret = of_property_read_u32(np, "reg", offset); + if (!ret) { + *regmap = np->parent ? syscon_node_to_regmap(np->parent) : ERR_PTR(-EINVAL); + ret = PTR_ERR_OR_ZERO(*regmap); + } + of_node_put(np); + + return ret; +} + +/* Automatic power management off. Returns the word as it was in @saved. */ +int asahi_g17_power_auto_off(struct device *dev, u32 *saved) +{ + struct regmap *regmap; + u32 offset; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, saved); + if (ret) + return ret; + + return regmap_write(regmap, offset, *saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS)); +} + +/* The target off, then wait for the domain to finish going off. */ +int asahi_g17_power_lower(struct device *dev) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + regmap_write(regmap, offset, + (val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS); + + ret = regmap_read_poll_timeout(regmap, offset, cur, + !FIELD_GET(PS_ACTUAL, cur) && !(cur & PS_BUSY), + PS_POLL_US, PS_TIMEOUT_US); + dev_dbg(dev, "GPU domain %#x, now %#x\n", val, cur); + + return ret; +} + +static u32 asahi_g17_ps_on(u32 val, u32 saved) +{ + val &= ~(PS_AUTO_ENABLE | PS_AUTO | PS_BUSY | PS_FLAGS | PS_ACTUAL | PS_TARGET); + + return val | (saved & PS_AUTO) | FIELD_PREP(PS_TARGET, PS_ACTIVE); +} + +/* The target on, and wait for it. */ +int asahi_g17_power_raise(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + dev_dbg(dev, "GPU domain before raising it: %#x\n", val); + + regmap_write(regmap, offset, asahi_g17_ps_on(val, saved)); + + return regmap_read_poll_timeout(regmap, offset, cur, + FIELD_GET(PS_ACTUAL, cur) == PS_ACTIVE, + PS_POLL_US, PS_TIMEOUT_US); +} + +/* Automatic power management as it was. */ +int asahi_g17_power_auto_on(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + return regmap_write(regmap, offset, asahi_g17_ps_on(val, saved) | (saved & PS_AUTO_ENABLE)); +} + +/* + * What the sleep's commit needs to stop the GPU with interrupts off, taken in .prepare, and + * what it saw, for the log after the sleep. + */ +struct asahi_g17_commit { + struct device *dev; + unsigned int count; + struct apple_rtkit *rtk[2]; + struct apple_mbox *mbox[2]; + void __iomem *cpu[2]; + void __iomem *gating; + void __iomem *bridge; + void __iomem *ps; + struct device_node *request; + + bool ran; + int idle[2]; + unsigned int idle_us[2]; + u32 status[2]; + u32 words[SGX_BRIDGE_WORDS + 1]; + int request_ret; + u32 saved; + u32 lowered; + int lower_ret; +}; + +/* + * down() at the commit, every refusal behind it: both instances idled by polling, mailboxes + * and run bits off, the gating hold, the PMP request withdrawn while the PMP still runs, then + * the bridge and the domain. Told to idle, the firmware can't carry on, so the sleep goes on + * whatever it answers. + */ +static int asahi_g17_commit_stop(void *data) +{ + struct asahi_g17_commit *c = data; + unsigned int i; + u32 val; + + c->ran = true; + for (i = 0; i < c->count; i++) + c->idle[i] = apple_rtkit_sleep_idle_polled(c->rtk[i], c->cpu[i], &c->idle_us[i]); + for (i = 0; i < c->count; i++) { + c->status[i] = readl(c->cpu[i] + ASC_CPU_STATUS); + apple_rtkit_mailbox_recv_off(c->rtk[i]); + apple_mbox_ascwrap_v6_stop_cpu_atomic(c->mbox[i]); + if (!i && c->bridge) + writel(readl(c->gating) | SGX_GATING_HOLD, c->gating); + } + c->request_ret = c->request ? apple_pmp_report_entry_withdraw_polled(c->request) : 0; + + c->saved = readl(c->ps); + writel(c->saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS), c->ps); + if (c->bridge) { + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + c->words[i] = readl(c->bridge + 4 * i); + c->words[i] = readl(c->gating); + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + writel(c->words[i] & ~SGX_BRIDGE_ENABLE, c->bridge + 4 * i); + } + val = readl(c->ps); + writel((val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS, c->ps); + c->lower_ret = readl_poll_timeout_atomic(c->ps, c->lowered, + !FIELD_GET(PS_ACTUAL, c->lowered) && + !(c->lowered & PS_BUSY), + 10, PS_TIMEOUT_US); + + return (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) ? 1 : 0; +} + +static void __iomem *asahi_g17_map(struct device_node *np, const char *name, int index, + u32 offset, u32 size) +{ + struct resource res; + + if (name) + index = of_property_match_string(np, "reg-names", name); + if (index < 0 || of_address_to_resource(np, index, &res) || + offset + size > resource_size(&res)) + return NULL; + + return ioremap(res.start + offset, size); +} + +static void asahi_g17_commit_free(struct asahi_g17_commit *c) +{ + if (c->request) { + apple_pmp_report_entry_commit_owned(c->request, false); + of_node_put(c->request); + } + if (c->ps) + iounmap(c->ps); + if (c->bridge) + iounmap(c->bridge); + if (c->gating) + iounmap(c->gating); + kfree(c); +} + +/* + * Leaves the GPU's whole stop to the sleep's commit: maps what the stop touches, keeps the + * PMP request for it, and registers it with the sleep driver, ready. Nothing is stopped + * here. @secondary_rtk is NULL on a single instance, and @bridge false on a target that runs + * without the bridge. + */ +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge) +{ + struct asahi_g17_commit *c; + struct device_node *np; + unsigned int i; + u32 offset; + int ret; + + c = kzalloc_obj(*c); + if (!c) + return ERR_PTR(-ENOMEM); + c->dev = dev; + c->rtk[0] = primary_rtk; + c->mbox[0] = primary_mbox; + c->rtk[1] = secondary_rtk; + c->mbox[1] = secondary_mbox; + c->count = secondary_rtk ? 2 : 1; + + ret = -EOPNOTSUPP; + for (i = 0; i < c->count; i++) { + c->cpu[i] = apple_mbox_ascwrap_v6_cpu_control(c->mbox[i]); + if (!c->rtk[i] || !c->cpu[i]) + goto err; + } + + ret = -ENODEV; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + goto err; + if (!of_property_read_u32(np, "reg", &offset) && np->parent) + c->ps = asahi_g17_map(np->parent, NULL, 0, offset, 4); + of_node_put(np); + c->gating = asahi_g17_map(dev->of_node, "sgx", 0, SGX_GATING, 4); + if (bridge) + c->bridge = asahi_g17_map(dev->of_node, "sgx", 0, SGX_BRIDGE, + 4 * SGX_BRIDGE_WORDS); + if (!c->ps || !c->gating || (bridge && !c->bridge)) + goto err; + + c->request = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (c->request) + apple_pmp_report_entry_commit_owned(c->request, true); + + ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + if (ret) + goto err; + apple_sleep_coproc_set_ready(dev, true); + + return c; + +err: + asahi_g17_commit_free(c); + return ERR_PTR(ret); +} + +/* + * After the sleep, refused or not: whether the commit stopped the GPU, and in @saved the + * domain's word from before, if it did. Frees @c. + */ +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved) +{ + bool stopped = apple_sleep_coproc_stopped(c->dev) || c->ran; + + apple_sleep_coproc_unregister(c->dev); + if (c->ran) { + if (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) + dev_warn(c->dev, "GPU stop at the commit: idle %d %d, PMP request %d, domain %#x (%d)\n", + c->idle[0], c->idle[1], c->request_ret, c->lowered, c->lower_ret); + dev_dbg(c->dev, "GPU stopped at the commit: idle in %u %u us, status %#x %#x\n", + c->idle_us[0], c->idle_us[1], c->status[0], c->status[1]); + dev_dbg(c->dev, "GPU at the commit: bridge %#x %#x gating %#x, domain %#x now %#x\n", + c->words[0], c->words[1], c->words[2], c->saved, c->lowered); + *saved = c->saved; + } + asahi_g17_commit_free(c); + + return stopped; +} diff --git a/drivers/gpu/drm/asahi/g17/sleep.h b/drivers/gpu/drm/asahi/g17/sleep.h new file mode 100644 index 0000000000000..21b91dfca0a2e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SLEEP_H +#define ASAHI_G17_SLEEP_H +#include + +struct apple_mbox; +struct apple_rtkit; +struct asahi_g17_commit; +struct device; + +int asahi_g17_pmp_request(struct device *dev, bool enable); +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge); +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved); +int asahi_g17_power_auto_off(struct device *dev, u32 *saved); +int asahi_g17_power_lower(struct device *dev); +int asahi_g17_power_raise(struct device *dev, u32 saved); +int asahi_g17_power_auto_on(struct device *dev, u32 saved); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sleep.rs b/drivers/gpu/drm/asahi/g17/sleep.rs new file mode 100644 index 0000000000000..0db5f0a9a3d23 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.rs @@ -0,0 +1,808 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! System sleep: the firmware goes idle, its domain goes off, and it restarts warm. +//! +//! The firmware isn't initialized again. It keeps its rings and state in DRAM, so a restart is +//! the RTKit handshake and the end of the power-on transition. Publication stays closed from +//! before the idle until after the restart, and nothing is in flight across it: work that doesn't +//! finish in time refuses the sleep. +//! +//! Order, on both instances, is what macOS does on the chip whose [`super::Config`] sets +//! `sleep_restart`. Other chips refuse. +//! +//! Only a real wake restarts the firmware. A sleep refused after the firmware stopped leaves it +//! stopped and publication closed until the next real sleep, with the domain on so that everything +//! behind it stays reachable. That sleep then only has the domain left to lower. +//! +//! Where the sleep driver takes it, `.prepare` only drains and checks for quiet, and the sleep's +//! commit makes the whole stop in C, with interrupts off and every refusal behind it. A sleep +//! refused before then finds the firmware never stopped. Elsewhere `.prepare` stops it. + +use super::{Firmware, Role, Shared}; +use core::sync::atomic::Ordering; +use kernel::{ + bindings, + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, + workqueue::{DelayedWork, Work}, +}; + +/// Time work in flight gets to finish before the sleep is refused. +const DRAIN_NS: u64 = 3_000_000_000; +const DRAIN_TICK_MS: i64 = 2; + +unsafe extern "C" { + fn asahi_g17_pmp_request(dev: *mut core::ffi::c_void, enable: bool) -> core::ffi::c_int; + fn asahi_g17_power_auto_off(dev: *mut core::ffi::c_void, saved: *mut u32) -> core::ffi::c_int; + fn asahi_g17_power_lower(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_g17_power_raise(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_power_auto_on(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_commit_prepare( + dev: *mut core::ffi::c_void, + primary_rtk: *mut bindings::apple_rtkit, + primary_mbox: *mut core::ffi::c_void, + secondary_rtk: *mut bindings::apple_rtkit, + secondary_mbox: *mut core::ffi::c_void, + bridge: bool, + ) -> *mut core::ffi::c_void; + fn asahi_g17_commit_finish(commit: *mut core::ffi::c_void, saved: *mut u32) -> bool; +} + +/// What `.complete` does after a sleep whose stop was left to the commit. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum AfterCommit { + /// Refused before the commit: the firmware never stopped, so only publication reopens. + Running, + /// Stopped at the commit, or its power lost anyway: as after [`down`]. + Stopped, +} + +fn after_commit(stopped: bool, real: bool) -> AfterCommit { + if stopped || real { + AfterCommit::Stopped + } else { + AfterCommit::Running + } +} + +/// What stopping and restarting the firmware touches, in the order [`down`] and [`up`] use it. +pub(super) trait Hardware { + fn has_secondary(&self) -> bool; + /// RTKit AP and IOP idle, each acknowledged. + fn idle(&mut self, role: Role) -> Result; + /// Back from idle with no restart, after a refusal. One that fails loses the device. + fn wake(&mut self, role: Role); + /// Mailbox off, then run bit off, for an instance that acknowledged idle. + fn stop(&mut self, role: Role); + /// The PMP's request for the GPU. It may use a device it's told is in use, so the request is + /// only ever set with the domain on. + fn request(&mut self, enable: bool); + /// Core gating as it is while the firmware runs, or held for a domain about to go off. + fn gating(&mut self, enable: bool) -> Result; + /// The GPU-to-fabric bridge. + fn bridge(&mut self, enable: bool) -> Result; + /// Automatic power management off, keeping the word for the way back. + fn auto_off(&mut self) -> Result; + /// The domain off, waiting until it has settled. + fn lower(&mut self) -> Result; + /// The domain on, waiting until it is. + fn raise(&mut self) -> Result; + /// Automatic power management as it was. + fn auto_on(&mut self) -> Result; + /// Run bit on, RTKit handshake to AP on. + fn restart(&mut self, role: Role) -> Result; + /// Ends the power-on transition, on every instance. + fn power_on_done(&mut self) -> Result; +} + +/// Stops the firmware and lowers the domain. An error leaves the firmware running, so the caller +/// refuses the sleep. Both instances idle before either stops, so that a refusal by the second +/// still has a way back. A domain that is slow to go off isn't an error here: the sleep driver +/// checks it at the commit. +pub(super) fn down(hw: &mut H) -> Result { + if let Err(error) = hw.idle(Role::Primary) { + hw.wake(Role::Primary); + return Err(error); + } + if hw.has_secondary() { + if let Err(error) = hw.idle(Role::Secondary) { + hw.wake(Role::Secondary); + hw.wake(Role::Primary); + return Err(error); + } + } + hw.stop(Role::Primary); + let _ = hw.gating(false); + if hw.has_secondary() { + hw.stop(Role::Secondary); + } + hw.request(false); + lower_domain(hw); + Ok(()) +} + +/// Lowers the domain, with the firmware stopped. +fn lower_domain(hw: &mut H) { + let _ = hw.auto_off(); + let _ = hw.bridge(false); + let _ = hw.lower(); +} + +/// Brings the domain back for firmware that stays stopped after a refused sleep. +fn hold(hw: &mut H) -> Result { + hw.raise()?; + hw.auto_on() +} + +/// Restarts the firmware. Any error loses the device: there is no other way back. +pub(super) fn up(hw: &mut H) -> Result { + hw.raise()?; + hw.bridge(true)?; + hw.auto_on()?; + hw.request(true); + hw.gating(true)?; + if hw.has_secondary() { + hw.restart(Role::Secondary)?; + } + hw.restart(Role::Primary)?; + hw.power_on_done() +} + +impl Hardware for Firmware { + fn has_secondary(&self) -> bool { + self.secondary.is_some() + } + + fn idle(&mut self, role: Role) -> Result { + self.coprocessor(role)?.rtkit.with(|rtkit| rtkit.idle()) + } + + fn wake(&mut self, role: Role) { + let result = match self.coprocessor(role) { + Ok(coprocessor) => coprocessor.rtkit.with(|rtkit| rtkit.wake()), + Err(error) => Err(error), + }; + // EINVAL: it never left the on state. + if result.is_err_and(|error| error != EINVAL) { + self.primary.state.shared.lose_device_quietly(); + } + } + + fn stop(&mut self, role: Role) { + if let Ok(coprocessor) = self.coprocessor(role) { + coprocessor.rtkit.with(|rtkit| rtkit.mailbox_off()); + coprocessor.stop(); + } + } + + fn request(&mut self, enable: bool) { + self.primary.state.shared.pmp_request(enable); + } + + fn gating(&mut self, enable: bool) -> Result { + self.bridge.set_gating(enable) + } + + fn bridge(&mut self, enable: bool) -> Result { + if let Some([first, second, gating]) = self.bridge.words() { + dev_dbg!( + self.primary.state.shared.dev.as_ref(), + "GPU before bridge {}: {:#x} {:#x} gating {:#x}\n", + if enable { "on" } else { "off" }, + first, + second, + gating + ); + } + self.bridge.set(enable) + } + + fn auto_off(&mut self) -> Result { + self.primary.state.shared.power_auto_off() + } + + fn lower(&mut self) -> Result { + self.primary.state.shared.power_lower() + } + + fn raise(&mut self) -> Result { + let result = self.primary.state.shared.power_raise(); + if result.is_err() { + // The registers aren't reachable, so teardown can't clear the bridge. + self.bridge.forget(); + } + result + } + + fn auto_on(&mut self) -> Result { + self.primary.state.shared.power_auto_on() + } + + fn restart(&mut self, role: Role) -> Result { + let shared = self.primary.state.shared.clone(); + let coprocessor = self.coprocessor(role)?; + let result = coprocessor.restart(); + if let (Err(_), Ok((control, status))) = (&result, coprocessor.cpu.state()) { + dev_err!( + shared.dev.as_ref(), + "GPU firmware {:?}: control {:#x} status {:#x}\n", + role, + control, + status + ); + } + result + } + + fn power_on_done(&mut self) -> Result { + if !self.variant.power_on_done { + return Ok(()); + } + self.primary.notify(super::MSG_POWER_ON_DONE)?; + if let Some(secondary) = self.secondary.as_ref() { + secondary.notify(super::MSG_POWER_ON_DONE)?; + } + Ok(()) + } +} + +impl Firmware { + /// Hands the sleep's commit what its stop needs. Nothing stops here. + fn leave_for_commit(&mut self) -> Result<*mut core::ffi::c_void> { + let shared = self.primary.state.shared.clone(); + let primary_rtk = self.primary.rtkit.with(|rtkit| rtkit.as_raw()); + let primary_mbox = self.primary.cpu.as_raw().cast(); + let (secondary_rtk, secondary_mbox) = match self.secondary.as_ref() { + Some(secondary) => ( + secondary.rtkit.with(|rtkit| rtkit.as_raw()), + secondary.cpu.as_raw().cast(), + ), + None => (core::ptr::null_mut(), core::ptr::null_mut()), + }; + // SAFETY: `dev` is the live GPU device. The RTKit and mailbox pointers stay valid while + // the firmware exists, and `.complete` frees the result before anything drops them. + let commit = unsafe { + asahi_g17_commit_prepare( + shared.dev.as_raw().cast(), + primary_rtk, + primary_mbox, + secondary_rtk, + secondary_mbox, + self.bridge.enabled, + ) + }; + kernel::error::from_err_ptr(commit) + } + + /// The books for both instances, which the commit stopped. + fn stopped_at_commit(&mut self) { + if let Some(secondary) = self.secondary.as_mut() { + secondary.stopped_at_commit(); + } + self.primary.stopped_at_commit(); + } + + fn coprocessor(&mut self, role: Role) -> Result<&mut super::Coprocessor> { + match role { + Role::Primary => Ok(&mut self.primary), + Role::Secondary => self.secondary.as_mut().ok_or(ENODEV), + } + } +} + +impl Shared { + pub(super) fn sleeping(&self) -> bool { + self.sleeping.load(Ordering::Acquire) + } + + /// A PMP that doesn't answer in time only gets a warning: nothing better is left to do here. + fn pmp_request(&self, enable: bool) { + // SAFETY: `dev` is the live GPU device. + let result = kernel::error::to_result(unsafe { + asahi_g17_pmp_request(self.dev.as_raw().cast(), enable) + }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "PMP request for the GPU, {}: {:?}\n", enable, error); + } + } + + fn power_auto_off(&self) -> Result { + let mut saved = 0; + // SAFETY: `dev` is the live GPU device and `saved` is writable. + let result = kernel::error::to_result(unsafe { + asahi_g17_power_auto_off(self.dev.as_raw().cast(), &mut saved) + }); + self.sleep_ps.store(saved, Ordering::Relaxed); + result + } + + fn power_lower(&self) -> Result { + // SAFETY: `dev` is the live GPU device. + let result = + kernel::error::to_result(unsafe { asahi_g17_power_lower(self.dev.as_raw().cast()) }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "GPU domain slow to go off: {:?}\n", error); + } + result + } + + fn power_raise(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { asahi_g17_power_raise(self.dev.as_raw().cast(), saved) }) + } + + fn power_auto_on(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { + asahi_g17_power_auto_on(self.dev.as_raw().cast(), saved) + }) + } + + fn cancel_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: As in `disable_timer`: a pending callback owned one Arc that it can no longer + // reclaim. Future enqueues stay possible and run into the sleeping check. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, bindings::delayed_work, work); + bindings::cancel_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: See above. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// With the sleeping flag set, joins whatever may be reading firmware state. The growth work + /// isn't joined: it may be waiting on the closed gate, and can't publish while it is closed. + fn join_event_workers(self: &Arc) { + self.cancel_timer(&self.poll); + self.cancel_timer(&self.idle); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + } + + /// Waits for what's been published to retire. The event worker keeps running meanwhile, and + /// its own idle check takes the power and effort holds down. + fn wait_quiet(self: &Arc, deadline: u64) -> Result { + loop { + if self.crashed.load(Ordering::Acquire) { + return Err(ENODEV); + } + let now = super::now_ns(); + { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + // Nothing else retries this with no render publication coming. + if firmware.registration_power_release_pending { + let _ = firmware.release_registration_power(); + } + if firmware.sleep_quiet(now)? { + return Ok(()); + } + } + if now >= deadline { + return Err(EBUSY); + } + self.queue_events(); + fsleep(Delta::from_millis(DRAIN_TICK_MS)); + } + } + + /// Undoes everything short of a stopped firmware. A builder that never finishes would keep + /// publication closed for good, and that is a lost device. + fn sleep_cancel(self: &Arc) { + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + let drained = self + .preparations + .wait_drained_until(super::now_ns().saturating_add(DRAIN_NS)); + if (drained.is_err() || !self.preparations.reopen()) + && !self.crashed.load(Ordering::Acquire) + { + self.lose_device_quietly(); + } + self.queue_events(); + } + + /// Keeps the firmware stopped after a sleep that never reached the hardware. False if there's + /// no firmware to keep, or no domain to keep it in. + fn sleep_hold(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { + return false; + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return false; + }; + if hold(firmware).is_err() { + return false; + } + dev_warn!( + self.dev.as_ref(), + "sleep refused: GPU firmware stays stopped until a real sleep\n" + ); + true + } + + /// Leaves the stop to the commit. False, with the reason logged, if it can't be. + fn sleep_leave_for_commit(&self, firmware: &mut Firmware) -> bool { + match firmware.leave_for_commit() { + Ok(commit) => { + self.sleep_commit.store(commit, Ordering::Release); + dev_dbg!(self.dev.as_ref(), "GPU firmware left running for the sleep's commit\n"); + true + } + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "no GPU stop at the commit ({:?}): stopping it now\n", + error + ); + false + } + } + } + + /// Whether the commit stopped the firmware, if the stop was left to it. + fn sleep_commit_finish(&self) -> Option { + let commit = self.sleep_commit.swap(core::ptr::null_mut(), Ordering::AcqRel); + if commit.is_null() { + return None; + } + let mut saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `commit` came from `asahi_g17_commit_prepare` and is finished only once. + let stopped = unsafe { asahi_g17_commit_finish(commit, &mut saved) }; + self.sleep_ps.store(saved, Ordering::Relaxed); + Some(stopped) + } + + /// After the commit stopped the firmware: the books it left, as [`down`] would have them. + fn sleep_settle_commit(&self) { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.stopped_at_commit(); + } + } + + /// Closes publication, waits for quiet, and leaves the stop to the sleep's commit where the + /// sleep driver takes it, or stops the firmware. A refusal changes nothing. + fn sleep_down(self: &Arc) -> Result { + if self.sleeping() { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + lower_domain(firmware); + } + return Ok(()); + } + let deadline = super::now_ns().saturating_add(DRAIN_NS); + self.sleep_gate.store(true, Ordering::Release); + self.preparations.close(); + let result = self.preparations.wait_drained_until(deadline).map_err(|_| EBUSY) + .and_then(|_| self.wait_quiet(deadline)) + .and_then(|_| { + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !firmware.sleep_quiet(super::now_ns())? { + return Err(EBUSY); + } + if self.sleep_leave_for_commit(firmware) { + return Ok(()); + } + down(firmware) + }); + match result { + Ok(()) => Ok(()), + Err(error) if error == ENODEV || self.crashed.load(Ordering::Acquire) => { + // Nothing to save: let the worker finish detaching what's left, and then the + // domain only has to be off for the sleep. + self.sleeping.store(false, Ordering::Release); + self.queue_events(); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + self.pmp_request(false); + let _ = self.power_auto_off(); + let _ = self.power_lower(); + Ok(()) + } + Err(error) => { + self.sleep_cancel(); + Err(error) + } + } + } + + /// Restarts the firmware and reopens publication. A failure loses the device. + fn sleep_up(self: &Arc) -> Result { + let started = super::now_ns(); + let result = { + let mut state = self.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if !self.crashed.load(Ordering::Acquire) => { + let result = up(firmware); + if result.is_ok() { + // The cores lost power, so the next command on each pair installs it again. + firmware.queues.forget_render_registrations(); + firmware.rings.invalidate_statistics(); + firmware.port.statistics_epoch_bump(); + } + result + } + _ => self + .power_raise() + .and_then(|_| self.power_auto_on()) + .inspect(|_| self.pmp_request(true)), + } + }; + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + match result { + Ok(()) => { + if !self.crashed.load(Ordering::Acquire) { + dev_dbg!( + self.dev.as_ref(), + "GPU firmware back in {} us\n", + super::now_ns().saturating_sub(started) / 1000 + ); + } + self.preparations.reopen(); + self.queue_events(); + Ok(()) + } + Err(error) => { + dev_err!(self.dev.as_ref(), "GPU firmware did not come back: {:?}\n", error); + self.lose_device_quietly(); + Err(error) + } + } + } +} + +impl super::Gpu { + /// Called before any device suspends. Only a sleep that powers the SoC down gets here. + pub(crate) fn sleep_prepare(&self) -> Result { + if !self.cfg.sleep_restart { + dev_warn!( + self.shared.dev.as_ref(), + "no firmware restart on this chip, refusing sleep\n" + ); + return Err(EBUSY); + } + let result = self.shared.sleep_down(); + if let Err(error) = result { + dev_warn!(self.shared.dev.as_ref(), "GPU refused sleep: {:?}\n", error); + } + result + } + + /// Called after every device resumed, whether or not the sleep reached the hardware. + pub(crate) fn sleep_complete(&self) { + if !self.shared.sleeping() { + return; + } + // SAFETY: Reads a global flag word. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if let Some(stopped) = self.shared.sleep_commit_finish() { + if after_commit(stopped, real) == AfterCommit::Running { + dev_info!( + self.shared.dev.as_ref(), + "sleep refused before the commit: GPU firmware never stopped\n" + ); + self.shared.sleep_cancel(); + return; + } + self.shared.sleep_settle_commit(); + } + if !real && self.shared.sleep_hold() { + return; + } + let _ = self.shared.sleep_up(); + } +} + +#[kernel::macros::kunit_tests(asahi_g17_sleep)] +mod tests { + use super::*; + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + enum Call { + Idle(Role), + Wake(Role), + Stop(Role), + Request(bool), + Gating(bool), + Bridge(bool), + AutoOff, + Lower, + Raise, + AutoOn, + Restart(Role), + Done, + } + + struct Fake { + secondary: bool, + fail: Option, + calls: [Option; 14], + count: usize, + } + + impl Fake { + fn new(secondary: bool, fail: Option) -> Self { + Self { secondary, fail, calls: [None; 14], count: 0 } + } + + fn note(&mut self, call: Call) -> Result { + self.calls[self.count] = Some(call); + self.count += 1; + if self.fail == Some(call) { Err(EIO) } else { Ok(()) } + } + + fn called(&self, expected: &[Call]) -> bool { + self.count == expected.len() + && expected.iter().zip(self.calls.iter()).all(|(a, b)| Some(*a) == *b) + } + } + + impl Hardware for Fake { + fn has_secondary(&self) -> bool { self.secondary } + fn idle(&mut self, role: Role) -> Result { self.note(Call::Idle(role)) } + fn wake(&mut self, role: Role) { let _ = self.note(Call::Wake(role)); } + fn stop(&mut self, role: Role) { let _ = self.note(Call::Stop(role)); } + fn request(&mut self, enable: bool) { let _ = self.note(Call::Request(enable)); } + fn gating(&mut self, enable: bool) -> Result { self.note(Call::Gating(enable)) } + fn bridge(&mut self, enable: bool) -> Result { self.note(Call::Bridge(enable)) } + fn auto_off(&mut self) -> Result { self.note(Call::AutoOff) } + fn lower(&mut self) -> Result { self.note(Call::Lower) } + fn raise(&mut self) -> Result { self.note(Call::Raise) } + fn auto_on(&mut self) -> Result { self.note(Call::AutoOn) } + fn restart(&mut self, role: Role) -> Result { self.note(Call::Restart(role)) } + fn power_on_done(&mut self) -> Result { self.note(Call::Done) } + } + + use Role::{Primary as P, Secondary as S}; + + #[test] + fn down_idles_the_primary_first_and_lowers_the_domain_last() { + let mut hw = Fake::new(true, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Idle(S), + Call::Stop(P), + Call::Gating(false), + Call::Stop(S), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + } + + #[test] + fn up_restarts_the_secondary_first_and_ends_the_transition_last() { + let mut hw = Fake::new(true, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_single_instance_never_names_a_secondary() { + let mut hw = Fake::new(false, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Stop(P), + Call::Gating(false), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + let mut hw = Fake::new(false, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_refused_idle_touches_nothing_past_the_firmware() { + for call in [Call::Idle(P), Call::Idle(S)] { + let mut hw = Fake::new(true, Some(call)); + assert_eq!(down(&mut hw), Err(EIO)); + let past = |c: &Call| !matches!(c, Call::Idle(_) | Call::Wake(_)); + assert!(!hw.calls.iter().flatten().any(past)); + } + } + + #[test] + fn a_primary_that_refuses_idle_is_woken_and_nothing_stops() { + let mut hw = Fake::new(true, Some(Call::Idle(P))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Wake(P)])); + } + + #[test] + fn a_secondary_that_refuses_idle_wakes_both_and_stops_nothing() { + let mut hw = Fake::new(true, Some(Call::Idle(S))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Idle(S), Call::Wake(S), Call::Wake(P)])); + } + + #[test] + fn a_refused_sleep_only_brings_the_domain_back() { + let mut hw = Fake::new(true, None); + assert_eq!(hold(&mut hw), Ok(())); + assert!(hw.called(&[Call::Raise, Call::AutoOn])); + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(hold(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + } + + #[test] + fn the_sleep_after_a_refusal_only_lowers_the_domain() { + let mut hw = Fake::new(true, None); + lower_domain(&mut hw); + assert!(hw.called(&[Call::AutoOff, Call::Bridge(false), Call::Lower])); + } + + #[test] + fn a_slow_domain_does_not_fail_down() { + let mut hw = Fake::new(false, Some(Call::Lower)); + assert_eq!(down(&mut hw), Ok(())); + } + + #[test] + fn up_stops_at_the_first_failure() { + let mut hw = Fake::new(true, Some(Call::Restart(S))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + ])); + // No domain, so no bridge and no request either. + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + // No bridge, so the PMP isn't told about the GPU. + let mut hw = Fake::new(true, Some(Call::Bridge(true))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise, Call::Bridge(true)])); + } + + #[test] + fn a_stop_left_to_the_commit_is_undone_only_if_it_never_ran() { + assert_eq!(after_commit(false, false), AfterCommit::Running); + assert_eq!(after_commit(true, false), AfterCommit::Stopped); + assert_eq!(after_commit(true, true), AfterCommit::Stopped); + // No stop, but the power went anyway: the firmware has to restart. + assert_eq!(after_commit(false, true), AfterCommit::Stopped); + } + +} diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 5f6b9944a811e..7f7f7f485176d 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -55,6 +55,7 @@ pub(crate) const CONFIG: Config = Config { chip: None, adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on // the firmware's own blame. recovery_slot_keys: None, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 956aa0cc7728b..df2327a9a3119 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -46,6 +46,7 @@ const BASE: Config = Config { chip: Some(&CHIP), adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G16, firmware: fw::variant::Firmware::Identified(&[]), diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 02381c2aae344..0461fd6a6b8eb 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -54,6 +54,7 @@ pub(crate) const CONFIG: Config = Config { chip: None, adopt_root_entries: true, fabric_bridge: true, + sleep_restart: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 976c789930068..4b6af520956c1 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -80,6 +80,7 @@ const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) /// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. const G17_GATING_CONTROL: usize = 0xd06030; const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) +const G17_GATING_HOLD: u32 = 0x4; // BIT(2) /// These performance selectors must be sampled before firmware can gate the core registers. const G17_PERF_CONTROL: usize = 0xe0141c; @@ -281,6 +282,37 @@ impl CpuControl { } } } + + /// The provider, for a system sleep's last step to stop the CPU with interrupts off. + pub(crate) fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + + /// Settles the provider's books for a CPU that a system sleep's last step stopped. + pub(crate) fn stopped_elsewhere(&self) { + // SAFETY: The constructor established the live, role-checked provider. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_cpu_stopped(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!( + self.dev.as_ref(), + "Could not settle a stopped GPU coprocessor: {:?}\n", + error + ); + } + } + } + + /// The CPU's control and status words. + pub(crate) fn state(&self) -> Result<(u32, u32)> { + let (mut control, mut status) = (0, 0); + // SAFETY: The constructor established the live, role-checked provider, and both pointers + // are writable. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_cpu_state(self.mbox, &mut control, &mut status) + })?; + Ok((control, status)) + } } /// Device resources for this GPU instance. @@ -827,6 +859,25 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } + + /// Hold part of the gating off, for a domain about to go off. The firmware must be idle. + pub(crate) fn hold_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val | G17_GATING_HOLD, G17_GATING_CONTROL) + } + + /// Both bridge words and the gating word. + pub(crate) fn g17_fabric_words(&self) -> Result<[u32; 3]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + Ok([ + sgx.try_read32(G17_BRIDGE_CONTROL[0])?, + sgx.try_read32(G17_BRIDGE_CONTROL[1])?, + sgx.try_read32(G17_GATING_CONTROL)?, + ]) + } } /// Split the packed per-cluster core masks into one mask per cluster. diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index a752c224959de..68a31b88c23dc 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include "../hid-ids.h" @@ -86,6 +87,9 @@ struct dchid_init_hdr { #define CMD_REGISTER_RING 0x91 #define CMD_SEND_FIRMWARE 0x95 #define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_HOST_STATE 0xc1 +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 #define CMD_ACK_GPIO_CMD 0xa1 struct dchid_init_block_hdr { @@ -202,6 +206,8 @@ struct dockchannel_hid { bool inherited_mtp; /* Between the system sleep suspend and resume callbacks */ bool suspended; + /* The firmware was told of a sleep and has not been told of the wake */ + bool host_asleep; /* Held from probe on when the multi-touch interface uses power method 2 */ struct gpio_desc *afe_reset; @@ -447,6 +453,14 @@ static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); } +static int dchid_host_state(struct dockchannel_hid *dchid, u8 state) +{ + u8 msg[] = { CMD_HOST_STATE, state }; + int ret = dchid_comm_cmd(dchid, msg, sizeof(msg)); + + return ret < 0 ? ret : 0; +} + static int dchid_enable_interface(struct dchid_iface *iface) { u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; @@ -513,7 +527,7 @@ static int dchid_register_ring(struct dockchannel_hid *dchid) .size = DCHID_RING_SIZE, }; /* Observed in both macOS and standalone T8132 bringup before 0x91. */ - u8 prepare[] = { 0xc1, 0x02 }; + u8 prepare[] = { CMD_HOST_STATE, HOST_STATE_AWAKE }; int ret = 0; if (!dchid->use_ring) @@ -1768,10 +1782,13 @@ static void dockchannel_hid_remove(struct platform_device *pdev) } /* - * The coprocessor keeps running across suspend-to-idle: there is no known - * sleep command for it, and its RTKit session and the trackpad AFE power - * sequence are not restartable. Only mark the sleep window, in which input - * may wake the system and requests from the HID core fail fast. + * The coprocessor keeps running across suspend-to-idle, and its RTKit session + * and the trackpad AFE power sequence are not restartable. Mark the sleep + * window, in which input may wake the system and requests from the HID core + * fail fast. A firmware-driven suspend also sets the host state, after the idle + * notice the helper sends in its .prepare, and the wake sets it back after the + * helper's .resume has sent the on notice, the device link ordering the two. That + * is macOS's order. */ static int dockchannel_hid_suspend(struct device *dev) { @@ -1788,6 +1805,13 @@ static int dockchannel_hid_suspend(struct device *dev) if (ret) goto unwind; } + if (pm_suspend_via_firmware()) { + /* A firmware that doesn't answer isn't a reason to stay awake. */ + ret = dchid_host_state(dchid, HOST_STATE_SLEEP); + if (ret) + dev_warn(dev, "host state not set: %d\n", ret); + dchid->host_asleep = !ret; + } WRITE_ONCE(dchid->suspended, true); return 0; @@ -1807,6 +1831,14 @@ static int dockchannel_hid_resume(struct device *dev) struct dockchannel_hid *dchid = dev_get_drvdata(dev); int i, ret = 0; + if (dchid->host_asleep) { + dchid->host_asleep = false; + ret = dchid_host_state(dchid, HOST_STATE_AWAKE); + if (ret) + dev_warn(dev, "host state not restored: %d\n", ret); + ret = 0; + } + /* Allow feature requests again before notifying any HID driver. */ WRITE_ONCE(dchid->suspended, false); for (i = 0; i < MAX_INTERFACES; i++) { diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 853dfa73b7b8b..fb3fe3fdb6920 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -13,6 +13,7 @@ #include #include #include +#include /** * struct macsmc_input @@ -20,14 +21,18 @@ * @smc: Pointer to apple_smc struct of the mfd parent * @input: Allocated input_dev; devres managed * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) + * @pm_nb: Notifier block for the start and end of a system suspend * @wakeup_mode: Set to true when system is suspended and power button events should wake it + * @dark: Going back to sleep from a dark wake: the lid is read again before it */ struct macsmc_input { struct device *dev; struct apple_smc *smc; struct input_dev *input; struct notifier_block nb; + struct notifier_block pm_nb; bool wakeup_mode; + bool dark; }; #define SMC_EV_BTN 0x7201 @@ -84,6 +89,10 @@ static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long eve u8 lid_state = !!((event >> 8) & 0xff); pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + /* Through a sleep, .complete reports where the lid ended up. */ + if (smcin->wakeup_mode) + return; + input_report_switch(smcin->input, SW_LID, lid_state); input_sync(smcin->input); } @@ -106,6 +115,40 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +/* + * A suspend's first steps run before .prepare, and the SMC reports the lid and the button as + * plain events until its AP idle notice. Treat them as wakes from the suspend's start, so a lid + * opened or a button pressed while the machine goes down ends the suspend, as on macOS. + */ +static int macsmc_input_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, pm_nb); + + if (action == PM_SUSPEND_PREPARE) { + WRITE_ONCE(smcin->wakeup_mode, true); + } else if (action == PM_POST_SUSPEND) { + u8 val; + + WRITE_ONCE(smcin->wakeup_mode, false); + smcin->dark = false; + /* A suspend that ended before .prepare held its lid events back too. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { + input_report_switch(smcin->input, SW_LID, val); + input_sync(smcin->input); + } + } + + return NOTIFY_DONE; +} + +static void macsmc_input_unregister_pm_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + unregister_pm_notifier(&smcin->pm_nb); +} + static void macsmc_input_unregister_notifier(void *data) { struct macsmc_input *smcin = data; @@ -198,14 +241,29 @@ static int macsmc_input_probe(struct platform_device *pdev) if (error) return error; - return 0; + smcin->pm_nb.notifier_call = macsmc_input_pm_notify; + error = register_pm_notifier(&smcin->pm_nb); + if (error) + return error; + + return devm_add_action_or_reset(&pdev->dev, macsmc_input_unregister_pm_notifier, smcin); } static int macsmc_input_pm_prepare(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); + u8 val; smcin->wakeup_mode = true; + /* A lid opened as the dark wake ended may have sent its event already. */ + if (smcin->dark) { + smcin->dark = false; + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0 && !val) { + dev_info(dev, "the lid opened during a dark wake: not sleeping again\n"); + return -EBUSY; + } + } return 0; } @@ -213,7 +271,22 @@ static void macsmc_input_pm_complete(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); + /* Still armed: a lid opened or a button pressed before the next .prepare wakes. */ + if (pm_dark_wake()) { + smcin->dark = true; + return; + } smcin->wakeup_mode = false; + + /* The lid may have moved while the AP was off, or its events were held back. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD))) { + u8 val; + + if (apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { + input_report_switch(smcin->input, SW_LID, val); + input_sync(smcin->input); + } + } } static const struct dev_pm_ops macsmc_input_pm_ops = { diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index e9929282d0f78..5f54c1ec7eb31 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -305,6 +305,11 @@ struct apple_dart_fw_root { u64 *accepted; }; +struct apple_dart_sleep_list { + const u32 *pairs; + unsigned int count; +}; + /* * Private structure associated with each DART device. * @@ -321,11 +326,29 @@ struct apple_dart_fw_root { * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ +#define APPLE_DART_GAPF_PROTECT BIT(4) + struct apple_dart { struct device *dev; const struct apple_dart_hw *hw; void __iomem *regs; + void __iomem *gapf; + /* The GAPF slice as it read at probe; a deep sleep resets it to 0 */ + u32 gapf_boot; + bool sleep_flushed; + /* + * A display DART whose streams are off for a power-down: its registers may be + * unpowered, so maps use TCR as it was, and invalidations wait for the streams. + */ + bool display_dark; + bool display_tlb_stale; + DECLARE_BITMAP(display_tcr_four, DART_MAX_STREAMS); + /* The address filter the loader programs, from the device tree, replayed after a sleep. */ + void __iomem *dapf; + u64 *dapf_ranges; + u32 *dapf_r20; + unsigned int dapf_count; int irq; struct clk_bulk_data *clks; @@ -344,15 +367,21 @@ struct apple_dart { u32 per_stream : 1; u32 isp_host : 1; u32 power_retained : 1; + /* State a consumer saved for a firmware-driven suspend and will restore. */ + u32 sleep_saved : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; + /* Clocked by its host's port: the host resumes it after a real sleep. */ + u32 host_resumes : 1; bool commands_gated; /* Resume restored the translations, but the last steps of its reset failed. */ bool reset_incomplete; struct apple_tunable *tunables; + /* The bootloader sets these; written back where a power-off lost them. */ + struct apple_tunable *resume_tunables; u16 version; /* @@ -407,6 +436,14 @@ struct apple_dart { unsigned int pm_users; wait_queue_head_t pm_wq; u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ + /* The register writes around a firmware-driven suspend, from the device tree. */ + struct apple_dart_sleep_list sleep_prepare; + struct apple_dart_sleep_list sleep_restore; + struct apple_dart_sleep_list sleep_restore_late; + u32 sleep_enable[BITS_TO_U32(DART_MAX_STREAMS)]; + /* Its own config words at the sleep, which the restore keeps. */ + u32 sleep_cfg[20]; + /* Retained radio/display state left in place across suspend-to-idle. */ bool s2idle_kept; /* Resources reachable by a retained domain outlive device removal. */ @@ -638,21 +675,27 @@ static inline u32 apple_dart_readl(struct apple_dart *dart, u32 offset) return value; } -static void apple_dart_apply_tunables(struct apple_dart *dart) +/* Returns how many words it had to change. */ +static unsigned int apple_dart_apply_tunables(struct apple_dart *dart, + const struct apple_tunable *tunables) { + unsigned int changed = 0; size_t i; - for (i = 0; i < dart->tunables->sz; ++i) { + for (i = 0; i < tunables->sz; ++i) { u32 old_value, value; - old_value = apple_dart_readl(dart, - dart->tunables->values[i].offset); - value = old_value & ~dart->tunables->values[i].mask; - value |= dart->tunables->values[i].value; - if (value != old_value) + old_value = apple_dart_readl(dart, tunables->values[i].offset); + value = old_value & ~tunables->values[i].mask; + value |= tunables->values[i].value; + if (value != old_value) { apple_dart_writel(dart, value, - dart->tunables->values[i].offset); + tunables->values[i].offset); + changed++; + } } + + return changed; } /* @@ -2276,6 +2319,17 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) continue; + /* + * Its display is powered off: publish to memory only. With its streams off + * nothing walks the tables, and the TLB is invalidated before they're on. + */ + if (READ_ONCE(stream.dart->display_dark)) { + ret = stream.dart->locked ? + apple_dart_hw_sync_locked(cfg, &stream, true) : 0; + WRITE_ONCE(stream.dart->display_tlb_stale, true); + err = err ?: ret; + continue; + } ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { dev_err_ratelimited(stream.dart->dev, @@ -2406,7 +2460,10 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, } continue; } - four = apple_dart_readl(dart, DART_TCR(dart, sid)) & + if (READ_ONCE(dart->display_dark)) + four = test_bit(sid, dart->display_tcr_four); + else + four = apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level; idx = four ? 0 : addr >> (pgshift + 2 * bits); slot = (addr >> (pgshift + (four ? 2 : 1) * bits)) & @@ -4322,6 +4379,134 @@ static bool apple_dart_has_live_rtkit_client(struct device_node *np) return false; } +static void apple_dart_parse_sleep_list(struct apple_dart *dart, const struct resource *res, + const char *name, struct apple_dart_sleep_list *list) +{ + struct device_node *np = dart->dev->of_node; + unsigned int i; + u32 *pairs; + int count; + + count = of_property_count_u32_elems(np, name); + if (count <= 0) + return; + if (count % 2) { + dev_err(dart->dev, "%s: not a list of offset and value\n", name); + return; + } + pairs = devm_kcalloc(dart->dev, count, sizeof(*pairs), GFP_KERNEL); + if (!pairs || of_property_read_u32_array(np, name, pairs, count)) + return; + for (i = 0; i < count; i += 2) { + if (pairs[i] % 4 || pairs[i] + 4 > resource_size(res)) { + dev_err(dart->dev, "%s: bad offset %#x\n", name, pairs[i]); + return; + } + } + list->pairs = pairs; + list->count = count / 2; +} + +/* The config words a sleep list writes: 0x200 to 0x22c and 0x300 to 0x31c. */ +#define DART_SLEEP_CFG_WORDS 20 + +static unsigned int apple_dart_sleep_cfg_off(unsigned int i) +{ + return i < 12 ? 0x200 + 4 * i : 0x300 + 4 * (i - 12); +} + +static void apple_dart_sleep_cfg_read(struct apple_dart *dart, u32 *cfg) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + cfg[i] = readl(dart->regs + apple_dart_sleep_cfg_off(i)); +} + +/* + * The DART's address filter, which the loader programs and a real sleep clears: one entry + * per range, 0x40 apart, written in the order the loader writes them. + * apple,dapf-ranges gives start and end pairs from the ADT's dapf-instance-0, and the + * optional apple,dapf-r20 each entry's last word, 1 if absent. + */ +#define DART_DAPF_LOCK 0x208 +#define DART_DAPF_LOCKED BIT(2) +#define DART_DAPF_STRIDE 0x40 +#define DART_DAPF_CONFIG 0x31 + +static int apple_dart_dapf_parse(struct platform_device *pdev, struct apple_dart *dart) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf"); + int n = of_property_count_u64_elems(np, "apple,dapf-ranges"); + unsigned int i; + + if (n <= 0 || n % 2) + return dev_err_probe(dev, -EINVAL, "apple,dapf-ranges needs start and end pairs\n"); + dart->dapf_count = n / 2; + if (resource_size(res) < dart->dapf_count * DART_DAPF_STRIDE) + return dev_err_probe(dev, -EINVAL, "DAPF region too small for %u ranges\n", + dart->dapf_count); + + dart->dapf = devm_platform_ioremap_resource_byname(pdev, "dapf"); + if (IS_ERR(dart->dapf)) + return PTR_ERR(dart->dapf); + dart->dapf_ranges = devm_kcalloc(dev, n, sizeof(*dart->dapf_ranges), GFP_KERNEL); + dart->dapf_r20 = devm_kcalloc(dev, dart->dapf_count, sizeof(*dart->dapf_r20), GFP_KERNEL); + if (!dart->dapf_ranges || !dart->dapf_r20) + return -ENOMEM; + of_property_read_u64_array(np, "apple,dapf-ranges", dart->dapf_ranges, n); + for (i = 0; i < dart->dapf_count; i++) + dart->dapf_r20[i] = 1; + if (of_property_count_u32_elems(np, "apple,dapf-r20") == dart->dapf_count) + of_property_read_u32_array(np, "apple,dapf-r20", dart->dapf_r20, dart->dapf_count); + + return 0; +} + +/* Entry 0 as written: the end reads back with its low two bits set, the last word's bytes. */ +static bool apple_dart_dapf_holds(struct apple_dart *dart) +{ + return readq(dart->dapf + 0x08) == dart->dapf_ranges[0] && + (readq(dart->dapf + 0x10) | 3) == (dart->dapf_ranges[1] | 3); +} + +/* + * After a power-off, before any consumer's firmware starts. Not while it's kept, and not + * while it's locked. + */ +static void apple_dart_dapf_replay(struct apple_dart *dart) +{ + unsigned int i; + + if (!dart->dapf_count) + return; + if (apple_dart_dapf_holds(dart)) + return; + if (readl(dart->regs + DART_DAPF_LOCK) & DART_DAPF_LOCKED) { + dev_warn(dart->dev, "DAPF locked after the sleep: not replayed\n"); + return; + } + + for (i = 0; i < dart->dapf_count; i++) { + void __iomem *entry = dart->dapf + i * DART_DAPF_STRIDE; + + writel(0, entry + 0x04); + writeq(dart->dapf_ranges[2 * i], entry + 0x08); + writeq(dart->dapf_ranges[2 * i + 1], entry + 0x10); + writel(DART_DAPF_CONFIG, entry + 0x00); + writel(dart->dapf_r20[i], entry + 0x20); + } + /* Its consumer's firmware faults on its first access without it: say so plainly. */ + if (!apple_dart_dapf_holds(dart)) { + dev_err(dart->dev, "DAPF write-back didn't take: entry 0 reads %#llx..%#llx\n", + readq(dart->dapf + 0x08), readq(dart->dapf + 0x10)); + return; + } + dev_dbg(dart->dev, "DAPF: %u ranges written back after the sleep\n", dart->dapf_count); +} + /* * The setup a T8110 DART needs when nothing initialised it before Linux: * apple,tunable (applied at every reset), apple,bypass-sids (streams kept in @@ -4565,6 +4750,17 @@ static int apple_dart_probe(struct platform_device *pdev) dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "gapf")) { + dart->gapf = devm_platform_ioremap_resource_byname(pdev, "gapf"); + if (IS_ERR(dart->gapf)) + return PTR_ERR(dart->gapf); + dart->gapf_boot = readl(dart->gapf); + } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf")) { + ret = apple_dart_dapf_parse(pdev, dart); + if (ret) + return ret; + } if (tunneled) { dart->tunables = devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable", res); @@ -4585,6 +4781,14 @@ static int apple_dart_probe(struct platform_device *pdev) } } } + dart->resume_tunables = devm_apple_tunable_parse(dev, dev->of_node, + "apple,resume-tunable", res); + if (IS_ERR(dart->resume_tunables)) { + if (PTR_ERR(dart->resume_tunables) != -ENOENT) + return dev_err_probe(dev, PTR_ERR(dart->resume_tunables), + "failed to parse resume tunables\n"); + dart->resume_tunables = NULL; + } if (j713_dcpext || j713_dispext) { /* @@ -4631,7 +4835,7 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled && dart->tunables) { dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", dart->tunables->sz); - apple_dart_apply_tunables(dart); + apple_dart_apply_tunables(dart, dart->tunables); } if (tunneled) { @@ -4805,6 +5009,10 @@ static int apple_dart_probe(struct platform_device *pdev) ret = -EBUSY; goto err_clk_disable; } + apple_dart_parse_sleep_list(dart, res, "apple,sleep-prepare", &dart->sleep_prepare); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore", &dart->sleep_restore); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore-late", + &dart->sleep_restore_late); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); /* Preserve our retained-root path when no reserved-table handoff exists. */ @@ -5309,11 +5517,135 @@ static void apple_dart_save_state(struct apple_dart *dart) } } +static void apple_dart_sleep_write(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, list->pairs[2 * i + 1], list->pairs[2 * i]); +} + +/* + * A sleep list's config words go back as this DART had them before the sleep, not as + * the list has them: Linux's own setup of those words can differ from macOS's. + */ +static u32 apple_dart_sleep_value(struct apple_dart *dart, u32 off, u32 val) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + if (apple_dart_sleep_cfg_off(i) == off) + return dart->sleep_cfg[i]; + return val; +} + +/* A sleep list, with its config words as the DART had them before the sleep. */ +static void apple_dart_sleep_write_kept(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, + apple_dart_sleep_value(dart, list->pairs[2 * i], + list->pairs[2 * i + 1]), + list->pairs[2 * i]); +} + +static void apple_dart_sleep_wait_tlb(struct apple_dart *dart) +{ + u32 status; + + if (readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT)) + dev_err(dart->dev, "TLB command still busy around a sleep: %#x\n", status); +} + +/** + * apple_dart_sleep_prepare() - Save a DART whose consumer is about to gate itself + * @dev: the DART + * + * For a DART with apple,sleep-restore in its node. A consumer calls this before + * telling its firmware to sleep, while the DART's state can be read, and + * apple_dart_sleep_restore() before starting that firmware again. The DART's own + * callbacks then leave it alone. Without the property nothing happens here, and + * they do what they always do. + * + * Return: 0, or -ENODEV if the DART driver isn't bound. + */ +int apple_dart_sleep_prepare(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + if (!dart) + return -ENODEV; + if (!dart->sleep_restore.count) + return 0; + + apple_dart_sleep_cfg_read(dart, dart->sleep_cfg); + spin_lock_irqsave(&dart->lock, flags); + apple_dart_sleep_write(dart, &dart->sleep_prepare); + apple_dart_sleep_wait_tlb(dart); + apple_dart_save_state(dart); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + dart->sleep_enable[i] = apple_dart_readl(dart, dart->hw->enable_streams + 4 * i); + dart->sleep_saved = true; + spin_unlock_irqrestore(&dart->lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_prepare); + +/** + * apple_dart_sleep_restore() - Put back a DART saved by apple_dart_sleep_prepare() + * @dev: the DART + * + * A real sleep loses the DART's registers. A suspend that was refused lost nothing, + * so nothing is written then. + */ +void apple_dart_sleep_restore(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx, i; + unsigned long flags; + + if (!dart || !dart->sleep_saved) + return; + + spin_lock_irqsave(&dart->lock, flags); + dart->sleep_saved = false; + if (pm_resume_via_firmware()) { + apple_dart_sleep_write_kept(dart, &dart->sleep_restore); + for (sid = 0; sid < dart->num_streams; sid++) { + if (!dart->save_tcr[sid] && !dart->save_ttbr[sid][0]) + continue; + apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + } + apple_dart_sleep_write_kept(dart, &dart->sleep_restore_late); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, dart->sleep_enable[i], + dart->hw->enable_streams + 4 * i); + apple_dart_sleep_wait_tlb(dart); + } + spin_unlock_irqrestore(&dart->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_restore); + static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); dart->tunnel_state_restored = false; + /* Its consumer saved this already and restores it: see apple_dart_sleep_prepare(). */ + if (dart->sleep_saved) + return 0; if (READ_ONCE(dart->commands_gated)) return 0; /* @@ -5449,7 +5781,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) int ret; /* No reset or restore: see apple_dart_suspend(). */ - if (dart->cold_ane) + if (dart->cold_ane || dart->sleep_saved) + return 0; + /* Its block can't answer yet: apple_dart_host_resume() restores it. */ + if (dart->host_resumes && READ_ONCE(dart->commands_gated)) return 0; /* Powered down by its DCP, which restores it at power-up. */ if (READ_ONCE(dart->host_off)) @@ -5526,12 +5861,146 @@ static __maybe_unused int apple_dart_resume(struct device *dev) ret = apple_dart_hw_finish_reset(dart); } + /* The address filter went with the same power-off, and so may these. */ + apple_dart_dapf_replay(dart); + if (dart->resume_tunables) { + unsigned int changed = apple_dart_apply_tunables(dart, dart->resume_tunables); + + if (changed) + dev_dbg(dev, "%u of %zu tunables written back\n", changed, + dart->resume_tunables->sz); + } + /* Should they fail again, the next resume repeats them. */ dart->reset_incomplete = ret; return ret; } -static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); +static __maybe_unused int apple_dart_resume_noirq(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + /* + * After a real sleep a DART clocked by its host's port reads nothing until the host + * sets the port up again. Gate it, so that nothing touches it, until then. + */ + if (dart->host_resumes && pm_resume_via_firmware()) { + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, true); + return pm_runtime_force_resume(dev); + } + + ret = pm_runtime_force_resume(dev); + + /* + * A real sleep took the registers even of a DART runtime PM had suspended, which + * force_resume leaves alone. Its consumer's firmware may DMA through it before + * anything resumes it at runtime, so write the saved state back now. Not one with its + * own sleep list, as the SEP's: its consumer restores that with macOS's writes before + * starting its firmware, and this would land on top. + */ + if (!ret && pm_resume_via_firmware() && pm_runtime_status_suspended(dev) && + !dart->sleep_restore.count) { + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after sleep failed: %d\n", ret); + else + dev_dbg(dev, "restored after sleep\n"); + } + if (pm_resume_via_firmware()) + apple_dart_dapf_replay(dart); + + return ret; +} + +/* The host will set this DART's port clock up again after a real sleep, and resume it then. */ +int apple_dart_set_host_resumes(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart) + return -EPROBE_DEFER; + if (dart->tunneled || dart->power_retained || dart->sleep_saved || dart->cold_ane) + return -EINVAL; + dart->host_resumes = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_set_host_resumes); + +/* The host's port is up again: ungate the DART and restore what it saved. */ +int apple_dart_host_resume(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + if (!dart || !dart->host_resumes) + return -EINVAL; + if (!READ_ONCE(dart->commands_gated)) + return 0; + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, false); + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after its host came back failed: %d\n", ret); + else + dev_info(dev, "restored after its host came back\n"); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_host_resume); + +/* + * Its consumer's own power cycle, outside a system sleep: the registers saved and put back. + * Only for a device apple-dart is bound to. + */ +int apple_dart_state_save(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_suspend(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_save); + +int apple_dart_state_restore(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_resume(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_restore); + +/* + * A firmware-driven sleep takes the DARTs in the noirq phase, which every device finishes + * before any early resume: coprocessors such as the PMP and SIO restart their firmware in + * their early resume, and its first DMA has to find the DART already restored. Any other + * sleep keeps the normal phase, which the PCIe-C tunnel DARTs need: their host stops and + * restarts the port clock in its own noirq callbacks. + */ +static __maybe_unused int apple_dart_sys_suspend(struct device *dev) +{ + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_suspend(dev); +} + +static __maybe_unused int apple_dart_sys_resume(struct device *dev) +{ + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_resume(dev); +} + +static __maybe_unused int apple_dart_sys_suspend_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? pm_runtime_force_suspend(dev) : 0; +} + +static __maybe_unused int apple_dart_sys_resume_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? apple_dart_resume_noirq(dev) : 0; +} + +static const struct dev_pm_ops apple_dart_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend, apple_dart_sys_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend_noirq, apple_dart_sys_resume_noirq) + RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) +}; static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -5903,6 +6372,157 @@ int apple_dart_dcp_dispext_state(struct device_node *consumer) EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_state); #endif +/* + * Bit 4 of this DART's GAPF slice makes the fabric reject writes to its coprocessor's + * CPU power state. It's clear from the flush until the streams come back on. + */ +static int apple_dart_gapf_release(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf) + return 0; + + val = readl(dart->gapf); + writel(val & ~APPLE_DART_GAPF_PROTECT, dart->gapf); + if (readl(dart->gapf) & APPLE_DART_GAPF_PROTECT) { + dev_err(dart->dev, "GAPF slice stayed protected: %#x\n", val); + return -EIO; + } + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); + + return 0; +} + +/* Back to the probe value: after a wake the slice reads 0, and protect alone would give 0x10. */ +static void apple_dart_gapf_protect(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf || !dart->sleep_flushed) + return; + + val = readl(dart->gapf); + writel(val | dart->gapf_boot | APPLE_DART_GAPF_PROTECT, dart->gapf); + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); +} + +/* Before its coprocessor's CPU domain is written: a protected slice would fault the write. */ +int apple_dart_gapf_ready(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart->gapf || !(readl(dart->gapf) & APPLE_DART_GAPF_PROTECT)) + return 0; + + return apple_dart_gapf_release(dart); +} +EXPORT_SYMBOL_GPL(apple_dart_gapf_ready); + +/* + * What macOS writes to T8140's display DARTs around a display power cycle: + * measured under aura, with the offsets the driver has no name for. It never + * writes their TCR or TTBR, and reads the same values back after each power-on, + * deep sleep included, so neither does this. + * + * Stopping again after a sleep that never reached the commit does nothing: the + * domains may be off by then. + */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long bits = streams; + unsigned int sid; + u32 status; + int ret; + + if (on) { + apple_dart_gapf_protect(dart); + dart->sleep_flushed = false; + /* Maps or unmaps while it was dark left their invalidations here. */ + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); + writel(0, dart->regs + 0x210); + writel(0, dart->regs + 0x4060); + writel(streams, dart->regs + DART_T8110_ENABLE_STREAMS); + return 0; + } + + if (flush && dart->sleep_flushed) + return 0; + + if (!READ_ONCE(dart->display_dark)) { + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + } + + writel(U32_MAX, dart->regs + 0x4060); + writel(0, dart->regs + 0x4000); + writel(U32_MAX, dart->regs + 0x704); + for_each_set_bit(sid, &bits, BITS_PER_TYPE(streams)) + writel(BIT(sid), dart->regs + DART_T8110_DISABLE_STREAMS); + if (!flush) + return 0; + + writel(0xa000, dart->regs + DART_T8110_TLB_CMD); + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT); + if (ret) { + dev_err(dev, "flush for a power-down didn't finish: %#x\n", status); + return ret; + } + ret = apple_dart_gapf_release(dart); + if (ret) + return ret; + dart->sleep_flushed = true; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_display_streams); + +/* + * A DART whose domain goes off with its consumer's, at runtime, while other users still map + * into it: dark, maps reach memory only and their invalidations wait; lit, those run. Its + * streams are left as they are. Call it dark while it's still powered. + */ +void apple_dart_power_dark(struct device *dev, bool dark) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid; + + if (dark) { + if (READ_ONCE(dart->display_dark)) + return; + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + return; + } + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); +} +EXPORT_SYMBOL_GPL(apple_dart_power_dark); + static struct platform_driver apple_dart_driver = { .driver = { .name = "apple-dart", diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 1125008130ef4..1afc73dd62617 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -388,6 +391,8 @@ struct aic_irq_chip { int nr_die; int max_die; + u32 *sleep_masks; + struct aic_info info; }; @@ -410,6 +415,47 @@ static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) writel_relaxed(val, ic->base + reg); } +/* + * A sleep that a firmware engine finishes waits for the CPUs to stop, and a raised + * line ends WFI even with interrupts off. Every line is masked for that last step. + */ +void aic_sleep_mask_all(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, reg, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++) { + reg = ic->info.mask_set + off + i * 4; + *saved++ = aic_ic_read(ic, reg); + aic_ic_write(ic, reg, U32_MAX); + } + } +} + +void aic_sleep_restore(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++, saved++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, *saved); + aic_ic_write(ic, ic->info.mask_clr + off + i * 4, ~*saved); + } + } +} + +/* Takes the next event, which also masks its line. */ +u32 aic_sleep_ack(void) +{ + return readl(aic_irqc->event + aic_irqc->info.event); +} + /* * IRQ irqchip */ @@ -1083,6 +1129,134 @@ static void build_fiq_affinity(struct aic_irq_chip *ic, struct device_node *aff) } } +#ifdef CONFIG_PM_SLEEP +/* + * T8140's firmware-driven suspend powers the controller down and back up. How + * much it keeps isn't measured, and macOS saves and restores it in software. So + * mask everything before the commit, so that no line can end the final WFI, and + * put it all back after. Linux never writes IRQ_CFG or these global registers + * outside of config's enable bit, so they hold what the bootloader set. + */ +static struct { + u32 config; + u32 timeout; + u32 cluster_prio; + u32 *irq_cfg; + bool saved; +} aic_sleep; + +static int aic_syscore_suspend(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!pm_suspend_via_firmware()) + return 0; + + aic_sleep.config = aic_ic_read(ic, AIC2_CONFIG); + aic_sleep.timeout = aic_ic_read(ic, AIC2_TIMEOUT); + aic_sleep.cluster_prio = aic_ic_read(ic, AIC2_CLUSTER_PRIO); + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < ic->nr_irq; i++) + aic_sleep.irq_cfg[die * ic->nr_irq + i] = + aic_ic_read(ic, ic->info.irq_cfg + off + i * 4); + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + off += ic->info.die_stride; + } + aic_sleep.saved = true; + + return 0; +} + +/* The boot CPU doesn't pass through hotplug, so aic_init_cpu() doesn't rerun. */ +static void aic_resume_boot_cpu(void) +{ + u32 fiq = __this_cpu_read(aic_fiq_unmasked); + + aic_init_cpu(smp_processor_id()); + /* + * aic_init_cpu() is for a CPU with no FIQ users yet. This one's kept theirs: + * a timer left marked masked here would never be unmasked at its next EOI. + */ + __this_cpu_write(aic_fiq_unmasked, fiq); + if (is_kernel_in_hyp_mode()) { + if (fiq & BIT(AIC_TMR_EL02_PHYS)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_P, false); + if (fiq & BIT(AIC_TMR_EL02_VIRT)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V, false); + isb(); + } +} + +static void aic_syscore_resume(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!aic_sleep.saved) + return; + aic_sleep.saved = false; + + /* Otherwise the suspend was refused, and the controller stayed powered. */ + if (pm_resume_via_firmware()) { + aic_ic_write(ic, AIC2_TIMEOUT, aic_sleep.timeout); + aic_ic_write(ic, AIC2_CLUSTER_PRIO, aic_sleep.cluster_prio); + aic_ic_write(ic, AIC2_CONFIG, aic_sleep.config); + aic_resume_boot_cpu(); + } + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + aic_ic_write(ic, ic->info.sw_clr + off + i * 4, U32_MAX); + } + for (i = 0; i < ic->nr_irq; i++) { + u32 saved = aic_sleep.irq_cfg[die * ic->nr_irq + i]; + u32 reg = ic->info.irq_cfg + off + i * 4; + unsigned int virq; + + if (aic_ic_read(ic, reg) != saved) + aic_ic_write(ic, reg, saved); + + /* Restore from the IRQ core's books, not from the hardware's. */ + virq = irq_find_mapping(ic->hw_domain, AIC_IRQ_HWIRQ(die, i)); + if (virq && !irqd_irq_masked(irq_get_irq_data(virq))) + aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(i), + MASK_BIT(i)); + } + off += ic->info.die_stride; + } +} + +static const struct syscore_ops aic_syscore_ops = { + .suspend = aic_syscore_suspend, + .resume = aic_syscore_resume, +}; + +static struct syscore aic_syscore = { + .ops = &aic_syscore_ops, +}; + +static void __init aic_sleep_init(struct aic_irq_chip *irqc, struct device_node *node) +{ + /* The only Apple SoC with a suspend-to-RAM path. */ + if (!of_device_is_compatible(node, "apple,t8140-aic3")) + return; + + aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); + if (WARN_ON(!aic_sleep.irq_cfg)) + return; + + register_syscore(&aic_syscore); +} +#else +static void __init aic_sleep_init(struct aic_irq_chip *irqc, struct device_node *node) +{ +} +#endif + static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { int i, die; @@ -1168,6 +1342,11 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->info.die_stride = off - start_off; + irqc->sleep_masks = kcalloc(irqc->nr_die * (irqc->max_irq >> 5), + sizeof(*irqc->sleep_masks), GFP_KERNEL); + if (WARN_ON(!irqc->sleep_masks)) + goto err_unmap; + irqc->hw_domain = irq_domain_create_tree(of_fwnode_handle(node), &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) @@ -1219,6 +1398,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); + aic_sleep_init(irqc, node); + if (is_kernel_in_hyp_mode()) { struct irq_fwspec mi = { .fwnode = of_fwnode_handle(node), diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 191d8ada0d265..30fe53484b0a5 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -545,6 +545,61 @@ static void isp_test_command_ack_failures(struct kunit *test) KUNIT_EXPECT_EQ(test, state.calls, 0); } +/* The firmware's answer to the stop key request, as macOS reads it */ +static int isp_test_stop_key_reply(struct apple_isp *isp, + struct isp_channel *chan, + unsigned long timeout) +{ + struct kunit *test = kunit_get_current_test(); + unsigned int *calls = test->priv; + u8 *cmd = isp->cmd_virt; + + (*calls)++; + KUNIT_EXPECT_PTR_EQ(test, chan, isp->chan_dg); + KUNIT_EXPECT_EQ(test, chan->req.arg1, 0x20); + KUNIT_EXPECT_EQ(test, chan->req.arg2, 0x20); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(cmd + 4), 0x1d); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(cmd + 8), 1); + KUNIT_EXPECT_TRUE(test, timeout); + put_unaligned_le64(0x10001e00010ULL, cmd + 0x10); + put_unaligned_le64(0x8042006, cmd + 0x18); + chan->rsp = (struct isp_message){ .arg0 = chan->req.arg0 | ISP_IPC_FLAG_ACK, + .arg1 = chan->req.arg2 }; + return 0; +} + +static void isp_test_stop_key_on_debug(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct cmd_stop_key key; + unsigned int calls = 0; + + isp->cmd_virt = kunit_kzalloc(test, 0x400, GFP_KERNEL); + isp->chan_io = kunit_kzalloc(test, sizeof(*isp->chan_io), GFP_KERNEL); + isp->chan_dg = kunit_kzalloc(test, sizeof(*isp->chan_dg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_io); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_dg); + isp->chan_io->name = "IO"; + isp->chan_dg->name = "DEBUG"; + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_COMMAND; + test->priv = &calls; + kunit_deactivate_static_stub(test, isp_cmd_send); + kunit_activate_static_stub(test, ipc_chan_send, isp_test_stop_key_reply); + + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), 0); + KUNIT_EXPECT_EQ(test, calls, 1); + KUNIT_EXPECT_EQ(test, key.addr, 0x10001e00010ULL); + KUNIT_EXPECT_EQ(test, key.key, 0x8042006ULL); + + /* No DEBUG command ring: nothing is sent, on IO either. */ + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_REPLY; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + isp->chan_dg = NULL; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + KUNIT_EXPECT_EQ(test, calls, 1); +} + static void isp_test_call_crop(struct kunit *test) { struct isp_preset source = { @@ -661,6 +716,7 @@ static struct kunit_case isp_cam_test_cases[] = { KUNIT_CASE(isp_test_setfile_failure_stops_setup), KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failures), + KUNIT_CASE(isp_test_stop_key_on_debug), KUNIT_CASE(isp_test_sensor_geometry), KUNIT_CASE(isp_test_sensor_config_list), KUNIT_CASE(isp_test_sensor_config_failure), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 4b096aa29dddf..0f8ae7eb2f554 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -1102,7 +1102,7 @@ int apple_isp_start_capture(struct apple_isp *isp) void apple_isp_stop_capture(struct apple_isp *isp) { - if (isp->ipc_failed || isp->fw_state == ISP_FW_DEAD) + if (isp->ipc_failed || isp->fw_state != ISP_FW_RUNNING) return; isp_cmd_ch_stop(isp, 0); // TODO channel mask isp_cmd_ch_buffer_return(isp, isp->current_ch); diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 75b8704ccedcc..a1a5557111ec0 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -16,26 +16,26 @@ #define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) #define CISP_TIMEOUT msecs_to_jiffies(3000) +/* what macOS allows a DEBUG command */ +#define CISP_DEBUG_TIMEOUT msecs_to_jiffies(1000) #define CISP_SEND_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) #define CISP_SEND_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) #define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) #define CISP_POST_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, 0)) #define CISP_POST_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), 0)) -int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +/* Send through the command area on a command channel. */ +static int isp_cmd_send_chan(struct apple_isp *isp, struct isp_channel *chan, + void *args, u32 insize, u32 outsize, + unsigned long timeout) { - struct isp_channel *chan; struct isp_message *req; int err; -#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) - KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); -#endif /* Before the command area is overwritten */ err = isp_ipc_resync(isp); if (err) return err; - chan = isp->chan_io; req = &chan->req; req->arg0 = isp->cmd_iova; @@ -69,6 +69,15 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int return err; } +int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); +#endif + return isp_cmd_send_chan(isp, isp->chan_io, args, insize, outsize, + timeout); +} + static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, u32 outsize) { @@ -99,6 +108,16 @@ int isp_cmd_stop(struct apple_isp *isp, u32 mode) return CISP_SEND_IN(isp, args); } +/* T8140's cold stop, which macOS sends as a command with its reply. */ +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + int isp_cmd_power_down(struct apple_isp *isp) { struct cmd_power_down args = { @@ -133,6 +152,30 @@ int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) return CISP_SEND_INOUT(isp, args); } +/* + * macOS asks for the key on the DEBUG channel. On IO, 0x1d is another + * command, and the firmware answers neither it nor the STOP after it. + */ +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args) +{ + struct isp_channel *chan = isp->chan_dg; + int err; + + if (!chan || chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + return -ENODEV; + + memset(args, 0, sizeof(*args)); + args->opcode = CISP_OPCODE(CISP_DEBUG_CMD_STOP_KEY_GET); + args->enable = 1; + err = isp_cmd_send_chan(isp, chan, args, sizeof(*args), sizeof(*args), + CISP_DEBUG_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, sizeof(*args)); + return 0; +} + int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) { args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 9e2678bd1610c..ab98f3447d6f7 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -101,6 +101,9 @@ #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD 0xc11d #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH 0xc120 +/* Opcodes of the DEBUG channel, which has its own numbering */ +#define CISP_DEBUG_CMD_STOP_KEY_GET 0x001d + #define CISP_POOL_TYPE_META 0x0 #define CISP_POOL_TYPE_RENDERED 0x1 #define CISP_POOL_TYPE_FD 0x2 @@ -159,6 +162,19 @@ struct cmd_print_enable { } __packed; static_assert(sizeof(struct cmd_print_enable) == 0xc); +/* + * CISP_DEBUG_CMD_STOP_KEY_GET: the firmware answers with a word of its + * memory and the value it writes there once it has stopped. + */ +struct cmd_stop_key { + u64 opcode; + u32 enable; + u32 pad_c; + u64 addr; + u64 key; +} __packed; +static_assert(sizeof(struct cmd_stop_key) == 0x20); + struct cmd_trace_enable { u64 opcode; u32 enable; @@ -263,10 +279,12 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout); int isp_cmd_start(struct apple_isp *isp, u32 mode); int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode); int isp_cmd_power_down(struct apple_isp *isp); int isp_cmd_suspend(struct apple_isp *isp); int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args); int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 21dd9cdc1be2a..2006351c49a53 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "isp-cam.h" @@ -525,6 +526,17 @@ isp_check_firmware_version(struct apple_isp *isp) return compat; } +/* After a system sleep, see apple_isp_resume(). */ +static void apple_isp_resume_work(struct work_struct *work) +{ + struct apple_isp *isp = container_of(work, struct apple_isp, resume_work); + + mutex_lock(&isp->video_lock); + isp->fw_sleep_stopped = false; + mutex_unlock(&isp->video_lock); + apple_isp_video_resume(isp); +} + static int apple_isp_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -647,6 +659,7 @@ static int apple_isp_probe(struct platform_device *pdev) } mutex_init(&isp->video_lock); + INIT_WORK(&isp->resume_work, apple_isp_resume_work); spin_lock_init(&isp->buf_lock); init_waitqueue_head(&isp->wait); INIT_LIST_HEAD(&isp->gc); @@ -723,6 +736,7 @@ static void apple_isp_remove(struct platform_device *pdev) { struct apple_isp *isp = platform_get_drvdata(pdev); + cancel_work_sync(&isp->resume_work); apple_isp_remove_video(isp); /* Stop resident firmware before freeing what it may still use. */ apple_isp_firmware_halt(isp); @@ -954,6 +968,7 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .fw_restart = true, .meta_exposure = true, }; @@ -1244,9 +1259,43 @@ static __maybe_unused int apple_isp_runtime_resume(struct device *dev) return 0; } +/* + * Stop resident firmware. For a system sleep it starts again on resume where + * it can; otherwise, or for hibernation, the camera is gone until the next + * boot. + */ +static void apple_isp_stop_resident(struct apple_isp *isp, bool sleep) +{ + bool restart = sleep && isp->hw->fw_restart; + + if (!isp->hw->resident_fw) + return; + + mutex_lock(&isp->video_lock); + if (isp->fw_state == ISP_FW_RUNNING) { + if (restart) + dev_dbg(isp->dev, "stopping the firmware for system sleep\n"); + else + dev_warn(isp->dev, "stopping the firmware for %s, the camera is unavailable until the next boot\n", + sleep ? "system sleep" : "hibernation"); + isp->fw_sleep_stopped = restart; + } + apple_isp_firmware_halt(isp); + /* Hibernation resumes in another boot, where that memory isn't ours. */ + if (!restart) + isp->fw_state = ISP_FW_DEAD; + mutex_unlock(&isp->video_lock); +} + static __maybe_unused int apple_isp_suspend(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + bool power_lost = isp->hw->resident_fw && pm_suspend_via_firmware(); + + /* A stream that couldn't restart on the other side keeps the system up. */ + if (power_lost && !isp->hw->fw_restart && + test_bit(ISP_STATE_STREAMING, &isp->state)) + return -EBUSY; /* We must restore V4L2 context on system resume. If we were streaming * before, we (essentially) stop streaming and start streaming again. @@ -1254,9 +1303,13 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) apple_isp_video_suspend(isp); /* - * Resident firmware stays up, idle, across suspend-to-idle and - * suspend-to-RAM: its domains stay on, see apple_isp_attach_genpd(). + * Resident firmware stays up, idle, across suspend-to-idle: its domains + * stay on, see apple_isp_attach_genpd(). A sleep that powers the SoC down + * takes it anyway, so stop it first. */ + if (power_lost) + apple_isp_stop_resident(isp, true); + return 0; } @@ -1272,13 +1325,7 @@ static __maybe_unused int apple_isp_freeze(struct device *dev) * that started it. Stop it cleanly instead; if it ran, the camera is * then unavailable until the next system boot. */ - if (isp->hw->resident_fw) { - mutex_lock(&isp->video_lock); - if (isp->fw_state == ISP_FW_RUNNING) - dev_warn(dev, "stopping the firmware for hibernation, the camera is unavailable until the next boot\n"); - apple_isp_firmware_halt(isp); - mutex_unlock(&isp->video_lock); - } + apple_isp_stop_resident(isp, false); return 0; } @@ -1287,6 +1334,16 @@ static __maybe_unused int apple_isp_resume(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + /* + * Firmware stopped for the sleep starts again in process context: its + * handshake here held up the whole resume. A stream that ran across the + * sleep restarts from a freezable work item, once tasks thaw; without one + * the firmware starts at the next stream. + */ + if (isp->fw_sleep_stopped) { + queue_work(system_freezable_wq, &isp->resume_work); + return 0; + } apple_isp_video_resume(isp); return 0; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 66fa3e1264576..94081701d9a5a 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -241,12 +242,15 @@ struct apple_isp_hw { bool fw_restore; /* - * The firmware can be started only once per system boot, so once - * started it is kept running while the driver is bound. It starts at - * probe, or at the first stream with deferred_start. + * The firmware isn't started for each stream: once started it is kept + * running while the driver is bound. It starts at probe, or at the + * first stream with deferred_start. */ bool resident_fw; + /* Resident firmware that starts again after a stop, from its own memory */ + bool fw_restart; + /* * Start the firmware, and detect the camera, at the first stream * instead of at probe, so the ISP stays off until then. The firmware @@ -393,6 +397,10 @@ struct apple_isp { u64 heap_base; u64 heap_top; u64 data_top; /* 0 unless the bootloader loads the data */ + /* the coprocessor's reset vector at the first start */ + u64 entry; + /* the coprocessor has run since probe */ + bool started; } fw; /* changes under video_lock once the video device is registered */ enum isp_fw_state fw_state; @@ -401,6 +409,11 @@ struct apple_isp { /* configurations the firmware has, once a deferred start detected them */ u32 num_fw_presets; + /* stopped for system sleep, to start again on resume */ + bool fw_sleep_stopped; + /* restarts a stream that ran across that sleep, see apple_isp_resume() */ + struct work_struct resume_work; + struct isp_surf *ipc_surf; struct isp_surf *extra_surf; struct isp_surf *data_surf; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 302b18cd189c6..6204ae335241c 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -7,7 +7,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -29,6 +32,12 @@ #define ISP_COPROC_IN_WFI 0x3 +/* the waits for firmware that has been told to power down */ +#define ISP_STOP_KEY_TRIES 20 +#define ISP_STOP_KEY_MS 10 +#define ISP_STOP_WFI_POLL_US (10 * USEC_PER_MSEC) +#define ISP_STOP_WFI_US (190 * USEC_PER_MSEC) + static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) { return readl(isp->coproc + reg); @@ -254,9 +263,12 @@ static void isp_disable_irq(struct apple_isp *isp) if (isp->hw->mbox_irq_route || isp->hw->mbox_extra_enables) { isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, 0x0); isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, 0x0); + if (isp->hw->mbox_irq_route) + isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, 0x0); } free_irq(isp->irq, isp); isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ + fsleep(10); } static int isp_enable_irq(struct apple_isp *isp) @@ -415,6 +427,39 @@ static void isp_restore_fw_memory(struct apple_isp *isp) clear_highpage(tail->pages[i]); } +/* + * The coprocessor starts at its reset vector, which the bootloader sets and + * which bit 0 latches. Note it at the first start; where a power-off cleared + * it, set it again. + */ +static int isp_coproc_entry(struct apple_isp *isp) +{ + struct isp_firmware *fw = &isp->fw; + u64 rvbar = readq(isp->coproc + ISP_COPROC_RVBAR); + + if (!fw->started) { + fw->entry = rvbar & ~1ULL; + dev_dbg(isp->dev, "firmware reset vector %#llx\n", rvbar); + return 0; + } + + if (!fw->entry || rvbar == (fw->entry | 1)) + return 0; + + if (rvbar & 1) { + isp_err(isp, "reset vector %#llx latched, not the firmware's %#llx\n", + rvbar, fw->entry); + return -EIO; + } + + writeq(fw->entry | 1, isp->coproc + ISP_COPROC_RVBAR); + dev_dbg(isp->dev, "reset vector was %#llx, set to %#llx\n", rvbar, + readq(isp->coproc + ISP_COPROC_RVBAR)); + + return (readq(isp->coproc + ISP_COPROC_RVBAR) & ~1ULL) == fw->entry ? + 0 : -EIO; +} + static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { /* Only a coprocessor that was started has to settle and quiesce */ @@ -477,6 +522,12 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) if (err < 0) goto shutdown; + if (isp->hw->fw_restart) { + err = isp_coproc_entry(isp); + if (err < 0) + goto shutdown; + } + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); isp_gpio_write32(isp, ISP_GPIO_1, 0x0); isp_gpio_write32(isp, ISP_GPIO_2, 0x0); @@ -490,6 +541,7 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); isp_coproc_write32(isp, isp_coproc_control(isp), 0x10); + isp->fw.started = true; /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { @@ -887,10 +939,92 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) return 0; } +/* Wait for the word of its own memory the firmware sets once it has stopped. */ +static void isp_wait_stop_key(struct apple_isp *isp, u64 addr, u32 key) +{ + u64 iova = isp_fw_iova(isp, addr); + phys_addr_t phys = 0; + u32 *word = NULL; + + if (iova < isp_fw_iova(isp, isp->fw.heap_top)) + phys = iommu_iova_to_phys(isp->domain, iova); + if (phys) + word = memremap(phys, sizeof(*word), MEMREMAP_WC); + if (!word) { + dev_warn(isp->dev, "stop key at 0x%llx isn't in firmware memory\n", + addr); + return; + } + + for (int i = 0; i < ISP_STOP_KEY_TRIES && READ_ONCE(*word) != key; i++) + msleep(ISP_STOP_KEY_MS); + + if (READ_ONCE(*word) != key) + dev_warn(isp->dev, "stop key at 0x%llx reads 0x%x, not 0x%x\n", + addr, READ_ONCE(*word), key); + + memunmap(word); +} + +/* + * Stop firmware that starts cold next time. CISP_CMD_SUSPEND is for firmware + * that resumes: it isn't followed by a cold start. + */ +static int isp_stop_firmware_cold(struct apple_isp *isp) +{ + struct cmd_stop_key stop_key = {}; + ktime_t start = ktime_get(); + u32 status; + int err; + + /* Without the key the rest of the stop still works. */ + if (isp_cmd_stop_key_get(isp, &stop_key)) + stop_key.addr = 0; + + err = isp_cmd_stop_cold(isp, 0); + if (err) { + isp_err(isp, "isp_cmd_stop_cold() failed\n"); + return err; + } + + /* The firmware answers the power-down in this word, not on the ring. */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + err = isp_cmd_power_down(isp); + if (err) { + isp_err(isp, "isp_cmd_power_down() failed\n"); + return err; + } + + if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + if (stop_key.addr) + isp_wait_stop_key(isp, stop_key.addr, stop_key.key); + + /* Its domains go off next, so its CPU should be idle by now. */ + if (readl_poll_timeout(isp->coproc + ISP_COPROC_STATUS_T8140, status, + status & ISP_COPROC_IN_WFI, ISP_STOP_WFI_POLL_US, + ISP_STOP_WFI_US)) + dev_warn(isp->dev, "coprocessor not in WFI (status: 0x%x)\n", + status); + else + dev_dbg(isp->dev, "firmware stopped in %lld ms (key at 0x%llx, status 0x%x)\n", + ktime_ms_delta(ktime_get(), start), stop_key.addr, + status); + + return 0; +} + static int isp_stop_command_processor(struct apple_isp *isp) { int res, ret; + if (isp->hw->fw_restart) + return isp_stop_firmware_cold(isp); + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); /* @@ -1085,26 +1219,48 @@ static void isp_firmware_shutdown(struct apple_isp *isp) /* * Stop the coprocessor before releasing any memory the firmware - * uses, also when it did not acknowledge the suspend request. + * uses, also when it did not acknowledge the stop request. */ if (isp_stop_command_processor(isp)) - dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); + dev_warn(isp->dev, "firmware did not stop, stopping the coprocessor\n"); isp_disable_irq(isp); isp_firmware_stop(isp); } +/* + * After a restart, the firmware must find the camera found at probe. The + * query also shows that it takes commands. + */ +static int isp_check_restarted(struct apple_isp *isp) +{ + struct cmd_config_get args = {}; + int err; + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + if (min_t(u32, args.num_channels, ISP_MAX_CHANNELS) != isp->num_channels) { + isp_err(isp, "firmware found %u channels after a restart, %d before\n", + args.num_channels, isp->num_channels); + return -ENODEV; + } + + return 0; +} + /* * Resident firmware (hw->resident_fw) is loaded by the bootloader and - * does not start again once it has been stopped: after CISP_CMD_SUSPEND, - * a coprocessor reset and a power cycle of its domains it never completes - * the first handshake. It is booted once, at probe or with - * deferred_start at the first stream, and runs until the - * driver is unbound, probe fails or the system hibernates, also across - * suspend; after it has stopped the device stays unusable until the next - * system boot. + * changes its own memory as it runs. It is booted at probe, or with + * deferred_start at the first stream, and runs until the driver is unbound, + * probe fails, the system hibernates or a sleep powers the SoC down; + * suspend-to-idle keeps it running. With hw->fw_restart it starts again, and + * rebuilds that memory itself; other stopped firmware stays down until the + * next system boot. */ int apple_isp_firmware_boot(struct apple_isp *isp) { + bool restart = isp->hw->resident_fw && isp->fw.started; int err; switch (isp->fw_state) { @@ -1131,6 +1287,11 @@ int apple_isp_firmware_boot(struct apple_isp *isp) } err = isp_firmware_boot(isp); + if (!err && restart) { + err = isp_check_restarted(isp); + if (err) + isp_firmware_shutdown(isp); + } if (err) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); pm_runtime_put_sync(isp->dev); @@ -1163,7 +1324,7 @@ void apple_isp_firmware_shutdown(struct apple_isp *isp) isp_free_released_surfaces(isp); } -/* Stop the firmware; resident firmware cannot be started again. */ +/* Stop the firmware; resident firmware without hw->fw_restart stays down. */ void apple_isp_firmware_halt(struct apple_isp *isp) { /* Only a capture services the watchdog, but be sure. */ @@ -1175,5 +1336,6 @@ void apple_isp_firmware_halt(struct apple_isp *isp) isp_firmware_shutdown(isp); pm_runtime_put_sync(isp->dev); isp->ipc_stale = NULL; - isp->fw_state = isp->hw->resident_fw ? ISP_FW_DEAD : ISP_FW_OFF; + isp->fw_state = isp->hw->resident_fw && !isp->hw->fw_restart ? + ISP_FW_DEAD : ISP_FW_OFF; } diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 81964ab1916c9..f6df02e6fe8df 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -611,6 +611,8 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) { struct apple_isp *isp = vb2_get_drv_priv(q); + /* A restart still due from a sleep has nothing left to restart. */ + clear_bit(ISP_STATE_SLEEPING, &isp->state); apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); v4l2_ctrl_grab(isp->exposure_priority, false); diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index c4f9329dbc01c..16d6fdae7c12e 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -9,8 +9,16 @@ struct macsmc_test_context { int result; unsigned int calls; bool require_sram_lock; + /* What the sleep tests saw, in order: keys read, values written, notices. */ + u32 seen[8]; + unsigned int seen_count; + int ap_power_result; + u64 record; }; +#define MACSMC_TEST_AP_IDLE 0 +#define MACSMC_TEST_AP_ON 1 + static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -18,6 +26,9 @@ static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); ctx->calls++; + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = cmd == SMC_MSG_WRITE_KEY ? + get_unaligned((u32 *)ctx->shmem) : arg; if (ret_data && ctx->result >= 0) *ret_data = ctx->data; return ctx->result; @@ -199,6 +210,128 @@ static void macsmc_key_info_lock_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); } +static int macsmc_test_ap_power(struct apple_smc *smc, bool on) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + /* The answer needs a worker that may be waiting for this mutex. */ + KUNIT_EXPECT_FALSE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = on ? MACSMC_TEST_AP_ON : MACSMC_TEST_AP_IDLE; + return ctx->ap_power_result; +} + +static void macsmc_sleep_order_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + apple_smc_sleep_warn(&ctx->smc); + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + + /* As in the trace, values included. */ + KUNIT_ASSERT_EQ(test, ctx->seen_count, 7U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[5], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[6], SMC_KEY(waka)); +} + +static void macsmc_sleep_refused_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->ap_power_result = -ETIME; + ctx->result = 4; + + /* No answer: refuse, undo the notice and leave commands working. */ + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), -ETIME); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_ON); +} + +static void macsmc_notice_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + /* The test trigger writes no MBSE value: those announce a real sleep. */ + KUNIT_EXPECT_EQ(test, apple_smc_notice_idle(&ctx->smc), 0); + apple_smc_notice_on(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(MBSW)); +} + +/* MBSW reads ctx->record, MSLD reads closed. */ +static int macsmc_test_lid_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = key; + memset(buf, 0, size); + if (key == SMC_KEY(MBSW) && size == sizeof(ctx->record)) + put_unaligned_le64(ctx->record, buf); + else if (key == SMC_KEY(MSLD)) + *(u8 *)buf = 1; + return size; +} + +static void macsmc_lid_wake_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_read, macsmc_test_lid_read); + ctx->result = 4; + + /* The lid alone: MSLD is read last, after macOS's own order. */ + ctx->record = BIT_ULL(10); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 5U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MSLD)); + + /* The lid with anything else is a normal wake: the lid isn't read. */ + ctx->seen_count = 0; + ctx->record = BIT_ULL(10) | BIT_ULL(0); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 4U); +} + +static void macsmc_wake_reason_test(struct kunit *test) +{ + char names[128]; + + KUNIT_EXPECT_EQ(test, apple_smc_wake_reason_names(0, names, sizeof(names)), 0); + KUNIT_EXPECT_STREQ(test, names, ""); + + /* A bit with no name is left out. */ + apple_smc_wake_reason_names(BIT_ULL(23) | BIT_ULL(42) | BIT_ULL(1), names, sizeof(names)); + KUNIT_EXPECT_STREQ(test, names, "wifibt USB-C_plug"); +} + static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_key_info_lock_test), KUNIT_CASE(macsmc_key_index_test), @@ -209,6 +342,11 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_request_bounds_test), KUNIT_CASE(macsmc_sram_value_test), KUNIT_CASE(macsmc_key_info_test), + KUNIT_CASE(macsmc_sleep_order_test), + KUNIT_CASE(macsmc_sleep_refused_test), + KUNIT_CASE(macsmc_notice_test), + KUNIT_CASE(macsmc_lid_wake_test), + KUNIT_CASE(macsmc_wake_reason_test), {} }; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 927ddaa00e0df..801718b9b24d4 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -22,7 +22,10 @@ #include #include #include +#include #include +#include +#include #include #define SMC_ENDPOINT 0x20 @@ -120,6 +123,14 @@ static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) memcpy_fromio(buf, smc->shmem.iomem, size); } +/* In a noirq phase the mailbox's interrupt is off, so its reply has to be fetched. */ +static bool apple_smc_reply_arrived(struct apple_smc *smc) +{ + apple_rtkit_poll(smc->rtk); + + return completion_done(&smc->cmd_done); +} + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -136,6 +147,9 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return -EIO; if (smc->atomic_mode) return -EIO; + /* Between the AP power notices the firmware expects no commands. */ + if (smc->ap_notified) + return -EAGAIN; reinit_completion(&smc->cmd_done); @@ -152,7 +166,17 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return ret; } - if (wait_for_completion_timeout(&smc->cmd_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) <= 0) { + if (smc->polled) { + bool done; + + ret = read_poll_timeout(apple_smc_reply_arrived, done, done, 100, + SMC_TIMEOUT_MS * USEC_PER_MSEC, false, smc); + } else { + ret = wait_for_completion_timeout(&smc->cmd_done, + msecs_to_jiffies(SMC_TIMEOUT_MS)) > 0 ? + 0 : -ETIMEDOUT; + } + if (ret) { dev_err(smc->dev, "Command timed out (%llx)", msg); return -ETIMEDOUT; } @@ -598,6 +622,10 @@ static int apple_smc_probe(struct platform_device *pdev) ARRAY_SIZE(apple_smc_telemetry_devs), NULL, 0, NULL); + ret = devm_device_init_wakeup(dev); + if (ret) + return ret; + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -634,10 +662,287 @@ static const struct of_device_id apple_smc_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_smc_of_match); +/* + * The SMC keeps running through a firmware-driven suspend. It is told first and + * stops last, and on a wake it hears first. In order, measured on T8140 and + * unverified on other SoCs: MBSE = slpw before any other coprocessor is told; + * MBSE = slpa and the AP power notice after all the others; on a wake the notice, + * MBSE and MBSW read, MBSE = waka. The noirq phases give that order whatever the + * device order is, so these commands poll. What the values mean is inferred + * from their letters. + * + * SMC.OutboxNotEmpty wakes the SoC while anything waits in the outbox. An event that arrives + * during the sleep's entry (the lid, a key) is left there, so the sleep returns at once. The + * debug link's cut fills it too, and the SMC counts what it reports after the notice as wake + * reasons, so the cut comes before the notice. + */ + +/* The gaps macOS leaves before the AP idle notice and after it. */ +#define SMC_SLEEP_ARM_MS 6 +#define SMC_SLEEP_IDLE_MS 30 + +/* + * After the debug link's cut. Its port and charger events keep coming for over a second, + * and any that lands after the AP idle notice wakes the SoC at once. + */ +#define SMC_OUTBOX_QUIET_MS 500 +#define SMC_OUTBOX_SETTLE_MS 5000 + +static int apple_smc_ap_power(struct apple_smc *smc, bool on) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_ap_power, smc, on); + + return smc->polled ? apple_rtkit_ap_power_noirq(smc->rtk, on) : + apple_rtkit_ap_power(smc->rtk, on); +} + +static void apple_smc_set_ap_notified(struct apple_smc *smc, bool notified) +{ + guard(mutex)(&smc->mutex); + + smc->ap_notified = notified; +} + +static void apple_smc_set_polled(struct apple_smc *smc, bool polled) +{ + guard(mutex)(&smc->mutex); + + smc->polled = polled; +} + +static void apple_smc_write_mbse(struct apple_smc *smc, smc_key value) +{ + int ret; + + ret = apple_smc_write_u32(smc, SMC_KEY(MBSE), value); + if (ret) + dev_err(smc->dev, "Failed to issue MBSE = %p4ch: %d\n", &value, ret); +} + +static void apple_smc_sleep_warn(struct apple_smc *smc) +{ + apple_smc_write_mbse(smc, SMC_KEY(slpw)); + smc->sleep_warned = true; +} + +static int apple_smc_notice_idle(struct apple_smc *smc) +{ + int ret; + + /* + * Commands fail from here on. The mutex isn't held across the notice: the + * answer comes through the same worker as the notifications, and those can + * be waiting for this mutex. + */ + apple_smc_set_ap_notified(smc, true); + + ret = apple_smc_ap_power(smc, false); + if (ret) { + /* Taken and not answered, perhaps, so undo it. */ + apple_smc_ap_power(smc, true); + apple_smc_set_ap_notified(smc, false); + dev_err(smc->dev, "SMC did not take the sleep notice: %d\n", ret); + } else { + dev_dbg(smc->dev, "AP idle notice answered\n"); + } + + return ret; +} + +static int apple_smc_sleep_commit(struct apple_smc *smc) +{ + int ret; + + apple_smc_write_mbse(smc, SMC_KEY(slpa)); + /* Device interrupts are off here. */ + mdelay(SMC_SLEEP_ARM_MS); + + ret = apple_smc_notice_idle(smc); + if (!ret) + mdelay(SMC_SLEEP_IDLE_MS); + + return ret; +} + +/* MBSW's bits, as smc-pmu's event_name properties number them on T8140. Other SoCs differ. */ +static const struct { + u8 bit; + const char *name; +} apple_smc_wake_reasons[] = { + { 0, "pwrbtn" }, { 9, "ipd_touch" }, { 10, "lid" }, { 16, "ipd_kbd" }, + { 18, "charger" }, { 23, "wifibt" }, { 40, "acattach" }, { 41, "acdetach" }, + { 42, "USB-C_plug" }, { 43, "USB2_plug" }, { 44, "USB2_wake" }, { 45, "aupo" }, + { 48, "rtc" }, +}; + +static int apple_smc_wake_reason_names(u64 record, char *buf, size_t size) +{ + int i, len = 0; + + buf[0] = 0; + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (record & BIT_ULL(apple_smc_wake_reasons[i].bit)) + len += scnprintf(buf + len, size - len, "%s%s", len ? " " : "", + apple_smc_wake_reasons[i].name); + + return len; +} + +static void apple_smc_notice_on(struct apple_smc *smc) +{ + u8 record[8] = {}; + char names[128]; + u32 event = 0; + int ret; + + ret = apple_smc_ap_power(smc, true); + if (ret) + dev_err(smc->dev, "SMC did not take the wake notice: %d\n", ret); + apple_smc_set_ap_notified(smc, false); + + ret = apple_smc_read_u32(smc, SMC_KEY(MBSE), &event); + if (ret) + dev_warn(smc->dev, "Failed to read MBSE after a wake: %d\n", ret); + ret = apple_smc_read(smc, SMC_KEY(MBSW), record, sizeof(record)); + if (ret < 0) + dev_warn(smc->dev, "Failed to read MBSW after a wake: %d\n", ret); + + smc->wake_record = get_unaligned_le64(record); + apple_smc_wake_reason_names(smc->wake_record, names, sizeof(names)); + dev_info(smc->dev, "wake record: MBSE %#x, MBSW %8ph%s%s\n", event, record, + names[0] ? ": " : "", names); +} + +/* MBSW's bit for a wake reason, by its name, or -1. */ +static int apple_smc_wake_reason_bit(const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (!strcmp(apple_smc_wake_reasons[i].name, name)) + return apple_smc_wake_reasons[i].bit; + + return -1; +} + +/* + * Woken for the lid alone, which the SMC reports for the lid opening and for it closing: + * tell the sleep driver if it's closed now. + */ +static void apple_smc_lid_wake(struct apple_smc *smc) +{ + int bit = apple_smc_wake_reason_bit("lid"); + u8 lid = 0; + int ret; + + if (bit < 0 || smc->wake_record != BIT_ULL(bit)) + return; + + ret = apple_smc_read_u8(smc, SMC_KEY(MSLD), &lid); + if (ret < 0) { + dev_warn(smc->dev, "Failed to read the lid after a wake: %d\n", ret); + return; + } + dev_info(smc->dev, "woken for the lid, which is %s\n", lid ? "closed" : "open"); + if (lid) + apple_sleep_lid_closed_wake(); +} + +static void apple_smc_sleep_wake(struct apple_smc *smc) +{ + apple_smc_notice_on(smc); + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + apple_smc_lid_wake(smc); + /* A dark wake isn't one: with wakeup_count in use, it would abort the sleep after it. */ + if (smc->wake_record && !pm_dark_wake()) + pm_wakeup_event(smc->dev, 0); +} + +static int apple_smc_prepare(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (pm_suspend_via_firmware()) + apple_smc_sleep_warn(smc); + + return 0; +} + +/* Warned, and a refusal came before the notice: tell it the sleep is off. */ +static void apple_smc_complete(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (smc->sleep_warned) { + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + } +} + +/* The events go to their handlers as usual, through the RTKit worker. */ +static void apple_smc_outbox_settle(struct apple_smc *smc) +{ + unsigned int ms, quiet = 0, count = 0, last = 0; + int ret; + + for (ms = 0; ms < SMC_OUTBOX_SETTLE_MS && quiet < SMC_OUTBOX_QUIET_MS; ms++) { + ret = apple_rtkit_poll(smc->rtk); + if (ret > 0) { + count += ret; + last = ms; + quiet = 0; + } else { + quiet++; + } + usleep_range(1000, 1200); + } + dev_dbg(smc->dev, "debug link cut: %u messages, the last at %u ms, %s after %u ms\n", + count, last, quiet >= SMC_OUTBOX_QUIET_MS ? "quiet" : "still busy", ms); +} + +static int apple_smc_suspend_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + int ret; + + if (!smc->sleep_warned) + return 0; + + apple_smc_set_polled(smc, true); + if (apple_sleep_debug_link_cut()) + apple_smc_outbox_settle(smc); + ret = apple_smc_sleep_commit(smc); + apple_smc_set_polled(smc, false); + + return ret; +} + +static int apple_smc_resume_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (smc->ap_notified) { + apple_smc_set_polled(smc, true); + apple_smc_sleep_wake(smc); + apple_smc_set_polled(smc, false); + } + + return 0; +} + +static const struct dev_pm_ops apple_smc_pm_ops = { + .prepare = pm_sleep_ptr(apple_smc_prepare), + .complete = pm_sleep_ptr(apple_smc_complete), + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_smc_suspend_noirq, apple_smc_resume_noirq) +}; + static struct platform_driver apple_smc_driver = { .driver = { .name = "macsmc", .of_match_table = apple_smc_of_match, + .pm = pm_sleep_ptr(&apple_smc_pm_ops), }, .probe = apple_smc_probe, }; diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 99aa39372b49b..175f605b33cf1 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o wow.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += power-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o @@ -8,5 +8,6 @@ mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += move-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += wow-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 3a9db56813847..a8ad970fd3fc8 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -564,7 +564,8 @@ static void mt_cal_work(struct work_struct *work) int ret; mutex_lock(&m->command_mutex); - if (!READ_ONCE(m->rf_ready)) + /* Stopped for a sleep, the requests wait for the restart. */ + if (!READ_ONCE(m->rf_ready) || READ_ONCE(m->hif_suspended)) goto unlock; ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) @@ -633,6 +634,7 @@ int mt_register_interface(struct mt7932 *m) INIT_WORK(&m->connect_work, mt_connect_work); INIT_WORK(&m->disconnect_work, mt_disconnect_work); INIT_WORK(&m->power_work, mt_power_work); + INIT_WORK(&m->wow_work, mt_wow_work); init_completion(&m->assoc_start); init_completion(&m->assoc_done); init_completion(&m->disconnect_done); diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index e0ed45dccdae8..5d4bc8074bc66 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -49,6 +49,7 @@ void mt_stop_host(struct mt7932 *m) flush_work(&m->disconnect_work); cancel_work_sync(&m->cal_work); cancel_work_sync(&m->power_work); + cancel_work_sync(&m->wow_work); cancel_delayed_work_sync(&m->scan_timeout_work); cancel_work_sync(&m->scan_finish_work); /* With all producers joined, this is the sole owner of any remainder. */ diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 94388b56449e1..36eb79125166d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -7,6 +7,7 @@ #include #include "mt7932.h" +#include "wow.h" #include #include @@ -44,7 +45,8 @@ static void mt_shutdown(struct pci_dev *pdev); bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && - !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed); + !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed) && + !READ_ONCE(m->hif_suspended); } int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name) @@ -130,6 +132,30 @@ static void mt_rx_post(struct mt7932_ring *q, unsigned int index) d->control = cpu_to_le32(MT7932_RX_CAPACITY << 16); } +/* + * The host interface control command and event, the same ID both ways. The command is + * type, direction, stop, suspend in 36 bytes; the event's body is type, TX traffic, RX + * traffic, suspend, with at least as many bytes. + */ +#define MT7932_CID_HIF_CTRL 0xf6 +#define MT7932_HIF_PCIE 2 +#define MT7932_HIF_IDLE 2 +#define MT7932_HIF_LENGTH 36 + +/* response_lock held. */ +static void mt_hif_event(struct mt7932 *m, const struct mt7932_event *event) +{ + const u8 *body = event->packet + 36; + + if (!m->hif_waiting || event->length < 36 + MT7932_HIF_LENGTH || + body[0] != MT7932_HIF_PCIE || !!body[3] != m->hif_suspending) + return; + m->hif_waiting = false; + m->hif_result = body[3] && (body[1] != MT7932_HIF_IDLE || body[2] != MT7932_HIF_IDLE) ? + -EBUSY : 0; + complete(&m->hif_done); +} + static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) { struct mt7932_event event; @@ -153,10 +179,18 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) return; } spin_lock_irqsave(&m->response_lock, flags); + if (event.eid == MT7932_CID_HIF_CTRL) { + mt_hif_event(m, &event); + goto unlock; + } if (event.eid == 0xee) { mt_link_event(m, &event); goto unlock; } + if (event.eid == MT7932_EID_WOW_REASON) { + mt_wow_reason_event(m, &event); + goto unlock; + } if (event.eid == 0x0d) { mt_scan_event(m, &event); goto unlock; @@ -370,6 +404,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) return -ESHUTDOWN; + /* With the host interface stopped only its own control may reach the BAR. */ + if (READ_ONCE(m->hif_gate) && !READ_ONCE(m->hif_waiting)) + return -ESHUTDOWN; if (q->used >= q->slots) { /* Command buffers can wrap only after every published descriptor has * retired. Never reclaim by elapsed time or by a reply alone. @@ -1098,6 +1135,8 @@ static ssize_t removal_retry_store(struct device *dev, } static DEVICE_ATTR_WO(removal_retry); +static struct pci_driver mt_driver; + static int mt_retire_dma(struct mt7932 *m) { int ret; @@ -1120,11 +1159,35 @@ static int mt_retire_dma(struct mt7932 *m) return ret; } +/* + * Removal waits forever for a reset that fails, which a recovery in a worker can't do. So stop + * here first, where a failure keeps the function bound. A radio stopped this way stays down + * until it's removed, and remove then has nothing left to wait for. + */ +static int mt_radio_stop(struct pci_dev *pdev) +{ + struct mt7932 *m; + int ret = 0; + + device_lock(&pdev->dev); + m = pdev->dev.driver == &mt_driver.driver ? pci_get_drvdata(pdev) : NULL; + if (m) { + mt_stop_host(m); + ret = mt_retire_dma(m); + if (ret) + dev_err(&pdev->dev, "restart refused: DMA still owned: %d\n", ret); + } + device_unlock(&pdev->dev); + + return ret; +} + static void mt_remove(struct pci_dev *pdev) { struct mt7932 *m = pci_get_drvdata(pdev); int ret; + apple_piodma_radio_set_stop(NULL); mt_stop_host(m); while ((ret = mt_retire_dma(m))) { WRITE_ONCE(m->removal_error, ret); @@ -1207,6 +1270,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) init_completion(&m->response); init_completion(&m->cal_response); init_completion(&m->reset_retry); + init_completion(&m->hif_done); mutex_init(&m->command_mutex); spin_lock_init(&m->response_lock); ret = mt_ownership(m); @@ -1242,6 +1306,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = device_create_file(&pdev->dev, &dev_attr_removal_retry); if (ret) dev_warn(&pdev->dev, "removal retry attribute unavailable: %d\n", ret); + apple_piodma_radio_set_stop(mt_radio_stop); return 0; } @@ -1255,21 +1320,192 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* command_mutex held; the firmware answers once its host interface has changed state. */ +static int mt_hif_ctrl(struct mt7932 *m, bool suspend) +{ + u8 body[MT7932_HIF_LENGTH] = { MT7932_HIF_PCIE, 1, suspend, suspend }; + unsigned long flags; + int ret; + + reinit_completion(&m->hif_done); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_suspending = suspend; + m->hif_result = -ETIMEDOUT; + m->hif_waiting = true; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt_request(m, MT7932_CID_HIF_CTRL, true, true, false, body, sizeof(body)); + if (!ret) + wait_for_completion_timeout(&m->hif_done, msecs_to_jiffies(1000)); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_waiting = false; + if (!ret) + ret = m->hif_result; + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* Every published command and frame retired, ahead of the firmware's own stop. */ +static int mt_hif_tx_idle(struct mt7932 *m) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(500); + + do { + mt_data_clean(m); + mt_tx_clean(&m->tx[17]); + if (!READ_ONCE(m->tx[0].queued) && !m->tx[17].queued) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + return -EBUSY; +} + +/* + * Driver own again, DMA back on, then the firmware's host interface, and WoW + * off if the sleep had it. Rings stay as they were. + */ +static int mt_hif_restart(struct mt7932 *m) +{ + bool wow; + int ret; + + ret = mt_ownership(m); + if (ret) + return ret; + mt_write(m, 0x10188, 0xff); + if (mt_read(m, m->rx[0].reg) != lower_32_bits(m->rx[0].dma) || + mt_read(m, m->tx[17].reg) != lower_32_bits(m->tx[17].dma)) { + dev_err(&m->pdev->dev, "DMA rings lost across the sleep: RX0 %#x, TX17 %#x\n", + mt_read(m, m->rx[0].reg), mt_read(m, m->tx[17].reg)); + return -EIO; + } + mt_write(m, W + 0x208, m->hif_glo); + WRITE_ONCE(m->running, true); + mt_write(m, W + 0x204, m->irq_mask); + mutex_lock(&m->command_mutex); + wow = m->wow_armed; + ret = mt_hif_ctrl(m, false); + if (!ret) { + WRITE_ONCE(m->hif_gate, false); + ret = mt_wow_disarm(m); + if (ret) + WRITE_ONCE(m->hif_gate, true); + } + mutex_unlock(&m->command_mutex); + if (ret) { + /* Not left running into the next sleep. */ + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + mt_dma_stop(m); + return ret; + } + WRITE_ONCE(m->hif_suspended, false); + /* What arrived or was asked for while stopped, now that it can run. */ + if (READ_ONCE(m->cal_request_count)) + schedule_work(&m->cal_work); + if (wow) + schedule_work(&m->power_work); + if (READ_ONCE(m->interface_registered) && READ_ONCE(m->reg_pending)) + schedule_work(&m->startup_work); + if (m->netdev) + netif_device_attach(m->netdev); + return 0; +} + /* - * Stopping the firmware for sleep would take the whole remove and probe - * sequence, so leave it running across suspend-to-idle. A saved state keeps - * the PCI core from changing the function's power state or command register, - * and the awake path keeps the PCIe controller powered until resume. + * What macOS does at sleep. Without a WoWLAN configuration cfg80211 has already left + * the network; with one the firmware is given WoW first and stays associated. The firmware + * stops its host interface and is given ownership, and keeps running. PERST# then resets + * the link, not the firmware. + * + * Trouble here costs Wi-Fi at the wake, never the machine its sleep. + */ +static int mt_hif_suspend(struct mt7932 *m) +{ + int ret; + + if (!READ_ONCE(m->ready) || READ_ONCE(m->stopping) || READ_ONCE(m->hif_suspended)) + return 0; + /* New work waits from here, and what already runs finishes before the gate. */ + WRITE_ONCE(m->hif_suspended, true); + if (m->interface_registered) { + if (m->netdev) + netif_device_detach(m->netdev); + flush_work(&m->startup_work); + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + flush_work(&m->cal_work); + flush_work(&m->power_work); + mt_abort_scan(m->wiphy, &m->wdev); + ret = mt_scan_quiesce(m); + if (ret) + dev_warn(&m->pdev->dev, "scan still retiring at the sleep: %d\n", ret); + } + mutex_lock(&m->command_mutex); + mt_wow_arm(m); + WRITE_ONCE(m->hif_gate, true); + ret = mt_hif_tx_idle(m); + if (!ret) + ret = mt_hif_ctrl(m, true); + mutex_unlock(&m->command_mutex); + if (ret) + dev_err(&m->pdev->dev, "firmware did not stop its host interface: %d\n", ret); + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + m->hif_glo = mt_read(m, W + 0x208) & ~(BIT(1) | BIT(3)); + ret = mt_dma_stop(m); + if (ret) + dev_err(&m->pdev->dev, "DMA did not stop for the sleep: %d\n", ret); + mt_write(m, 0xe0010, 1); + ret = mt_poll(m, 0xe0010, BIT(2), BIT(2), 50000); + if (ret) + dev_err(&m->pdev->dev, "firmware did not take ownership for the sleep: %d\n", ret); + else + dev_dbg(&m->pdev->dev, "host interface stopped for the sleep, firmware kept\n"); + return 0; +} + +/* + * Stopping the firmware for sleep takes the whole remove and probe sequence, so + * leave it running across suspend-to-idle. A saved state keeps the PCI core from + * changing the function's power state or command register, and the awake path + * keeps the PCIe controller powered until resume. */ static int mt_suspend(struct device *dev) { if (!pm_suspend_no_platform()) - return -EOPNOTSUPP; + return mt_hif_suspend(pci_get_drvdata(to_pci_dev(dev))); pci_save_state(to_pci_dev(dev)); device_set_awake_path(dev); return 0; } +static int mt_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mt7932 *m = pci_get_drvdata(pdev); + unsigned long flags; + int ret; + + if (!READ_ONCE(m->hif_suspended)) + return 0; + /* As macOS does: a chip that no longer answers isn't touched further. */ + ret = pci_device_is_present(pdev) ? mt_hif_restart(m) : -ENODEV; + if (!ret) { + dev_dbg(dev, "back after the sleep, firmware kept\n"); + return 0; + } + dev_err(dev, "Wi-Fi didn't keep its firmware across the sleep: %d; restarting it\n", ret); + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + apple_piodma_radio_recover(); + return ret; +} + /* * Hibernation is refused. Its freeze phase does not call .suspend, so without * this the firmware would keep writing to memory while the image is saved, @@ -1286,6 +1522,7 @@ static int mt_freeze(struct device *dev) static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .resume = mt_resume, .freeze = mt_freeze, .poweroff = mt_freeze, }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index f267ffca4d496..2c534c331a82e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -92,6 +92,12 @@ struct mt7932 { bool dma_owned, module_pinned; struct completion reset_retry; int removal_error; + /* The firmware keeps running across a deep sleep with its host interface stopped. */ + struct completion hif_done; + int hif_result; + u32 hif_glo; + /* hif_suspended stops new work; hif_gate then closes the BAR to all but the control. */ + bool hif_waiting, hif_suspending, hif_suspended, hif_gate; bool ram_config; bool runtime_epoch; bool runtime_cal_valid; @@ -155,6 +161,14 @@ struct mt7932 { /* Desired/submitted PS configuration uses command_mutex; TIM uses response_lock. */ u8 power_mode, power_applied; bool power_applied_valid, power_tim; + /* WoWLAN detectors cfg80211 asked for at this sleep; 0 leaves the network. */ + u8 wow_detect; + /* The firmware was given WoW at the sleep; command_mutex. */ + bool wow_armed; + /* A wake reason is awaited, and the one received; response_lock. */ + bool wow_report; + u8 wow_reason; + struct work_struct wow_work; struct completion assoc_start, assoc_done, disconnect_done; struct completion discovery_done; struct completion scan_done; @@ -344,5 +358,15 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); +int mt_power_submit(struct mt7932 *m, u8 mode); +#ifdef CONFIG_PM +extern const struct wiphy_wowlan_support mt_wowlan_support; +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow); +int mt_wow_resume(struct wiphy *wiphy); +#endif +void mt_wow_arm(struct mt7932 *m); +int mt_wow_disarm(struct mt7932 *m); +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event); +void mt_wow_work(struct work_struct *work); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index f3c1c42863c9c..03653ba97eede 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -5,7 +5,7 @@ * submission is accepted configuration, not proof that the radio is asleep. * cfg80211 implements GET_POWER_SAVE from the accepted requested setting. */ -static int mt_power_submit(struct mt7932 *m, u8 mode) +int mt_power_submit(struct mt7932 *m, u8 mode) { u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 0a27e86995f99..4441150dcef4a 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -561,7 +561,8 @@ static void mt_startup_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); generation = m->reg_generation; *reg = m->reg_desired; - if (m->stopping || generation == m->reg_attempted) { + /* Stopped for a sleep, the restart runs this generation again. */ + if (m->stopping || READ_ONCE(m->hif_suspended) || generation == m->reg_attempted) { spin_unlock_irqrestore(&m->response_lock, flags); kfree(reg); return; @@ -804,6 +805,10 @@ static const struct cfg80211_ops mt_cfg_ops = { .dump_station = mt_dump_station, .set_power_mgmt = mt_set_power_mgmt, .set_cqm_rssi_config = mt_set_cqm_rssi_config, +#ifdef CONFIG_PM + .suspend = mt_wow_suspend, + .resume = mt_wow_resume, +#endif }; int mt_register_regulatory_gate(struct mt7932 *m) @@ -840,6 +845,9 @@ int mt_register_regulatory_gate(struct mt7932 *m) wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_MFP_OPTIONAL); m->wiphy->max_scan_ssids = MT7932_SCAN_SSIDS; m->wiphy->max_scan_ie_len = 600; +#ifdef CONFIG_PM + m->wiphy->wowlan = &mt_wowlan_support; +#endif m->wiphy->reg_notifier = mt_regulatory_notify; /* World roaming may relax NO_IR after observing an AP beacon without * changing alpha2. Request the callback so firmware's CID0f domain is diff --git a/drivers/net/wireless/mediatek/mt7932/wow-test.c b/drivers/net/wireless/mediatek/mt7932/wow-test.c new file mode 100644 index 0000000000000..1301fa93b7bbb --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow-test.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "mt7932.h" +#include "wow.h" + +static bool mt_wow_test_zero(const u8 *p, size_t from, size_t to) +{ + size_t i; + + for (i = from; i < to; i++) + if (p[i]) + return false; + return true; +} + +static void mt_wow_detect_test(struct kunit *test) +{ + struct cfg80211_wowlan wow = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(NULL), 0); + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), MT7932_WOW_DETECT_MAGIC); + wow.magic_pkt = false; + wow.disconnect = true; + /* Losing the AP's beacons ends the association as surely as a deauth. */ + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x14); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x15); +} + +static void mt_wow_ctrl_test(struct kunit *test) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)228); + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, true, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_START); + KUNIT_EXPECT_EQ(test, body[1], 0x15); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), 0x63); + KUNIT_EXPECT_EQ(test, body[4], 0); /* scenario */ + KUNIT_EXPECT_EQ(test, body[5], 1); /* pins */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 6, 8)); + KUNIT_EXPECT_EQ(test, body[8], 3); /* GPIO */ + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + /* Interval, second wake signal, port counts and lists. */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); + + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, false, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_STOP); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 1, 5)); + KUNIT_EXPECT_EQ(test, body[5], 1); + KUNIT_EXPECT_EQ(test, body[8], 3); + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); +} + +static void mt_wow_suspend_mode_test(struct kunit *test) +{ + u8 body[MT7932_SUSPEND_MODE_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)68); + memset(body, 0xa5, sizeof(body)); + mt7932_suspend_mode(body, true); + KUNIT_EXPECT_EQ(test, body[0], 0); /* BSS */ + KUNIT_EXPECT_EQ(test, body[1], 1); + KUNIT_EXPECT_EQ(test, body[2], 1); /* DTIM multiple */ + KUNIT_EXPECT_EQ(test, body[3], 1); /* WoW suspend */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 4, sizeof(body))); + mt7932_suspend_mode(body, false); + KUNIT_EXPECT_EQ(test, body[1], 0); + KUNIT_EXPECT_EQ(test, body[2], 1); + KUNIT_EXPECT_EQ(test, body[3], 1); +} + +static void mt_wow_config_test(struct kunit *test) +{ + static const u8 addr[ETH_ALEN] = { 0x02, 0x11, 0x22, 0x33, 0x44, 0x55 }; + u8 magic[MT7932_WOW_MAGIC_LENGTH], clear[MT7932_WOW_CLEAR_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(magic), (size_t)52); + KUNIT_EXPECT_EQ(test, sizeof(clear), (size_t)44); + memset(magic, 0xa5, sizeof(magic)); + mt7932_wow_magic(magic, addr); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 32), 0x06); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 34), 20); + KUNIT_EXPECT_EQ(test, magic[36], 0); /* entry */ + KUNIT_EXPECT_EQ(test, magic[37], 1); /* enabled */ + KUNIT_EXPECT_EQ(test, magic[38], 1); /* forward */ + KUNIT_EXPECT_EQ(test, magic[39], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 40), 0); + KUNIT_EXPECT_MEMEQ(test, magic + 42, addr, ETH_ALEN); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 48, sizeof(magic))); + + memset(clear, 0xa5, sizeof(clear)); + mt7932_wow_clear(clear); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 32), 0x18); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 34), 12); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 36, sizeof(clear))); +} + +static void mt_wow_reason_test(struct kunit *test) +{ + struct cfg80211_wowlan_wakeup wakeup; + u8 body[MT7932_WOW_REASON_LENGTH] = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body) - 1), -EINVAL); + body[0] = 9; + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body)), 9); + + KUNIT_EXPECT_FALSE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_NONE, &wakeup)); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_MAGIC, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_DISCONNECT, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.disconnect); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + /* A firmware reason with no cfg80211 trigger is still a wireless wake. */ + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(9, &wakeup)); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); +} + +#ifdef CONFIG_PM +static void mt_wow_test_work(struct work_struct *work) +{ +} + +static void mt_wow_test_device_release(struct device *dev) +{ +} + +static void mt_wow_test_put(void *data) +{ + put_device(data); +} + +static void mt_wow_reason_event_test(struct kunit *test) +{ + struct mt7932 *m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + u8 *packet = kunit_kzalloc(test, 36 + MT7932_WOW_REASON_LENGTH, GFP_KERNEL); + struct mt7932_event event = { + .eid = MT7932_EID_WOW_REASON, + .packet = packet, + .length = 36 + MT7932_WOW_REASON_LENGTH, + }; + unsigned long flags; + + KUNIT_ASSERT_NOT_NULL(test, m); + KUNIT_ASSERT_NOT_NULL(test, packet); + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, m->pdev); + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_wow_test_device_release; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mt_wow_test_put, &m->pdev->dev), 0); + KUNIT_ASSERT_EQ(test, dev_set_name(&m->pdev->dev, "mt7932-wow-test"), 0); + spin_lock_init(&m->response_lock); + INIT_WORK(&m->wow_work, mt_wow_test_work); + m->wow_reason = MT7932_WOW_REASON_NONE; + packet[36] = MT7932_WOW_REASON_MAGIC; + + /* Not a sleep WoW was armed for: ignored. */ + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* A short event leaves the wait open. */ + m->wow_report = true; + event.length--; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_TRUE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* One report per sleep. */ + event.length++; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_FALSE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); + packet[36] = MT7932_WOW_REASON_DISCONNECT; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); +} +#endif + +static struct kunit_case mt_wow_test_cases[] = { + KUNIT_CASE(mt_wow_detect_test), + KUNIT_CASE(mt_wow_ctrl_test), + KUNIT_CASE(mt_wow_suspend_mode_test), + KUNIT_CASE(mt_wow_config_test), + KUNIT_CASE(mt_wow_reason_test), +#ifdef CONFIG_PM + KUNIT_CASE(mt_wow_reason_event_test), +#endif + {} +}; + +static struct kunit_suite mt_wow_test_suite = { + .name = "mt7932-wow", + .test_cases = mt_wow_test_cases, +}; + +kunit_test_suite(mt_wow_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/wow.c b/drivers/net/wireless/mediatek/mt7932/wow.c new file mode 100644 index 0000000000000..279a0c01a7b1d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Wake-on-WLAN for a deep sleep. The firmware keeps the association while the + * host sleeps and wakes it through the radio's GPIO, which the SMC turns into + * an AP wake. Nothing in-band can wake the SoC from S2R. + */ +#include + +#include "mt7932.h" +#include "wow.h" + +#ifdef CONFIG_PM +const struct wiphy_wowlan_support mt_wowlan_support = { + .flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT, +}; + +/* response_lock held. */ +static bool mt_wow_associated_locked(struct mt7932 *m) +{ + return m->connected && m->peer_valid && !m->connecting && !m->disconnecting && + !m->link_failed && !m->stopping && m->rf_ready; +} + +/* + * cfg80211 calls this before the device's own suspend. Returning 1 makes it + * leave the network and call again without a configuration. + */ +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + u8 detect = mt7932_wow_detect(wow); + unsigned long flags; + bool associated; + + WRITE_ONCE(m->wow_detect, 0); + if (!wow) + return 0; + /* Only S2R hands the radio to the firmware; s2idle and hibernation leave. */ + if (!detect || pm_suspend_target_state != PM_SUSPEND_MEM || pm_suspend_no_platform()) + return 1; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) + return 1; + WRITE_ONCE(m->wow_detect, detect); + return 0; +} + +int mt_wow_resume(struct wiphy *wiphy) +{ + WRITE_ONCE(mt_from_wiphy(wiphy)->wow_detect, 0); + return 0; +} + +/* + * command_mutex held, before the host interface stops. The firmware takes the + * commands from the ring before it answers the stop. + */ +void mt_wow_arm(struct mt7932 *m) +{ + u8 detect = READ_ONCE(m->wow_detect); + u8 body[MT7932_WOW_CTRL_LENGTH]; + unsigned long flags; + bool associated; + int ret = 0; + + lockdep_assert_held(&m->command_mutex); + WRITE_ONCE(m->wow_detect, 0); + if (!detect) + return; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + m->wow_report = false; + m->wow_reason = MT7932_WOW_REASON_NONE; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) { + dev_dbg(&m->pdev->dev, "not associated at the sleep: Wi-Fi wake not armed\n"); + return; + } + /* Whatever reached the firmware is undone at the wake. */ + m->wow_armed = true; + if (detect & MT7932_WOW_DETECT_MAGIC) { + mt7932_wow_magic(body, m->mac); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_MAGIC_LENGTH); + } + if (!ret) + ret = mt_power_submit(m, MT7932_WOW_POWER_MODE); + if (!ret) { + mt7932_suspend_mode(body, true); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + } + if (!ret) { + mt7932_wow_ctrl(body, true, detect); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (ret) { + dev_err(&m->pdev->dev, "Wi-Fi wake not armed: %d\n", ret); + return; + } + spin_lock_irqsave(&m->response_lock, flags); + m->wow_report = true; + spin_unlock_irqrestore(&m->response_lock, flags); + dev_dbg(&m->pdev->dev, "Wi-Fi wake armed:%s%s, staying associated\n", + detect & MT7932_WOW_DETECT_MAGIC ? " magic packet" : "", + detect & MT7932_WOW_DETECT_DISCONNECT ? " disconnect" : ""); +} + +/* + * command_mutex held, once the host interface runs again. A command that + * cannot be queued leaves the firmware filtering, so the caller resets it. + */ +int mt_wow_disarm(struct mt7932 *m) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + int ret; + + lockdep_assert_held(&m->command_mutex); + if (!m->wow_armed) + return 0; + m->wow_armed = false; + mt7932_suspend_mode(body, false); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + if (!ret) { + mt7932_wow_ctrl(body, false, 0); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (!ret) { + mt7932_wow_clear(body); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_CLEAR_LENGTH); + } + /* The sleep's profile stays until power_work submits the requested one. */ + m->power_applied_valid = false; + if (ret) + dev_err(&m->pdev->dev, "Wi-Fi wake not stopped: %d\n", ret); + return ret; +} + +/* response_lock held. The firmware reports why it woke the host. */ +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ + int reason = event->length < 36 ? -EINVAL : + mt7932_wow_reason(event->packet + 36, event->length - 36); + + if (reason < 0) { + dev_dbg(&m->pdev->dev, "short wake reason event: %zu bytes\n", event->length); + return; + } + if (!m->wow_report) + return; + m->wow_report = false; + m->wow_reason = reason; + schedule_work(&m->wow_work); +} + +void mt_wow_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, wow_work); + struct cfg80211_wowlan_wakeup wakeup; + unsigned long flags; + u8 reason; + + spin_lock_irqsave(&m->response_lock, flags); + reason = m->wow_reason; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!mt7932_wow_wakeup(reason, &wakeup)) { + dev_dbg(&m->pdev->dev, "Wi-Fi did not wake the host\n"); + cfg80211_report_wowlan_wakeup(&m->wdev, NULL, GFP_KERNEL); + return; + } + dev_info(&m->pdev->dev, "woken by Wi-Fi: firmware reason %u%s\n", reason, + wakeup.magic_pkt ? " (magic packet)" : + wakeup.disconnect ? " (disconnect)" : ""); + cfg80211_report_wowlan_wakeup(&m->wdev, &wakeup, GFP_KERNEL); +} +#else +void mt_wow_arm(struct mt7932 *m) +{ +} + +int mt_wow_disarm(struct mt7932 *m) +{ + return 0; +} + +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ +} + +void mt_wow_work(struct work_struct *work) +{ +} +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/wow.h b/drivers/net/wireless/mediatek/mt7932/wow.h new file mode 100644 index 0000000000000..2bbe7d4529a1b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_WOW_H +#define MT7932_WOW_H + +#include +#include +#include +#include +#include +#include + +/* + * Wake-on-WLAN commands, in the order the stock driver sends them for an + * associated sleep: the magic packet entry on the WoW configuration command, + * the power-save profile, suspend mode, then WoW start. Waking reverses it. + */ +#define MT7932_CID_WOW_CTRL 0x4a +#define MT7932_CID_WOW_CONFIG 0x4f +#define MT7932_CID_SUSPEND_MODE 0x58 +#define MT7932_EID_WOW_REASON 0x62 + +#define MT7932_WOW_CTRL_LENGTH 228 +#define MT7932_SUSPEND_MODE_LENGTH 68 +/* The configuration command is a zeroed 32-byte header and one TLV. */ +#define MT7932_WOW_CONFIG_HEADER 32 +#define MT7932_WOW_MAGIC_LENGTH (MT7932_WOW_CONFIG_HEADER + 20) +#define MT7932_WOW_CLEAR_LENGTH (MT7932_WOW_CONFIG_HEADER + 12) +#define MT7932_WOW_TLV_MAGIC 0x06 +#define MT7932_WOW_TLV_CLEAR 0x18 + +#define MT7932_WOW_START 1 +#define MT7932_WOW_STOP 2 + +#define MT7932_WOW_DETECT_MAGIC BIT(0) +#define MT7932_WOW_DETECT_DISCONNECT BIT(2) +#define MT7932_WOW_DETECT_BEACON_LOST BIT(4) + +/* + * Drop everything but magic packets, EAPOL and ARP requests to this station, + * as the stock driver filters for a sleep. + */ +#define MT7932_WOW_FILTER 0x63 + +/* The stock configuration wakes the host on the radio's GPIO 13, level 5. */ +#define MT7932_WOW_HIF_GPIO 3 +#define MT7932_WOW_GPIO_PIN 13 +#define MT7932_WOW_GPIO_LEVEL 5 +#define MT7932_WOW_PIN_COUNT 1 + +/* Power-save profile for the sleep, and the suspend mode's DTIM multiple. */ +#define MT7932_WOW_POWER_MODE 2 +#define MT7932_WOW_MDTIM 1 + +#define MT7932_WOW_REASON_MAGIC 0 +#define MT7932_WOW_REASON_DISCONNECT 8 +#define MT7932_WOW_REASON_NONE 0xff +/* The stock driver ignores a shorter reason event. */ +#define MT7932_WOW_REASON_LENGTH 132 + +/* The firmware detectors for the triggers cfg80211 asked for; 0 if none applies. */ +static inline u8 mt7932_wow_detect(const struct cfg80211_wowlan *wow) +{ + u8 detect = 0; + + if (!wow) + return 0; + if (wow->magic_pkt) + detect |= MT7932_WOW_DETECT_MAGIC; + if (wow->disconnect) + detect |= MT7932_WOW_DETECT_DISCONNECT | MT7932_WOW_DETECT_BEACON_LOST; + return detect; +} + +/* Start, with these detectors, or stop. */ +static inline void mt7932_wow_ctrl(u8 out[MT7932_WOW_CTRL_LENGTH], bool start, u8 detect) +{ + memset(out, 0, MT7932_WOW_CTRL_LENGTH); + out[0] = start ? MT7932_WOW_START : MT7932_WOW_STOP; + if (start) { + out[1] = detect; + put_unaligned_le16(MT7932_WOW_FILTER, out + 2); + } + /* Scenario 0, then the wake signal: one pin, GPIO, which pin, level. */ + out[5] = MT7932_WOW_PIN_COUNT; + out[8] = MT7932_WOW_HIF_GPIO; + out[9] = MT7932_WOW_GPIO_PIN; + out[10] = MT7932_WOW_GPIO_LEVEL; + /* No UDP or TCP wake ports. */ +} + +/* BSS 0 into or out of suspend mode; the DTIM multiple and WoW flag stay set. */ +static inline void mt7932_suspend_mode(u8 out[MT7932_SUSPEND_MODE_LENGTH], bool enable) +{ + memset(out, 0, MT7932_SUSPEND_MODE_LENGTH); + out[1] = enable; + out[2] = MT7932_WOW_MDTIM; + out[3] = 1; +} + +/* Magic packet entry 0 for this address, matched from the start of the frame. */ +static inline void mt7932_wow_magic(u8 out[MT7932_WOW_MAGIC_LENGTH], const u8 *addr) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_MAGIC_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_MAGIC, tlv); + put_unaligned_le16(MT7932_WOW_MAGIC_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); + tlv[4] = 0; /* entry */ + tlv[5] = 1; /* enabled */ + tlv[6] = 1; /* offset counts forward */ + put_unaligned_le16(0, tlv + 8); + ether_addr_copy(tlv + 10, addr); +} + +/* Remove every WoW configuration entry. */ +static inline void mt7932_wow_clear(u8 out[MT7932_WOW_CLEAR_LENGTH]) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_CLEAR_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_CLEAR, tlv); + put_unaligned_le16(MT7932_WOW_CLEAR_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); +} + +/* + * The reason from a wake reason event body, or -EINVAL if it is short. + * MT7932_WOW_REASON_NONE means the radio did not wake the host. + */ +static inline int mt7932_wow_reason(const u8 *body, size_t length) +{ + if (length < MT7932_WOW_REASON_LENGTH) + return -EINVAL; + return body[0]; +} + +/* + * What to tell cfg80211 for a reason. False if the radio did not wake the + * host. A reason without a cfg80211 trigger reports a wake with none set. + */ +static inline bool mt7932_wow_wakeup(u8 reason, struct cfg80211_wowlan_wakeup *wakeup) +{ + memset(wakeup, 0, sizeof(*wakeup)); + wakeup->pattern_idx = -1; + switch (reason) { + case MT7932_WOW_REASON_NONE: + return false; + case MT7932_WOW_REASON_MAGIC: + wakeup->magic_pkt = true; + break; + case MT7932_WOW_REASON_DISCONNECT: + wakeup->disconnect = true; + break; + } + return true; +} + +#endif diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 2164b13f4e787..f3eb82be43a30 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -39,17 +39,21 @@ #include #include #include +#include +#include #include #include #include "nvme.h" #define APPLE_ANS_BOOT_TIMEOUT USEC_PER_SEC +#define APPLE_ANS_SLEEP_SHN_TIMEOUT USEC_PER_SEC /* RTKit handshake timeout for the T6050 ANS, see apple_nvme_alloc() */ #define APPLE_NVME_T6050_HANDSHAKE_TIMEOUT_MS 10000 #define APPLE_ANS_COPROC_CPU_CONTROL 0x44 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_ANS_COPROC_CPU_STATUS 0x48 #define APPLE_ANS_ACQ_DB 0x1004 #define APPLE_ANS_IOCQ_DB 0x100c @@ -210,6 +214,14 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; + /* Firmware-driven suspend has been traced on this controller. */ + bool sleep_traced; + /* The last endpoint map reply from that trace */ + u32 sleep_epmap_last_mask; + /* Higher namespaces are the firmware's, and a Flush to one faults the controller. */ + u32 sleep_flush_max_nsid; + /* CC.SHN for a sleep, as read back from the OS the sleep was traced on. */ + u32 sleep_shn; /* The ANS is never reset through PMGR here, so no reset is named. */ bool reset_optional; u32 max_queue_depth; @@ -236,6 +248,17 @@ struct apple_nvme { bool recover_to_live; bool suspended; bool final_shutdown; + /* This suspend is firmware-driven: ANS will lose power at its end. */ + bool sleep_stop; + bool sleep_idled; + u32 sleep_csts; + /* ANS came back across a sleep, or failed to. reset_work picks these up. */ + bool sleep_restarted; + bool sleep_failed; + /* Run bit set after a sleep, and not yet reported on. */ + bool sleep_restarting; + ktime_t sleep_run_time; + bool sleep_registered; unsigned long recovery_flags; void __iomem *mmio_coproc; @@ -1396,6 +1419,31 @@ static int apple_nvme_disable(struct apple_nvme *anv, bool shutdown) return apple_nvme_disable_locked(anv, shutdown); } +/* + * All of a sleep's stop that can be taken back. Nothing is in flight or can start, and + * the controller and its firmware keep running for a refused sleep to come back to. + */ +static int apple_nvme_sleep_quiesce(struct apple_nvme *anv) +{ + guard(mutex)(&anv->disable_lock); + if (READ_ONCE(anv->quarantined)) + return -EIO; + + if (!test_bit(NVME_CTRL_FROZEN, &anv->ctrl.flags)) + nvme_start_freeze(&anv->ctrl); + if (!nvme_wait_freeze_timeout(&anv->ctrl)) { + /* A reset cancels what's still in flight. Don't sleep on that. */ + dev_warn(anv->dev, "requests still in flight after the timeout\n"); + return -EBUSY; + } + nvme_quiesce_io_queues(&anv->ctrl); + nvme_quiesce_admin_queue(&anv->ctrl); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); + + return 0; +} + static void apple_nvme_recovery_work(struct work_struct *work) { struct apple_nvme *anv = container_of(work, struct apple_nvme, recovery_work); @@ -1614,6 +1662,8 @@ static void apple_nvme_wait_freeze(struct apple_nvme *anv) } } +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv); + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1634,6 +1684,17 @@ static void apple_nvme_reset_work(struct work_struct *work) return; } WRITE_ONCE(anv->abort_reset, false); + if (READ_ONCE(anv->sleep_restarting) && !apple_nvme_sleep_wait_on(anv)) + WRITE_ONCE(anv->sleep_failed, true); + if (READ_ONCE(anv->sleep_failed)) { + /* + * Starting it some other way can only reset it, which faults, and failing the + * controller fails every held write. Leave the queues quiesced and frozen. + */ + if (phase_locked) + mutex_unlock(&anv->disable_lock); + return; + } if (state != NVME_CTRL_RESETTING) { dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state); ret = -ENODEV; @@ -1690,6 +1751,12 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = -EIO; goto out; } + /* A real sleep took the controller's state with its power. */ + if (READ_ONCE(anv->sleep_restarted)) { + WRITE_ONCE(anv->sleep_restarted, false); + if (!(readl(anv->mmio_nvme + NVME_REG_CC) & NVME_CC_ENABLE)) + goto rtkit_ready; + } ret = apple_nvme_disable_locked(anv, false); if (ret) goto out; @@ -2532,6 +2599,57 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) return ERR_PTR(ret); } +/* + * A sleep's shutdown notification, with interrupts off, before anything else stops. macOS + * finds the controller enabled and shut down when it tells the firmware to idle. A failure + * refuses: the firmware hasn't been told, and resume's controller reset takes it back. + */ +static int apple_nvme_sleep_shutdown(void *data) +{ + struct apple_nvme *anv = data; + u32 csts; + int ret; + + anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; + anv->ctrl.ctrl_config |= anv->hw->sleep_shn; + writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); + ret = readl_poll_timeout_atomic(anv->mmio_nvme + NVME_REG_CSTS, csts, + (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, + 100, APPLE_ANS_SLEEP_SHN_TIMEOUT); + if (ret) + dev_err(anv->dev, "shutdown notification failed: CSTS %#x\n", csts); + anv->sleep_csts = csts; + + return ret; +} + +/* + * The rest of a sleep's stop, last of all. From the first idle message only a real sleep + * brings it back, so a failure goes on to the commit: what a flush acknowledged is on + * flash, and the watchdog is the net. + */ +static int apple_nvme_sleep_last_stop(void *data) +{ + struct apple_nvme *anv = data; + unsigned int us; + u32 cc, csts = anv->sleep_csts; + int ret; + + cc = readl(anv->mmio_nvme + NVME_REG_CC); + + anv->sleep_idled = true; + ret = apple_rtkit_sleep_idle_polled(anv->rtk, + anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL, &us); + if (ret) + dev_err(anv->dev, "ANS did not idle: %d\n", ret); + else + dev_dbg(anv->dev, "ANS idle, acknowledged in about %u us, CC %#x CSTS %#x\n", + us, cc, csts); + apple_rtkit_mailbox_recv_off(anv->rtk); + + return ret ? 1 : 0; +} + static int apple_nvme_probe(struct platform_device *pdev) { struct apple_nvme *anv; @@ -2560,6 +2678,19 @@ static int apple_nvme_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + if (anv->hw->sleep_traced) { + anv->sleep_registered = + !apple_sleep_coproc_register(anv->dev, anv->mmio_coproc, true, + apple_nvme_sleep_last_stop, anv); + /* Never idled without its shutdown notification. */ + if (anv->sleep_registered && + apple_sleep_coproc_set_last_check(anv->dev, apple_nvme_sleep_shutdown)) { + apple_sleep_coproc_unregister(anv->dev); + anv->sleep_registered = false; + } + apple_rtkit_set_warm_epmap_mask(anv->rtk, anv->hw->sleep_epmap_last_mask); + } + apple_nvme_set_reset_ready(anv, true); nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -2578,6 +2709,8 @@ static void apple_nvme_remove(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + if (anv->sleep_registered) + apple_sleep_coproc_unregister(anv->dev); WRITE_ONCE(anv->removing, true); apple_nvme_set_reset_ready(anv, false); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); @@ -2643,6 +2776,82 @@ static void apple_nvme_shutdown(struct platform_device *pdev) } } +/* macOS sees the firmware report on 68 ms after HELLO. Each try waits RTKit's timeout again. */ +#define APPLE_NVME_SLEEP_ON_TRIES 15 + +/* + * After a real sleep ANS has lost power, and its resident firmware restarts once the run bit + * is set. Measured on macOS under aura, the same in five runs: HELLO first and unprompted, + * then the endpoint map, the last part answered with endpoint 0x20 only. No endpoint is + * started and no buffer asked for. About 68 ms later the firmware reports on, and the AP state + * follows, with the controller's registers reading 0. A refused suspend never told the + * firmware anything, and takes a reset. + * + * The run bit goes in the early phase, so HELLO comes within milliseconds of the wake as on + * macOS. The wait for on doesn't: macOS rings the SEP a few ms after ANS's HELLO, before ANS + * reports on, so the reset work waits, after the SEP's own resume has had its turn. + */ +static int apple_nvme_resume_early(struct device *dev) +{ + struct apple_nvme *anv = dev_get_drvdata(dev); + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret; + + if (!anv->hw->needs_ioq_registers || !anv->suspended || !anv->sleep_idled) + return 0; + anv->sleep_idled = false; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + /* Its link doesn't come up with the pins as the sleep left them. */ + apple_sleep_restore_pins(); + /* macOS writes the SART's entries back again right before the run bit. */ + if (anv->sart) + apple_sart_replay(anv->sart); + + dev_dbg(dev, "ANS: restarting after the sleep, CPU control %#x\n", readl(control)); + anv->sleep_run_time = ktime_get(); + ret = apple_rtkit_sleep_return_start(anv->rtk, control); + if (ret) { + dev_crit(dev, "ANS restart after sleep failed: %d\n", ret); + anv->sleep_failed = true; + return 0; + } + WRITE_ONCE(anv->sleep_restarting, true); + + return 0; +} + +/* From the reset work: wait for ANS to report on. False means hold every request. */ +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv) +{ + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret, tries = 0; + + do { + ret = apple_rtkit_boot(anv->rtk); + if (ret != -ETIME) + break; + dev_warn(anv->dev, "ANS: not on %lld ms after the run bit, waiting again\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + } while (++tries < APPLE_NVME_SLEEP_ON_TRIES); + + WRITE_ONCE(anv->sleep_restarting, false); + if (ret) { + dev_crit(anv->dev, "ANS did not come back after sleep: %d after %lld ms, CPU control %#x status %#x, crashed %d\n", + ret, ktime_ms_delta(ktime_get(), anv->sleep_run_time), readl(control), + readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_STATUS), + apple_rtkit_is_crashed(anv->rtk)); + dev_crit(anv->dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return false; + } + WRITE_ONCE(anv->sleep_restarted, true); + /* The one line per wake that says ANS is back. */ + dev_info(anv->dev, "ANS: on %lld ms after the run bit, resetting the controller before any I/O\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + + return true; +} + static int apple_nvme_resume(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2652,6 +2861,20 @@ static int apple_nvme_resume(struct device *dev) return -EIO; if (!anv->suspended) return 0; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + anv->sleep_idled = false; + anv->sleep_stop = false; + /* + * Hold every request rather than fail it: the queues stay frozen and quiesced + * from the suspend, so nothing reaches a controller that isn't there and no + * filesystem sees a write error. Only a reboot brings ANS back from this. + */ + if (READ_ONCE(anv->sleep_failed)) { + dev_crit(dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return 0; + } + /* The reset work waits for ANS first, if it's still coming up. */ WRITE_ONCE(anv->suspended, false); return nvme_try_sched_reset(&anv->ctrl); } @@ -2680,19 +2903,74 @@ static void apple_nvme_keep_link_powered(struct apple_nvme *anv) device_set_awake_path(anv->pd_dev[i]); } +/* + * What is written has to be on flash before power goes: an fsynced overwrite is + * known to revert after an abrupt reset on these controllers unless flushed. + */ +static int apple_nvme_sleep_flush(struct apple_nvme *anv) +{ + struct nvme_ctrl *ctrl = &anv->ctrl; + struct nvme_ns *ns; + int srcu_idx, ret = 0; + + srcu_idx = srcu_read_lock(&ctrl->srcu); + list_for_each_entry_srcu(ns, &ctrl->namespaces, list, + srcu_read_lock_held(&ctrl->srcu)) { + struct nvme_command c = { }; + + if (ns->head->ns_id > anv->hw->sleep_flush_max_nsid) + continue; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(ns->head->ns_id); + ret = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (ret) + break; + } + srcu_read_unlock(&ctrl->srcu, srcu_idx); + + return ret; +} + +static bool apple_nvme_sleep_wanted(struct apple_nvme *anv) +{ + return anv->sleep_registered && + pm_suspend_target_state == PM_SUSPEND_MEM && pm_suspend_via_firmware() && + nvme_ctrl_state(&anv->ctrl) == NVME_CTRL_LIVE && anv->owns_rtkit && + apple_rtkit_is_running(anv->rtk); +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); int ret; if (anv->hw->needs_ioq_registers) { - if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) + anv->sleep_stop = apple_nvme_sleep_wanted(anv); + anv->sleep_idled = false; + anv->sleep_failed = false; + anv->sleep_restarted = false; + if (anv->sleep_stop) { + ret = apple_nvme_sleep_flush(anv); + if (ret) { + dev_err(dev, "refusing to sleep: flush failed: %d\n", ret); + anv->sleep_stop = false; + return -EIO; + } + } + if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) { + anv->sleep_stop = false; return -EBUSY; + } WRITE_ONCE(anv->suspended, true); } - ret = apple_nvme_disable(anv, true); - if (ret) - return ret; + if (anv->sleep_stop) { + anv->sleep_failed = apple_nvme_sleep_quiesce(anv); + } else { + ret = apple_nvme_disable(anv, true); + if (ret) + return ret; + } /* * Only the NVMe controller was disabled. Its owned RTKit session stays * live, so resume rebuilds queues without clearing CPU_CONTROL.RUN. @@ -2700,6 +2978,19 @@ static int apple_nvme_suspend(struct device *dev) if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); apple_nvme_keep_link_powered(anv); + if (anv->sleep_failed) { + /* Requests are stuck, so bring it back and refuse. */ + dev_err(dev, "refusing to sleep: NVMe didn't quiesce\n"); + anv->sleep_failed = false; + anv->sleep_stop = false; + WRITE_ONCE(anv->suspended, false); + nvme_try_sched_reset(&anv->ctrl); + return -EIO; + } + if (anv->sleep_stop) { + apple_sleep_coproc_set_ready(dev, true); + dev_dbg(dev, "NVMe suspended for a sleep\n"); + } return 0; } @@ -2712,8 +3003,10 @@ static int apple_nvme_suspend(struct device *dev) return ret; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, - apple_nvme_resume); +static const struct dev_pm_ops apple_nvme_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_nvme_suspend, apple_nvme_resume) + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_nvme_resume_early) +}; static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, @@ -2746,6 +3039,10 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .has_separate_nvmmu = true, .needs_ioq_registers = true, .max_queue_depth = 64, + .sleep_traced = true, + .sleep_epmap_last_mask = BIT(0), + .sleep_flush_max_nsid = 1, + .sleep_shn = NVME_CC_SHN_MASK, /* * An fsynced overwrite completed and later reverted after an abrupt * reset; explicit flushes persisted the preceding data. Let the block diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h index 21627bfa2ad4a..46dddb9f42610 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h @@ -15,7 +15,10 @@ #define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) #define PIODMA_IRQ_LIMIT 16 -/* Only RELEASE before publication can end guard ownership. No reset event. */ +/* + * Only RELEASE before publication can end guard ownership, or POWER_LOST: the + * engine's whole register file reading as reset after the SoC powered it off. + */ enum apple_piodma_state { PIODMA_IDLE, PIODMA_GUARDED, @@ -37,6 +40,7 @@ enum apple_piodma_event { PIODMA_COMPLETE, PIODMA_ADMIT, PIODMA_FAIL, + PIODMA_POWER_LOST, PIODMA_EVENT_COUNT, }; @@ -105,6 +109,10 @@ static inline int apple_piodma_next(enum apple_piodma_state state, if (state == PIODMA_RETAINED) return PIODMA_FAILED_RETAINED; break; + case PIODMA_POWER_LOST: + if (state == PIODMA_ADMITTED) + return PIODMA_GUARDED; + break; default: break; } diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c index 941a1786c0483..9cccb3ca4bcd0 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c @@ -61,12 +61,12 @@ static void apple_piodma_transition_guards_test(struct kunit *test) { int state, event, next; - /* Exhaust all transitions: no retained state can release or retry. */ + /* Exhaust all transitions: only a power loss can end a retained state. */ for (state = PIODMA_IDLE; state < PIODMA_STATE_COUNT; state++) { for (event = PIODMA_GUARD; event < PIODMA_EVENT_COUNT; event++) { next = apple_piodma_next(state, event); if (apple_piodma_retained(state)) { - if (next >= 0) + if (next >= 0 && event != PIODMA_POWER_LOST) KUNIT_EXPECT_TRUE(test, apple_piodma_retained(next)); if (event == PIODMA_RELEASE || event == PIODMA_PRIME || event == PIODMA_GUARD) @@ -128,8 +128,24 @@ static void apple_piodma_irq_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 1, 2)); } +/* Losing power reopens the guard from an admitted radio, and from nothing else. */ +static void apple_piodma_power_lost_test(struct kunit *test) +{ + static const enum apple_piodma_state refused[] = { + PIODMA_IDLE, PIODMA_GUARDED, PIODMA_ATTEMPTED, PIODMA_FAILED, + PIODMA_REJECTED, PIODMA_RETAINED, PIODMA_READY, PIODMA_FAILED_RETAINED, + }; + int i; + + KUNIT_EXPECT_EQ(test, apple_piodma_next(PIODMA_ADMITTED, PIODMA_POWER_LOST), + PIODMA_GUARDED); + for (i = 0; i < ARRAY_SIZE(refused); i++) + KUNIT_EXPECT_LT(test, apple_piodma_next(refused[i], PIODMA_POWER_LOST), 0); +} + static struct kunit_case apple_piodma_cases[] = { KUNIT_CASE(apple_piodma_lifecycle_test), + KUNIT_CASE(apple_piodma_power_lost_test), KUNIT_CASE(apple_piodma_failure_test), KUNIT_CASE(apple_piodma_transition_guards_test), KUNIT_CASE(apple_piodma_arena_test), diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c index a95dd5112f201..9261d23abee16 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c @@ -92,6 +92,16 @@ struct apple_t8140_bootstrap { struct kexec_blocker kexec_blocker; struct delayed_work admission_retry; struct device *retry_host; + /* The host stops the radios for a sleep that takes this engine's power. */ + bool sleep_armed; + bool power_lost; + /* After a real sleep: read the engine once the host's port is up again. */ + bool power_check; + bool irq_held; + int (*radio_stop)(struct pci_dev *pdev); + struct module *radio_stop_owner; + void (*radio_recover)(void *data); + void *radio_recover_data; }; /* A successful supplier probe owns its unpublished arena until full reset. */ @@ -193,6 +203,8 @@ static int apple_piodma_pm_prepare(struct device *dev) return 0; if (apple_piodma_sleep_allowed(apple_piodma_state(bootstrap), pm_suspend_no_platform())) return 0; + if (READ_ONCE(bootstrap->sleep_armed)) + return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this * supplier probed. Its later image/restore/suspend ioctls go straight @@ -372,6 +384,10 @@ static irqreturn_t apple_t8140_bootstrap_irq(int irq, void *data) u32 fabric, status, tag, active; int count; + /* After a real sleep its block can't answer until the host's port is back. */ + if (READ_ONCE(bootstrap->irq_held)) + return IRQ_NONE; + /* Acknowledge order follows the attributed prior controller contract. */ fabric = readl(bootstrap->fabric); writel(fabric, bootstrap->fabric); @@ -744,6 +760,60 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + WRITE_ONCE(bootstrap->sleep_armed, armed); + /* A check still pending means the block never came back, and stays unread. */ + if (armed) + bootstrap->power_lost = false; +} + +/* The next bootstrap starts as the first did, on the same retained memory. */ +int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + void *arena = bootstrap->arena; + int i; + + /* + * Everything reading as it did before the first command is what shows the sleep took + * the engine's pointers with it. Read only now: the block answers once the host's + * port is set up, after resume_noirq. + */ + if (bootstrap->power_check) { + bootstrap->power_check = false; + bootstrap->power_lost = !apple_t8140_bootstrap_idle(bootstrap); + if (!bootstrap->power_lost) + dev_err(supplier, "engine kept state across sleep; radios stay down\n"); + WRITE_ONCE(bootstrap->irq_held, false); + } + if (!bootstrap->power_lost || READ_ONCE(bootstrap->irq_fault)) + return -EIO; + bootstrap->power_lost = false; + + memset(arena, 0, PIODMA_ARENA_SIZE); + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { + void *slot = arena + i * PIODMA_SLOT_SIZE; + + memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); + *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); + memset(bootstrap->slots[i].packet, 0, sizeof(bootstrap->slots[i].packet)); + memset(bootstrap->slots[i].completion, 0, sizeof(bootstrap->slots[i].completion)); + bootstrap->slots[i].result = 0; + WRITE_ONCE(bootstrap->slots[i].matched_status, 0); + bootstrap->slots[i].submitted = false; + bootstrap->slots[i].observed = false; + reinit_completion(&bootstrap->slots[i].event); + } + dma_wmb(); + WRITE_ONCE(bootstrap->active_tag, 0); + atomic_set(&bootstrap->irq_count, 0); + + return apple_piodma_transition(bootstrap, PIODMA_POWER_LOST); +} + bool apple_piodma_bootstrap_retained(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -753,6 +823,43 @@ bool apple_piodma_bootstrap_retained(struct device *supplier) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); +/* The stop and the module it's in, set and read together. */ +static DEFINE_SPINLOCK(radio_stop_lock); + +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + int (*stop)(struct pci_dev *pdev); + struct module *owner; + bool held; + int ret; + + /* Its module stays while it runs. One that is unloading: its remove stops the radio. */ + spin_lock(&radio_stop_lock); + stop = bootstrap->radio_stop; + owner = bootstrap->radio_stop_owner; + held = !stop || try_module_get(owner); + spin_unlock(&radio_stop_lock); + if (!held) + return -EBUSY; + if (!stop) + return 0; + + ret = stop(pdev); + module_put(owner); + + return ret; +} + +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + bootstrap->radio_recover_data = data; + WRITE_ONCE(bootstrap->radio_recover, recover); +} + void apple_piodma_bootstrap_admitted(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -790,6 +897,31 @@ int apple_piodma_radio_check(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(apple_piodma_radio_check); +void apple_piodma_radio_recover(void) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + void (*recover)(void *data) = bootstrap ? READ_ONCE(bootstrap->radio_recover) : NULL; + + if (recover) + recover(bootstrap->radio_recover_data); +} +EXPORT_SYMBOL_GPL(apple_piodma_radio_recover); + +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + + if (!bootstrap) + return; + spin_lock(&radio_stop_lock); + bootstrap->radio_stop = stop; + bootstrap->radio_stop_owner = owner; + spin_unlock(&radio_stop_lock); +} +EXPORT_SYMBOL_GPL(__apple_piodma_radio_set_stop); + static int apple_t8140_bootstrap_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -822,7 +954,8 @@ static int apple_t8140_bootstrap_probe(struct platform_device *pdev) ret = apple_t8140_bootstrap_resources(pdev, bootstrap); if (ret) goto err_arena; - ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, 0, + /* The host's resume_noirq waits for one completion, with device interrupts still off. */ + ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, IRQF_EARLY_RESUME, dev_name(dev), bootstrap); if (ret) goto err_arena; @@ -873,12 +1006,20 @@ static int apple_t8140_bootstrap_sleep_check(struct apple_t8140_bootstrap *boots static int apple_t8140_bootstrap_suspend_noirq(struct device *dev) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); + int ret; if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !READ_ONCE(bootstrap->sleep_armed)) return -EOPNOTSUPP; - return apple_t8140_bootstrap_sleep_check(bootstrap); + /* Radios that failed to come back don't cost the machine a deep sleep. */ + if (READ_ONCE(bootstrap->sleep_armed) && + (!apple_piodma_ready(apple_piodma_state(bootstrap)) || bootstrap->power_check)) + return 0; + ret = apple_t8140_bootstrap_sleep_check(bootstrap); + if (!ret && READ_ONCE(bootstrap->sleep_armed)) + WRITE_ONCE(bootstrap->irq_held, true); + return ret; } static int apple_t8140_bootstrap_resume_noirq(struct device *dev) @@ -890,6 +1031,15 @@ static int apple_t8140_bootstrap_resume_noirq(struct device *dev) return 0; if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; + /* + * Its block reads nothing until the host sets its port up again, so neither read it + * nor take its interrupt until then: apple_piodma_bootstrap_power_lost() checks it. + */ + if (READ_ONCE(bootstrap->sleep_armed) && pm_resume_via_firmware()) { + bootstrap->power_check = true; + return 0; + } + WRITE_ONCE(bootstrap->irq_held, false); /* Genpd may gate this island; restore configuration, never a command. */ ret = apple_t8140_bootstrap_initialize(bootstrap); if (ret) diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h index b7c4eeedb92c9..713ad546cbedc 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h @@ -17,6 +17,11 @@ void apple_piodma_bootstrap_guard_end(struct device *supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed); +int apple_piodma_bootstrap_power_lost(struct device *supplier); +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev); +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data); #else static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { @@ -46,5 +51,24 @@ static inline bool apple_piodma_bootstrap_retained(struct device *supplier) { return false; } + +static inline void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ +} + +static inline int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + return -ENODEV; +} + +static inline void apple_piodma_bootstrap_set_recover(struct device *supplier, + void (*recover)(void *data), void *data) +{ +} #endif #endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 67d4f3050e06c..df1a6cbeeacbf 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -505,6 +506,19 @@ struct apple_pcie { bool neo_config_ready; u8 neo_secondary_bus; struct device *piodma_supplier; + /* What it takes to come back from a deep sleep, if the loader gave it. */ + struct apple_pcie_neo_cold *neo_cold; + struct notifier_block neo_pm_nb; + bool neo_sleep_armed; + struct work_struct neo_recover_work; + bool neo_recover; + /* The functions' state from the last sleep that found them answering */ + struct pci_saved_state *neo_fn_state[2]; + /* Their MSI-X, set aside while they're out of reach */ + bool neo_fn_msix[2]; + bool neo_rc_down; + /* Down already when the sleep began: a failed wake left it unprogrammed */ + bool neo_rc_was_down; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -533,6 +547,8 @@ struct apple_pcie_port { unsigned int link_irqs[2]; u32 saved_intmask; int sid_map_sz; + struct gpio_desc *perst_gpio; + struct gpio_desc *pwren_gpio; int idx; bool started; bool needs_stop; @@ -2363,6 +2379,13 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, u32 stat; int ret; + /* Bringing the link up again, after a deep sleep. */ + if (port->perst_gpio) { + reset = port->perst_gpio; + pwren = port->pwren_gpio; + goto have_gpios; + } + /* * Assert PERST# and configure the pin as output. * The Aquantia AQC113 10GB nic used desktop macs is sensitive to @@ -2400,7 +2423,12 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, else return PTR_ERR(pwren); } + if (!num_aux_resets) { + port->perst_gpio = reset; + port->pwren_gpio = pwren; + } +have_gpios: rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ @@ -3376,6 +3404,10 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); + /* Nothing answers on a host that has lost power and isn't set up again. */ + if (READ_ONCE(pcie->neo_rc_down)) + return NULL; + if (bus->number != cfg->busr.start) { /* Pair with ready publication before consuming the bus number. */ if (!smp_load_acquire(&pcie->neo_config_ready) || @@ -3932,6 +3964,233 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) return ret; } +/* + * What the bootloader did to this host at boot, on the same registers, in its + * order, with its tunables. On this SoC a deep sleep powers all of it off. + */ +struct apple_pcie_neo_cold { + void __iomem *axi2af; + void __iomem *cio3pll; + void __iomem *clkgen; + void __iomem *phy_block; + struct apple_tunable *axi2af_tunable; + struct apple_tunable *cio3pll_tunable; + struct apple_tunable *clkgen_tunable; + struct apple_tunable *phy_tunable; + struct apple_tunable *port_tunable; + struct apple_tunable *rc_tunable; + struct apple_tunable *rc_gen3_tunable; + struct apple_tunable *rc_gen4_tunable; + /* The port's DART, clocked by the port: restored after the port is set up. */ + struct device *dart; +}; + +#define NEO_PHY_CTRL 0x0 +#define NEO_PHY_CTRL_CLK0REQ BIT(0) +#define NEO_PHY_CTRL_CLK1REQ BIT(1) +#define NEO_PHY_CTRL_CLK0ACK BIT(2) +#define NEO_PHY_CTRL_CLK1ACK BIT(3) +#define NEO_PHY_CTRL_RESET BIT(4) +#define NEO_PHY_RUN 0x4 +#define NEO_PHY_READY 0x8 +#define NEO_PHY_CTRL_UP BIT(27) +#define NEO_PHY_SETTLE_US 300 +#define NEO_AXI_EN0 0x104 +#define NEO_AXI_EN1 0x108 +#define NEO_CLK_CTRL 0x0 +#define NEO_CLK_BUSY BIT(31) +#define NEO_CLK_OFF BIT(2) +#define NEO_CLK_START 0x4 +#define NEO_CLK_START_GO BIT(31) +#define NEO_PHY_BLOCK_COMMON 0x4000 +#define NEO_PHY_BLOCK_PHY 0x8000 +#define NEO_PHY_POLL_US 50000 +#define NEO_RC_WRITE_ZERO 0x04 +#define NEO_RC_READY_CFG 0x54 +#define NEO_RC_READY_START 0x50 +#define NEO_RC_READY_STAT 0x58 +#define NEO_RC_POLL_US 250000 +#define NEO_DBI_CAP_LNKCAP 0x7c +#define NEO_DBI_CAP_LNKCAP2 0x9c +#define NEO_DBI_LNKCAP2_SLS GENMASK(6, 1) +#define NEO_DBI_CAP_LNKCTL2 0xa0 +#define NEO_DBI_PORT_LINK_CONTROL 0x710 +#define NEO_DBI_PORT_LINK_MODE GENMASK(21, 16) +#define NEO_DBI_WIDTH_SPEED_CONTROL 0x80c +#define NEO_DBI_LINK_WIDTH GENMASK(12, 8) +#define NEO_DBI_SPEED_CHANGE BIT(17) +#define NEO_DBI_RO_WR 0x8bc +#define NEO_DBI_RO_WR_EN BIT(0) + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001000ff }, + { 0x82c, 0x00000000 }, +}; + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs2[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static int apple_pcie_neo_phy_clocks(void __iomem *phy) +{ + u32 stat; + int ret; + + rmw_set(NEO_PHY_CTRL_CLK0REQ, phy + NEO_PHY_CTRL); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK0ACK, + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(NEO_PHY_CTRL_CLK1REQ, phy + NEO_PHY_CTRL); + return readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK1ACK, + 10, NEO_PHY_POLL_US); +} + +/* + * The part of the block that ANS shares, from power-off in macOS's order, measured on + * one sleep. The sleep driver does it at the wake when ANS is bound, and it can't be + * redone under a running PHY. + */ +static int apple_pcie_neo_common_init(struct apple_pcie *pcie) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *phy = cold->phy_block + NEO_PHY_BLOCK_PHY; + u32 stat, val; + int ret; + + if (readl_relaxed(pcie->base + NEO_RC_READY_STAT) & BIT(0)) + return 0; + + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN0); + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN1); + apple_tunable_apply(cold->axi2af, cold->axi2af_tunable); + apple_tunable_apply(cold->cio3pll, cold->cio3pll_tunable); + apple_tunable_apply(cold->clkgen, cold->clkgen_tunable); + rmw_set(NEO_CLK_START_GO, cold->clkgen + NEO_CLK_START); + rmw_clear(NEO_CLK_OFF, cold->clkgen + NEO_CLK_CTRL); + ret = readl_relaxed_poll_timeout(cold->clkgen + NEO_CLK_CTRL, stat, !(stat & NEO_CLK_BUSY), + 1, NEO_PHY_POLL_US); + if (ret) + return ret; + apple_tunable_apply(cold->phy_block, cold->phy_tunable); + + ret = apple_pcie_neo_phy_clocks(phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, phy + NEO_PHY_CTRL); + udelay(1); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, !(stat & NEO_PHY_CTRL_RESET), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(BIT(0), phy + NEO_PHY_RUN); + udelay(NEO_PHY_SETTLE_US); + rmw_set(BIT(4), phy + NEO_PHY_RUN); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_READY, stat, stat & BIT(0), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + + val = readl_relaxed(cold->phy_block + NEO_PHY_BLOCK_COMMON); + writel_relaxed((val & ~GENMASK(1, 0)) | 1, cold->phy_block + NEO_PHY_BLOCK_COMMON); + rmw_set(NEO_PHY_CTRL_UP, phy + NEO_PHY_CTRL); + + writel_relaxed(0, pcie->base + NEO_RC_WRITE_ZERO); + writel_relaxed(0x140, pcie->base + NEO_RC_READY_CFG); + writel_relaxed(1, pcie->base + NEO_RC_READY_START); + + return readl_relaxed_poll_timeout(pcie->base + NEO_RC_READY_STAT, stat, stat & BIT(0), + 100, NEO_RC_POLL_US); +} + +static int apple_pcie_neo_cold_init(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + int speed = of_pci_get_max_link_speed(port->np); + u32 stat, val; + int i, ret; + + ret = apple_pcie_neo_common_init(pcie); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs[i].value, + apple_pcie_neo_port_regs[i].offset); + for (i = 0; i < 16; i++) + apple_pcie_port_writel(port, 0, 0x3000 + 4 * i); + for (i = 0; i < 512; i++) + apple_pcie_port_writel(port, 0, pcie->hw->port_msimap + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs2); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs2[i].value, + apple_pcie_neo_port_regs2[i].offset); + apple_tunable_apply(port->base, cold->port_tunable); + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* PERST# off before the PHY's clocks, as macOS has it coming from power-off. */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, NEO_RC_POLL_US); + if (ret) + return ret; + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, + !(stat & PORT_LINKSTS_BUSY), 100, NEO_RC_POLL_US); + if (ret) + return ret; + + rmw_clear(NEO_PHY_CTRL_CLK0REQ | NEO_PHY_CTRL_CLK1REQ, port->phy + NEO_PHY_CTRL); + ret = apple_pcie_neo_phy_clocks(port->phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, port->phy + NEO_PHY_CTRL); + rmw_set(BIT(9), port->phy + NEO_PHY_CTRL); + rmw_set(BIT(10), port->phy + NEO_PHY_CTRL); + + rmw_set(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + apple_tunable_apply(dbi, cold->rc_tunable); + apple_tunable_apply(dbi, cold->rc_gen3_tunable); + apple_tunable_apply(dbi, cold->rc_gen4_tunable); + if (speed > 0) { + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP); + writel_relaxed((val & ~PCI_EXP_LNKCAP_SLS) | speed, dbi + NEO_DBI_CAP_LNKCAP); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP2); + writel_relaxed((val & ~NEO_DBI_LNKCAP2_SLS) | + FIELD_PREP(NEO_DBI_LNKCAP2_SLS, (1 << speed) - 1), + dbi + NEO_DBI_CAP_LNKCAP2); + writew_relaxed((readw_relaxed(dbi + NEO_DBI_CAP_LNKCTL2) & ~PCI_EXP_LNKCTL2_TLS) | + speed, dbi + NEO_DBI_CAP_LNKCTL2); + rmw_set(NEO_DBI_SPEED_CHANGE, dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + } + val = readl_relaxed(dbi + NEO_DBI_PORT_LINK_CONTROL) & ~NEO_DBI_PORT_LINK_MODE; + writel_relaxed(val | FIELD_PREP(NEO_DBI_PORT_LINK_MODE, 1), + dbi + NEO_DBI_PORT_LINK_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_WIDTH_SPEED_CONTROL) & ~NEO_DBI_LINK_WIDTH; + writel_relaxed(val | FIELD_PREP(NEO_DBI_LINK_WIDTH, 1), dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP) & ~PCI_EXP_LNKCAP_MLW; + writel_relaxed(val | FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1), dbi + NEO_DBI_CAP_LNKCAP); + rmw_clear(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + + return 0; +} + static int apple_pcie_attach_domains(struct apple_pcie *pcie) { struct device *dev = pcie->dev; @@ -3970,6 +4229,169 @@ static int apple_pcie_attach_domains(struct apple_pcie *pcie) return 0; } +static void apple_pcie_neo_recover_work(struct work_struct *work); + +/* A radio left unusable by the wake, or its link. POST_SUSPEND runs this once all is thawed. */ +static void apple_pcie_neo_recover(void *data) +{ + struct apple_pcie *pcie = data; + + WRITE_ONCE(pcie->neo_recover, true); + /* Pairs with the barrier at PM_POST_SUSPEND: one side sees the other's write. */ + smp_mb(); + if (!READ_ONCE(pcie->neo_sleep_armed)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); +} + +/* Runs before the freezer. The radios stay bound, and their drivers do their own suspend. */ +static int apple_pcie_neo_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct apple_pcie *pcie = container_of(nb, struct apple_pcie, neo_pm_nb); + + switch (action) { + case PM_SUSPEND_PREPARE: + if (pm_suspend_default_s2idle()) + break; + flush_work(&pcie->neo_recover_work); + pcie->neo_sleep_armed = true; + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, true); + break; + case PM_POST_SUSPEND: + if (!pcie->neo_sleep_armed) + break; + WRITE_ONCE(pcie->neo_sleep_armed, false); + /* Pairs with the barrier in the recovery request. */ + smp_mb(); + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, false); + if (READ_ONCE(pcie->neo_recover)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +static void __iomem *apple_pcie_neo_map(struct platform_device *pdev, const char *name, + struct resource **res) +{ + *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + if (!*res) + return IOMEM_ERR_PTR(-ENOENT); + + /* These overlap each other and the port PHYs, so they can't be claimed. */ + return devm_ioremap(&pdev->dev, (*res)->start, resource_size(*res)) ?: + IOMEM_ERR_PTR(-ENOMEM); +} + +/* A shared block and the tunable the loader publishes for it. */ +static int apple_pcie_neo_get(struct platform_device *pdev, const char *name, const char *prop, + void __iomem **base, struct apple_tunable **tunable) +{ + struct resource *res; + + *base = apple_pcie_neo_map(pdev, name, &res); + if (IS_ERR(*base)) + return PTR_ERR(*base); + *tunable = devm_apple_tunable_parse(&pdev->dev, pdev->dev.of_node, prop, res); + + return PTR_ERR_OR_ZERO(*tunable); +} + +/* The DART in the host's iommu-map, which reads nothing until the port is set up. */ +static struct device *apple_pcie_neo_dart(struct device *dev) +{ + struct device_node *np = of_parse_phandle(dev->of_node, "iommu-map", 1); + struct platform_device *pdev = np ? of_find_device_by_node(np) : NULL; + int ret; + + of_node_put(np); + if (!pdev) + return NULL; + /* The reference is kept: this host can't be unbound. */ + ret = apple_dart_set_host_resumes(&pdev->dev); + /* The DART resumes, and gates itself, before this host restores it. */ + if (!ret && !device_link_add(dev, &pdev->dev, DL_FLAG_STATELESS)) + ret = -EINVAL; + if (ret) { + put_device(&pdev->dev); + dev_warn(dev, "no deep sleep: its DART can't wait for the port: %d\n", ret); + return NULL; + } + return &pdev->dev; +} + +/* Without all of this the host can't come back, and a deep sleep stays refused. */ +static void apple_pcie_neo_sleep_probe(struct platform_device *pdev, struct apple_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pcie_neo_cold *cold; + struct apple_pcie_port *port; + struct resource *res, *cfg; + int ret; + + port = list_first_entry_or_null(&pcie->ports, struct apple_pcie_port, entry); + cfg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!port || !cfg || !port->perst_gpio || !port->phy) { + dev_warn(dev, "no deep sleep: nothing here took the link up, so no pins to restart it\n"); + return; + } + cold = devm_kzalloc(dev, sizeof(*cold), GFP_KERNEL); + if (!cold) + return; + + if (apple_pcie_neo_get(pdev, "axi2af", "apple,tunable-axi2af", &cold->axi2af, + &cold->axi2af_tunable) || + apple_pcie_neo_get(pdev, "cio3pll", "apple,tunable-cio3pll", &cold->cio3pll, + &cold->cio3pll_tunable) || + apple_pcie_neo_get(pdev, "clkgen", "apple,tunable-clkgen", &cold->clkgen, + &cold->clkgen_tunable)) + goto absent; + cold->phy_block = apple_pcie_neo_map(pdev, "phy-block", &res); + if (IS_ERR(cold->phy_block)) + goto absent; + cold->phy_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-phy", res); + if (IS_ERR(cold->phy_tunable)) + goto absent; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "port0"); + if (!res) + goto absent; + cold->port_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-port", res); + if (IS_ERR(cold->port_tunable)) + goto absent; + cold->rc_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc", cfg); + if (IS_ERR(cold->rc_tunable)) + goto absent; + cold->rc_gen3_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen3", + cfg); + if (IS_ERR(cold->rc_gen3_tunable)) + goto absent; + cold->rc_gen4_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen4", + cfg); + if (IS_ERR(cold->rc_gen4_tunable)) + goto absent; + cold->dart = apple_pcie_neo_dart(dev); + if (!cold->dart) + goto absent; + + pcie->neo_cold = cold; + INIT_WORK(&pcie->neo_recover_work, apple_pcie_neo_recover_work); + apple_piodma_bootstrap_set_recover(pcie->piodma_supplier, apple_pcie_neo_recover, pcie); + pcie->neo_pm_nb.notifier_call = apple_pcie_neo_pm_notify; + ret = register_pm_notifier(&pcie->neo_pm_nb); + if (ret) + pcie->neo_cold = NULL; + else + dev_dbg(dev, "deep sleep: radios stay powered, link stopped and retrained\n"); + return; + +absent: + dev_warn(dev, "no deep sleep: the loader didn't publish what this host needs\n"); +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -4185,6 +4607,8 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) dev_err(dev, "radio admission failed=%d; host/supplier retained, no retry\n", ret); + else + apple_pcie_neo_sleep_probe(pdev, pcie); return 0; } @@ -4859,6 +5283,8 @@ static void apple_pcie_stop_for_sleep(struct device *dev) apple_pcie_tunnel_hierarchy_lost(dev); } +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie); + /* * The J716c radio drivers save PCI state without entering D3. The PCI core * keeps the root port in D0 as well. Its host and DART have no switchable @@ -4916,6 +5342,34 @@ static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); + if (pcie->neo_sleep_armed) { + struct apple_pcie_port *port; + + pcie->neo_rc_was_down = READ_ONCE(pcie->neo_rc_down); + /* A port that didn't come back from the last sleep keeps what it saved then. */ + list_for_each_entry(port, &pcie->ports, entry) { + int i; + + apple_pcie_port_disable_irq(port); + if (pcie->neo_rc_was_down) + continue; + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + } + /* The functions too: the core read them through closed config space. */ + if (!pcie->neo_rc_was_down) + apple_pcie_neo_save_functions(pcie); + WRITE_ONCE(pcie->neo_rc_down, true); + /* + * Genpd would gate the host here, and an abandoned sleep then finds + * it unprogrammed with nothing to bring it up. The commit lowers it. + */ + device_set_awake_path(dev); + return 0; + } + /* * The managed radio has no suspend boundary of its own; suspend-to-idle * leaves it in place like any other internal port. @@ -4981,6 +5435,324 @@ static int apple_pcie_reset_for_resume(struct apple_pcie *pcie) return 0; } +/* + * The radio kept its power and firmware, in reset since the commit. Its rail isn't touched: + * cycling it, or the port's own reset, is what a cold start needs. PERST# released, the link + * trains. Device interrupts are off in this phase, so this polls for it. + */ +static int apple_pcie_neo_link_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + usleep_range(1000, 2000); + ret = gpiod_set_value_cansleep(port->perst_gpio, 0); + if (ret) + return ret; + /* 100 ms after PERST# as at boot, and after the link (PCIe r5.0, 6.6.1). */ + msleep(100); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, 250000); + if (ret) + return ret; + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, stat & PORT_LINKSTS_UP, + 100, link_up_timeout * 1000); + if (ret) + return ret; + apple_pcie_port_writel(port, BIT(PORT_INT_LINK_UP), PORT_INTSTAT); + msleep(100); + + return 0; +} + +/* + * The PCI core saves the functions again at every sleep, and with the radio down it reads them + * through closed config space. Keep the last state they saved while they answered. + */ +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) + return; + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + kfree(pcie->neo_fn_state[i]); + pcie->neo_fn_state[i] = pci_store_saved_state(fn); + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* With the link back: what the core restores next is that state. */ +static void apple_pcie_neo_load_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int i; + + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (pcie->neo_fn_state[i]) + pci_load_saved_state(fn, pcie->neo_fn_state[i]); + pci_dev_put(fn); + } +} + +/* + * Closed config space keeps the PCI core's restore out of the functions, except for one thing: + * it writes an MSI-X table in their BAR, with no check of its own. With the functions out of + * reach, set that aside so that it has nothing to write; their reach back puts it in place. + */ +static void apple_pcie_neo_msix_gate(struct apple_pcie *pcie, bool reachable) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_msix); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (reachable && pcie->neo_fn_msix[i]) { + fn->msix_enabled = 1; + pcie->neo_fn_msix[i] = false; + } else if (!reachable && fn->msix_enabled) { + fn->msix_enabled = 0; + pcie->neo_fn_msix[i] = true; + } + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* + * The link back, and the one engine read that native ECAM needs on it. The root port's + * bus number is restored first, as that read crosses it. Not a firmware load. + */ +static int apple_pcie_neo_radio_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root; + int ret; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return -ENODEV; + pci_restore_state(root); + ret = apple_pcie_neo_link_up(pcie, port); + if (!ret) + ret = apple_piodma_bootstrap_power_lost(pcie->piodma_supplier); + if (!ret) + ret = apple_piodma_bootstrap_prime(pcie->piodma_supplier, root); + if (!ret) { + apple_piodma_bootstrap_admitted(pcie->piodma_supplier); + apple_pcie_neo_load_functions(pcie, root); + /* The functions' configuration before the flag that says it's there. */ + smp_store_release(&pcie->neo_config_ready, true); + apple_pcie_neo_msix_gate(pcie, true); + } + pci_dev_put(root); + + return ret; +} + +/* + * The rail is the one thing that brings back a chip whose link won't train: macOS's last resort + * for a dead chip too. A cold chip then has its firmware loaded by the probe that follows. + */ +static int apple_pcie_neo_rail_cycle(struct apple_pcie *pcie, struct apple_pcie_port *port, + struct pci_dev *root) +{ + struct pci_dev *fn; + int i, ret; + + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = gpiod_set_value_cansleep(port->pwren_gpio, 0); + if (ret) + return ret; + msleep(pcie->hw->pwren_off_ms); + ret = gpiod_set_value_cansleep(port->pwren_gpio, 1); + if (ret) + return ret; + msleep(pcie->hw->pwren_on_ms); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) + return ret; + /* The PCI core's restore found them gone. Wi-Fi's stop needs its BAR back. */ + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + pci_set_power_state(fn, PCI_D0); + pci_restore_state(fn); + pci_dev_put(fn); + } + return 0; +} + +/* The way a hotplug would. Bluetooth first: Wi-Fi's reset needs it gone. */ +static int apple_pcie_neo_stop_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int ret = 0; + + /* Stopping Wi-Fi writes its BAR, which needs the chip to answer. */ + fn = root->subordinate ? pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)) : NULL; + if (fn && !pci_device_is_present(fn)) + ret = -ENODEV; + pci_dev_put(fn); + if (ret) + return ret; + + pci_lock_rescan_remove(); + if (root->subordinate) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 1)); + if (fn) { + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)); + if (fn) { + ret = apple_piodma_bootstrap_radio_stop(pcie->piodma_supplier, fn); + if (!ret) + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + } + pci_unlock_rescan_remove(); + + return ret; +} + +static void apple_pcie_neo_scan_functions(struct pci_dev *root) +{ + pci_lock_rescan_remove(); + if (root->subordinate) { + pci_scan_child_bus(root->subordinate); + pci_assign_unassigned_bridge_resources(root); + pci_bus_add_devices(root->subordinate); + } + pci_unlock_rescan_remove(); +} + +/* + * Keeping the firmware across a sleep is what's wanted, and this is for when it didn't work: the + * radio lost power, its rings or its IPC. Removing the functions resets the chip, and their + * probes load the firmware again from ROM, as at boot. The link only needs its rail cycled if + * it didn't come back. Otherwise the next sleep tries the whole wake again. + */ +static void apple_pcie_neo_recover_work(struct work_struct *work) +{ + struct apple_pcie *pcie = container_of(work, struct apple_pcie, neo_recover_work); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + struct pci_dev *root; + int ret = 0; + + WRITE_ONCE(pcie->neo_recover, false); + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + if (READ_ONCE(pcie->neo_rc_down)) { + ret = -EIO; + goto out; + } + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) { + dev_warn(pcie->dev, "radio link down after sleep: cycling its rail\n"); + ret = apple_pcie_neo_rail_cycle(pcie, port, root); + if (ret) + goto out; + } + ret = apple_pcie_neo_stop_functions(pcie, root); + if (ret != -ENODEV) + apple_pcie_neo_scan_functions(root); +out: + if (ret) + dev_err(pcie->dev, "radios stay down until the next sleep: %d\n", ret); + else + dev_info(pcie->dev, "radios restarted after sleep, firmware loaded again\n"); + pci_dev_put(root); +} + +/* + * Before the PCI core restores the functions, which need the link. macOS's host bridge + * holds them back the same way until its radio port is enabled. + */ +static int apple_pcie_neo_resume_noirq(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i, ret = 0; + + /* Refused: as before the sleep. A host a failed wake left unprogrammed stays down. */ + if (!pm_resume_via_firmware()) { + if (pcie->neo_rc_was_down) + return 0; + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + WRITE_ONCE(pcie->neo_rc_down, false); + return 0; + } + + list_for_each_entry(port, &pcie->ports, entry) { + /* The pin controller just put it back as it was: released, with no clock yet. */ + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = apple_pcie_neo_cold_init(pcie, port); + if (ret) { + dev_err(pcie->dev, "host did not come back after sleep: %d\n", ret); + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + apple_pcie_tunnel_restore_irq_hw(port); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, port->saved_rid2sid[i]); + } + ret = apple_dart_host_resume(pcie->neo_cold->dart); + if (ret) { + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + WRITE_ONCE(pcie->neo_rc_down, false); + apple_sleep_radio_port_up(); + + /* A failed link stays clocked, and its functions read as gone until the next sleep. */ + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) { + /* The functions' restore then finds nothing, and their drivers say so. */ + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + dev_err(pcie->dev, "radios did not come back after sleep: %d\n", ret); + apple_pcie_neo_recover(pcie); + } else { + dev_dbg(pcie->dev, "radio link back after sleep, rail and firmware kept\n"); + } + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + + return 0; +} + static int apple_pcie_resume_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -4989,6 +5761,8 @@ static int apple_pcie_resume_noirq(struct device *dev) /* The domain's hold for the sleep ends first, whatever follows. */ apple_pcie_release_domains(pcie); + if (pcie->neo_sleep_armed) + return apple_pcie_neo_resume_noirq(pcie); if (pcie->hw->adopt_link) { if (!pcie->link_kept) return 0; diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 36e5afed0227f..09eec7f9d6185 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -564,6 +564,33 @@ static void atcphy_test_update_port_taken(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); } +/* The root hub, once another update that held it has brought the port up again */ +static int atcphy_test_pm_get_after_other(struct apple_atcphy *atcphy, + struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('g'); + guard(mutex)(&atcphy->lock); + atcphy->usb3_port_off = false; + return 0; +} + +/* An update that waited for another one doing the same change leaves the port alone */ +static void atcphy_test_update_after_other(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get_after_other); + steps->mode_port_on = true; + + /* The lanes and the PIPE are in place, the port still owed by the other update */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3_DP); +} + /* * Zeroed memory in place of the core, PIPE handler and USB2 PHY registers: * enough to park the PIPE and power the PHY off, with a PIPE lock request @@ -641,18 +668,25 @@ static struct apple_atcphy *atcphy_test_alloc_failed_handoff(struct kunit *test) return atcphy; } +static int atcphy_test_count_refused_lock(struct apple_atcphy *atcphy); + /* Once dwc3 has stopped, its USB3 PHY power-off parks the PIPE without one */ static void atcphy_test_stopped_power_off_common(struct kunit *test, bool dp) { struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + unsigned int requests = 0; KUNIT_ASSERT_NOT_NULL(test, phy); phy_set_drvdata(phy, atcphy); if (!dp) atcphy->mode = APPLE_ATCPHY_MODE_USB3; + /* The stopped controller can't acknowledge a lock, so none is asked for. */ + test->priv = &requests; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, atcphy_test_count_refused_lock); KUNIT_EXPECT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); KUNIT_EXPECT_EQ(test, atcphy->mode, @@ -1368,6 +1402,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_retry), KUNIT_CASE(atcphy_test_update_unsupported), KUNIT_CASE(atcphy_test_update_port_taken), + KUNIT_CASE(atcphy_test_update_after_other), KUNIT_CASE(atcphy_test_set_port_on), KUNIT_CASE(atcphy_test_host_start), KUNIT_CASE(atcphy_test_stopped_power_off), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 33507b68de52e..b8b4d78817d84 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -558,6 +560,24 @@ enum atcphy_lane_mode { #define USB2PHY_MISCTUNE_APBCLK_GATE_OFF BIT(29) #define USB2PHY_MISCTUNE_REFCLK_GATE_OFF BIT(30) +/* + * The power control of the USB controller the T8140 PHY serves ("pwrctl"). macOS's + * controller driver takes the core into hibernation with it at a system sleep and back + * at the wake, in the sequences below. + */ +#define AUSBC_PWR_CFG 0x00 +#define AUSBC_PWR_CFG_STATE_REQ GENMASK(5, 4) +#define AUSBC_PWR_STATUS 0x04 +#define AUSBC_PWR_STATUS_SLEEP BIT(20) +#define AUSBC_PWR_EVENTS 0x08 +#define AUSBC_PWR_ENABLE 0x10 +#define AUSBC_PWR_CLEAR 0x14 +#define AUSBC_PWR_EVENT_MASK GENMASK(3, 0) +#define AUSBC_PWR_EVENT_D0_ON BIT(0) +#define AUSBC_PWR_EVENT_VDD_OFF BIT(1) +#define AUSBC_PWR_EVENT_WAKE GENMASK(3, 2) +#define AUSBC_PWR_TIMEOUT_US 200000 + enum atcphy_generation { ATCPHY_GENERATION_T8103, ATCPHY_GENERATION_T8122, @@ -858,6 +878,16 @@ struct apple_atcphy { bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; + /* + * The controller sleeps in place for this system sleep, the SuperSpeed side is checked + * against the mux after it, and the controller is in hibernation now. + */ + bool sleep_in_place; + bool sleep_ss_down; + bool hibernated; + int sleep_ret; + /* Going through hibernation failed on this boot: the controller stops from now on. */ + bool sleep_no_keep; bool usb3_port_off; struct delayed_work usb3_port_work; unsigned int usb3_port_retries; @@ -898,6 +928,7 @@ struct apple_atcphy { void __iomem *usb2phy_reg; void __iomem *pipehandler; void __iomem *lpdptx; + void __iomem *pwrctl; } regs; struct { @@ -2675,9 +2706,11 @@ static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) /* * The T8140 keeps per-device defaults for its secondary eUSB2 register * bank in the ADT. The power-off path asserts the PHY resets, so apply - * them after every power-on. + * them after every power-on, but not with the controller's core in + * hibernation: see atcphy_resume_noirq(). */ - apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + if (!atcphy->hibernated) + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); } static int atcphy_power_on(struct apple_atcphy *atcphy) @@ -4171,6 +4204,26 @@ static void atcphy_rpm_release(void *data) atcphy->rpm_held = false; } +/* What a power-down of the common block takes: the DisplayPort side and a tunnel clock. */ +static void atcphy_forget_common_state(struct apple_atcphy *atcphy) +{ + atcphy->dp_t8122.aux = false; + atcphy->dp_t8122.pll = false; + atcphy->dp_t8122.pairs = 0; + atcphy->dp_t8122.rate = 0; + atcphy->tunnel_clock_on = false; + atcphy->tunnel_rate = 0; + atcphy->tunnel_users = 0; + if (atcphy->hw->has_t8140_dp) { + atcphy->dp_owned_pmas = 0; + atcphy->dp_idle_pmas = 0; + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->aux_power_requested = false; + atcphy->mode_dirty = false; + } +} + /* * A T8122 PHY loses its register state whenever its power domain is switched * off. In system sleep that happens to a domain that no running device holds: @@ -4206,13 +4259,7 @@ static void atcphy_restore_after_pd_off(struct apple_atcphy *atcphy) return; WRITE_ONCE(atcphy->pd_was_off, false); - atcphy->dp_t8122.aux = false; - atcphy->dp_t8122.pll = false; - atcphy->dp_t8122.pairs = 0; - atcphy->dp_t8122.rate = 0; - atcphy->tunnel_clock_on = false; - atcphy->tunnel_rate = 0; - atcphy->tunnel_users = 0; + atcphy_forget_common_state(atcphy); if (!atcphy->dp_only) { _atcphy_dwc3_reset_assert(atcphy); @@ -5690,16 +5737,24 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return ret; } -static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +/* Whether an update to @target_mode has nothing left to do, recording the request if so */ +static bool atcphy_mux_done(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + guard(mutex)(&atcphy->lock); + if (!atcphy_mux_is_noop(atcphy, target_mode)) + return false; + atcphy->typec_mode = target_mode; + return true; +} + +static int atcphy_mux_update_to(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { struct atcphy_usb3_pm pm = {}; - enum atcphy_mode target_mode; bool in_mode, port_on, usb_err = true; int ret; scoped_guard(mutex, &atcphy->lock) { atcphy_restore_after_pd_off(atcphy); - target_mode = atcphy_typec_target_mode(atcphy, state); /* * Only a lane or PIPE change, or a SuperSpeed port still owed * its power, needs the USB3 root hub below. Taking it means @@ -5724,6 +5779,13 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux if (ret) goto out; + /* + * Another update holds the root hub for the whole of its change. One that finished + * while this one waited may have left nothing to do: don't cycle the port again. + */ + if (atcphy_mux_done(atcphy, target_mode)) + goto put; + /* * The order in which dwc3 is taken down and brought up around a PHY * change (see dwc3-apple.c), for the SuperSpeed side only: the @@ -5775,6 +5837,11 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux return ret; } +static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +{ + return atcphy_mux_update_to(atcphy, atcphy_typec_target_mode(atcphy, state)); +} + static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); @@ -6181,6 +6248,16 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph atcphy->res.usb2phy_reg = res; } + /* Optional: without it the controller stops for a system sleep. */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwrctl"); + if (res) { + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map pwrctl regs"); + atcphy->regs.pwrctl = addr; + } + return 0; } @@ -6443,6 +6520,353 @@ static int atcphy_probe(struct platform_device *pdev) return 0; } +/* + * A system sleep with the controller kept, as macOS does it on the J700: its host goes to + * Sleep, not Off. xHCI has halted and saved its state by the time this PHY suspends, as + * the controller is its consumer. The PHY then takes the controller's core into + * hibernation through its power control, holds the core's reset without the clamp, and + * puts the eUSB2 PHY in its sleep state: reference clock gated, APB in reset, the link + * left as it is. The sleep's commit lowers ATC0_USB and ATC0_COMMON and keeps + * ATC0_USB_AON, which holds the power control, the PIPE handler and the USB2 PHY. + * + * At the wake ATC0_COMMON is back with the common block at its reset values. It's set up + * in the mode it had again, the controller is brought to D0, and xHCI restores its own state. + * + * USB2 and USB3 qualify, without DisplayPort, whose link the common block's power-down + * would end. With USB3 the halted controller's PIPE is parked on the dummy backend just + * before the hibernation, as macOS locks and parks it when its host goes to Sleep + * (setUSB3Mode, static), and the lanes are left as they are: the commit takes them down. + * At the wake the lanes come back in USB3 and the PIPE is routed to them when the controller's + * core is set up, before xHCI starts, with no Type-C mode change and no SuperSpeed port power + * cycle. macOS's raise from Sleep to its saved USB3 mode is also just a handshake and the USB3 + * PHY's power level (static, setUSB3Mode). macOS also stops its PHY's fabric traffic before + * the clock gate; that block isn't known here, and a halted controller has none. + */ +static bool atcphy_can_sleep_in_place(const struct apple_atcphy *atcphy) +{ + enum atcphy_pipehandler_state pipe = atcphy_pipe_state_for(atcphy, atcphy->mode); + + return atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->regs.pwrctl && + !atcphy->sleep_no_keep && atcphy->host_active && + (atcphy->mode == APPLE_ATCPHY_MODE_USB2 || atcphy->mode == APPLE_ATCPHY_MODE_USB3) && + !atcphy->mode_dirty && !atcphy->pipe_pending && + (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY || + atcphy->pipe_state == pipe) && !atcphy->usb_off_pending && + !atcphy->dp_owned_pmas && !atcphy->aux_power_requested && !atcphy->tunnel_users; +} + +/* + * The halted controller's PIPE goes to the dummy backend for the hibernation. A lock it + * doesn't acknowledge is only logged, as macOS's sleep only logs it too; the PIPE is then + * parked unlocked, as for a stopped controller. + */ +static int atcphy_sleep_park_pipe(struct apple_atcphy *atcphy) +{ + bool dirty = atcphy->mode_dirty; + int ret; + + ret = atcphy_park_pipehandler(atcphy, false); + if (!ret) + return 0; + + dev_warn(atcphy->dev, "PIPE lock not acknowledged for the sleep (%d): parking unlocked\n", + ret); + atcphy->mode_dirty = dirty; + return atcphy_park_pipehandler(atcphy, true); +} + +static void atcphy_usb2_sleep_enter(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); +} + +static void atcphy_usb2_sleep_exit(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); +} + +/* + * macOS's raise from Sleep: D0 asked for, the core's reset released, D0_ON awaited. Returns + * -ENODEV only if the core may be unpowered, when touching its registers would be an SError. + * With @went_off the core's rail is known to have gone off, so only D0_ON will do. Otherwise + * a core no longer reporting SLEEP has left hibernation, and the event alone is missing. + */ +static int atcphy_controller_d0(struct apple_atcphy *atcphy, bool went_off) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 val, status; + int ret; + + /* All zeros: the block lost power in the sleep and was reset, so no D0_ON will come. */ + status = readl(pc + AUSBC_PWR_STATUS); + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, 0); + clear32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + /* As the reset's release does: a block that lost its state may have the clamp on. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + if (!went_off && !status) { + dev_dbg(atcphy->dev, "USB controller lost its power state in the sleep\n"); + return 0; + } + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, val, val & AUSBC_PWR_EVENT_D0_ON, 100, + AUSBC_PWR_TIMEOUT_US); + status = readl(pc + AUSBC_PWR_STATUS); + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, + AUSBC_PWR_EVENT_D0_ON | AUSBC_PWR_EVENT_WAKE); + if (!ret) + return 0; + + dev_err(atcphy->dev, "USB controller gave no D0_ON: events %#x, status %#x\n", val, status); + if (!went_off && !(status & AUSBC_PWR_STATUS_SLEEP)) + return 0; + + return -ENODEV; +} + +/* + * A failure refuses this sleep, and the controller stops for the later ones, so that it + * can't hold every other driver's sleep up. sleep_ret says what state the core is in. + */ +static int atcphy_controller_hibernate(struct apple_atcphy *atcphy) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 events, status = 0; + bool went_off; + int ret; + + set32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, events, events & AUSBC_PWR_EVENT_VDD_OFF, + 100, AUSBC_PWR_TIMEOUT_US); + went_off = !ret; + if (!ret) { + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_VDD_OFF); + /* Armed as macOS arms them. That they drive wake source 70 is inferred. */ + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_WAKE); + ret = readl_poll_timeout(pc + AUSBC_PWR_STATUS, status, + status & AUSBC_PWR_STATUS_SLEEP, 100, + AUSBC_PWR_TIMEOUT_US); + } + if (ret) { + dev_err(atcphy->dev, "USB controller did not hibernate: events %#x, status %#x\n", + events, status); + atcphy->sleep_no_keep = true; + atcphy->sleep_ret = atcphy_controller_d0(atcphy, went_off); + return ret; + } + + /* The core's reset, without the clamp that only macOS's Off path sets. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + atcphy_usb2_sleep_enter(atcphy); + dev_dbg(atcphy->dev, "USB controller in hibernation for the sleep\n"); + + return 0; +} + +/* + * The common block went off with ATC0_COMMON. Set it up again in the mode it had, USB2 or + * USB3, without the controller's reset or a USB2 PHY power cycle: atcphy_configure() powers + * the USB2 PHY on over the running one, which rewrites only what it holds, so the hub's link + * stays. The host role goes in again as at probe, for a USB2 PHY that lost its registers + * too. The PIPE stays parked until the controller's core is set up and routes it. + */ +static int atcphy_common_restore(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode = atcphy->mode; + int ret; + + atcphy_forget_common_state(atcphy); + ret = atcphy_power_off(atcphy); + if (ret) + dev_warn(atcphy->dev, "PHY did not power down after the sleep: %d\n", ret); + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy_setup_pipehandler(atcphy); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + + ret = atcphy_configure(atcphy, mode); + if (!ret || mode == APPLE_ATCPHY_MODE_USB2) + return ret; + /* USB2 keeps the hub; the mux update after the wake then tries the mode again. */ + dev_warn(atcphy->dev, "mode %d not set up again after the sleep (%d): USB2 for now\n", + mode, ret); + return atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB2); +} + +static int atcphy_suspend_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + int ret; + + guard(mutex)(&atcphy->lock); + /* Suspend-to-idle keeps every domain on: nothing to do. */ + if (!atcphy->sleep_in_place || !pm_suspend_via_firmware()) + return 0; + /* A mode change since the prepare: the controller would lose its PIPE. */ + if (!atcphy_can_sleep_in_place(atcphy)) { + dev_err(dev, "USB changed for the sleep (mode %d, PIPE %d, host %d): refusing it\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + atcphy->sleep_ret = -EBUSY; + return -EBUSY; + } + ret = atcphy_sleep_park_pipe(atcphy); + if (ret) { + dev_err(dev, "PIPE not parked for the sleep: %d, refusing it\n", ret); + atcphy->sleep_ret = -EBUSY; + return -EBUSY; + } + + ret = atcphy_controller_hibernate(atcphy); + if (ret) + return ret; + atcphy->hibernated = true; + apple_sleep_usb_set_asleep(true); + + return 0; +} + +/* + * A PHY in a DisplayPort mode doesn't sleep in place, and a sleep that reached its commit + * lowered ATC0_COMMON under it, behind its power domain's back: the lanes and the AUX + * channel went. Have atcphy_resume() set the tracked mode up again, before the display + * controller connects the link anew. + */ +static void atcphy_sleep_lost_dp(struct apple_atcphy *atcphy) +{ + if (!atcphy->hw->has_t8140_dp || !atcphy->hw->restore_after_pd_off || + !pm_resume_via_firmware() || !atcphy_modes[atcphy->mode].enable_dp_aux) + return; + WRITE_ONCE(atcphy->pd_was_off, true); + dev_dbg(atcphy->dev, "DisplayPort mode %d lost in the sleep: setting it up again\n", + atcphy->mode); +} + +static int atcphy_resume_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + int ret = 0; + + guard(mutex)(&atcphy->lock); + if (!atcphy->hibernated) { + atcphy_sleep_lost_dp(atcphy); + return 0; + } + apple_sleep_usb_set_asleep(false); + + atcphy_usb2_sleep_exit(atcphy); + /* Only a sleep that reached the commit lowered ATC0_COMMON. */ + if (pm_resume_via_firmware()) { + ret = atcphy_common_restore(atcphy); + if (ret) + dev_err(dev, "PHY not set up again after the sleep: %d\n", ret); + } + atcphy->sleep_ret = atcphy_controller_d0(atcphy, false) ?: ret; + atcphy->hibernated = false; + if (!atcphy->sleep_ret) { + dev_dbg(dev, "USB controller back in D0 after the sleep, PHY in mode %d\n", + atcphy->mode); + /* + * The APB reset may have taken the secondary bank's defaults. That bank is next + * to the core and macOS's PHY exit doesn't touch it, so it may be under the core's + * rail, and an access to it with the rail off would be an SError: only now. + */ + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + } + + return 0; +} + +static struct apple_atcphy *atcphy_from_usb2_phy(struct phy *phy) +{ + if (!phy || phy->ops != &apple_atc_usb2_phy_ops) + return NULL; + return phy_get_drvdata(phy); +} + +/* + * After a sleep in place the PHY goes back to what the mux last asked for. The wake set the + * block up in its mode again and the controller's core routed the PIPE, so a mux update is + * left only if the mux changed during the sleep or that setup failed. + */ +static void atcphy_sleep_ss_up(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode; + int ret; + + scoped_guard(mutex, &atcphy->lock) { + if (!atcphy->sleep_ss_down) + return; + atcphy->sleep_ss_down = false; + mode = atcphy->typec_mode; + atcphy_restore_after_pd_off(atcphy); + if (atcphy_mux_is_noop(atcphy, mode)) { + dev_dbg(atcphy->dev, "SuperSpeed side already in mode %d after the sleep\n", + mode); + return; + } + } + ret = atcphy_mux_update_to(atcphy, mode); + dev_dbg(atcphy->dev, "SuperSpeed side back after the sleep, mode %d: %d\n", mode, ret); +} + +/* No lane or port change here: anything the sleep needs is done at noirq, in a few ms. */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + + guard(mutex)(&atcphy->lock); + if (atcphy_can_sleep_in_place(atcphy)) { + atcphy->sleep_in_place = true; + atcphy->sleep_ss_down = true; + atcphy->sleep_ret = 0; + return 0; + } + if (atcphy->regs.pwrctl && atcphy->fixed_usb2) + dev_dbg(atcphy->dev, + "USB controller stops for the sleep: mode %d, PIPE %d, host %d\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + + return -EBUSY; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_prepare); + +int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + + return atcphy->sleep_in_place ? atcphy->sleep_ret : -ENODATA; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_result); + +/* + * A controller that was restarted after the sleep is set up as at boot and may hibernate + * again. Only the PHY's own failures stop it for the later sleeps; a sleep refused for a + * USB change doesn't. + */ +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return; + scoped_guard(mutex, &atcphy->lock) { + if (!kept && atcphy->sleep_ret && atcphy->sleep_ret != -EBUSY) + atcphy->sleep_no_keep = true; + atcphy->sleep_in_place = false; + atcphy->sleep_ret = 0; + } + atcphy_sleep_ss_up(atcphy); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish); + static int atcphy_suspend(struct device *dev) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); @@ -6519,7 +6943,10 @@ static void atcphy_shutdown(struct platform_device *pdev) dev_err(atcphy->dev, "USB3 common shutdown failed: %d\n", ret); } -static DEFINE_SIMPLE_DEV_PM_OPS(atcphy_pm_ops, atcphy_suspend, atcphy_resume); +static const struct dev_pm_ops atcphy_pm_ops = { + SYSTEM_SLEEP_PM_OPS(atcphy_suspend, atcphy_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(atcphy_suspend_noirq, atcphy_resume_noirq) +}; static const struct atcphy_hw atcphy_hw_t8103 = { .gen = ATCPHY_GENERATION_T8103, @@ -6566,6 +6993,8 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .has_usb2phy_reg = true, .has_t8140_dp = true, .optional_tunables = true, + .park_pipe_unlocked = true, + .restore_after_pd_off = true, }; static const struct atcphy_hw atcphy_hw_t8142 = { diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index d3c482fa4234c..82b901b6a8374 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -26,6 +27,7 @@ #include #include +#include #include "pinctrl-utils.h" #include "core.h" @@ -49,6 +51,11 @@ struct apple_gpio_pinctrl { struct pinctrl_desc pinctrl_desc; struct gpio_chip gpio_chip; u8 irq_group_base; + /* The pins' words across a firmware-driven suspend */ + u32 *saved; + unsigned long *saved_valid; + /* Pins this driver has written. Others belong to someone else. */ + unsigned long *touched; struct apple_gpio_irq_group irqgrps[]; }; @@ -91,6 +98,8 @@ static const struct regmap_config regmap_config = { static void apple_gpio_set_reg(struct apple_gpio_pinctrl *pctl, unsigned int pin, u32 mask, u32 value) { + if (pctl->touched) + set_bit(pin, pctl->touched); regmap_update_bits(pctl->map, REG_GPIO(pin), mask, value); } @@ -589,6 +598,15 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); + /* Only with the regmap reaching the hardware: a cache would hide the loss. */ + if (hw && hw->no_regcache) { + pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); + pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->saved || !pctl->saved_valid || !pctl->touched) + return -ENOMEM; + } + for (i = 0; i < npins; i++) { pins[i].number = i; pins[i].name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "PIN%u", i); @@ -640,10 +658,63 @@ static const struct of_device_id apple_gpio_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_gpio_pinctrl_of_match); +/* + * A firmware-driven suspend powers this block off, and neither the pin core nor + * a consumer sets a pin up again. Slots that fault aren't readable in the regmap, + * so they stay out of this. + */ +static int apple_gpio_pinctrl_suspend_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i; + + if (!pctl->saved || pm_suspend_target_state != PM_SUSPEND_MEM || + !pm_suspend_via_firmware()) + return 0; + + /* + * Only the pins this driver configured. The others belong to the firmware or another + * processor, and writing one back after the wake raises an SError. + */ + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) + if (!regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) + set_bit(i, pctl->saved_valid); + + return 0; +} + +static int apple_gpio_pinctrl_resume_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i, val, restored = 0; + + if (!pctl->saved || !pm_resume_via_firmware()) + return 0; + + for_each_set_bit(i, pctl->saved_valid, pctl->pinctrl_desc.npins) { + if (!regmap_read(pctl->map, REG_GPIO(i), &val) && val == pctl->saved[i]) + continue; + regmap_write(pctl->map, REG_GPIO(i), pctl->saved[i]); + restored++; + } + dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + + return 0; +} + +static const struct dev_pm_ops apple_gpio_pinctrl_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_gpio_pinctrl_suspend_noirq, + apple_gpio_pinctrl_resume_noirq) +}; + static struct platform_driver apple_gpio_pinctrl_driver = { .driver = { .name = "apple-gpio-pinctrl", .of_match_table = apple_gpio_pinctrl_of_match, + .pm = pm_sleep_ptr(&apple_gpio_pinctrl_pm_ops), .suppress_bind_attrs = true, }, .probe = apple_gpio_pinctrl_probe, diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 39e0972acd0e8..586e3f675db9e 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -1529,6 +1530,40 @@ static const struct apple_pmc_hw apple_pmc_hw_t6050 = { .has_pending = true, }; +/* + * A system sleep that reached the hardware may have taken the enables and floors + * with it. Put back what the driver's books say: these are no-ops if they stayed. + */ +static int apple_pmc_resume_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + unsigned long flags; + unsigned int rail; + int ret = 0, err; + + if (!pm_resume_via_firmware() || !pmc) + return 0; + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails; rail++) { + err = apple_pmc_enable_write(pmc, rail); + ret = ret ?: err; + if (pmc->floor_rails & BIT(rail)) { + err = apple_pmc_floor_apply(pmc, rail); + ret = ret ?: err; + } + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + if (ret) + dev_err(dev, "state not restored after sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_pmc_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmc_resume_early) +}; + static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t6050-pmc", .data = &apple_pmc_hw_t6050 }, { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, @@ -1545,6 +1580,7 @@ static struct platform_driver apple_pmc_driver = { .driver = { .name = "apple-pmc", .of_match_table = apple_pmc_of_match, + .pm = pm_sleep_ptr(&apple_pmc_pm_ops), /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index d6b0cc28b6c2c..d1e9f14840720 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,8 @@ struct apple_pmgr_ps { bool firmware_owned; /* apple,sleep-retained: kept on while the system sleeps */ bool sleep_retained; + /* apple,sleep-driver-owned: the driver behind it brings it back after a sleep */ + bool sleep_driver_owned; /* Never enable automatic power management (apple,no-auto-pm). */ bool no_auto_pm; bool inherited_on; @@ -444,6 +447,50 @@ static int apple_pmgr_ps_resume_early(struct device *dev) return 0; } +/* + * A firmware-driven suspend powers the SoC down and its sleep engine leaves the + * domains in states of its own: AUTO_ENABLE cleared, some at clock gating. genpd + * only powers a domain back on if it turned it off itself, so its idea of a + * domain it left on is now wrong. Put each domain back where genpd thinks it is. + * + * On T8140 macOS re-raises 22 domains in the first 5 ms after a wake and lowers + * 30 others (measured under aura). Which of them Linux needs isn't established, + * so this follows genpd's books rather than a list. + */ +static int apple_pmgr_ps_resume_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + u32 reg; + int ret; + + if (!pm_resume_via_firmware() || ps->firmware_owned || ps->inherited_on || + ps->sleep_driver_owned) + return 0; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ret = regmap_read(ps->regmap, ps->offset, ®); + if (!ret) { + if (ps->genpd.status == GENPD_STATE_OFF) { + if (FIELD_GET(APPLE_PMGR_PS_TARGET, reg) != APPLE_PMGR_PS_PWRGATE) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_PWRGATE, + false); + } else { + bool auto_enable = !ps->no_auto_enable && !ps->no_auto_pm; + + if (FIELD_GET(APPLE_PMGR_PS_TARGET, reg) != APPLE_PMGR_PS_ACTIVE || + (auto_enable && !(reg & APPLE_PMGR_AUTO_ENABLE))) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + auto_enable); + } + } + spin_unlock_irqrestore(&ps->genpd.slock, flags); + if (ret) + dev_err(dev, "PS %s: resync after sleep failed: %d\n", ps->genpd.name, ret); + + return 0; +} + static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); @@ -694,6 +741,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) "retention workaround needs a consumer switch\n"); } ps->sleep_retained = of_property_read_bool(node, "apple,sleep-retained"); + ps->sleep_driver_owned = of_property_read_bool(node, "apple,sleep-driver-owned"); /* * Only T8132 hands over inherited-on domains with this contract. Older * M3 device trees carry the property with another meaning; keep them @@ -1087,6 +1135,7 @@ MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmgr_ps_resume_noirq) }; static struct platform_driver apple_pmgr_ps_driver = { diff --git a/drivers/pmdomain/apple/pmp-report-test.c b/drivers/pmdomain/apple/pmp-report-test.c index 204e19fbf28ce..179b9e752e3f3 100644 --- a/drivers/pmdomain/apple/pmp-report-test.c +++ b/drivers/pmdomain/apple/pmp-report-test.c @@ -12,6 +12,16 @@ static void pmp_range(u8 *table, u32 id, u32 base, u32 count) put_unaligned_le32(count, table + 8); } +static void pmp_request_word_replay(struct kunit *test) +{ + /* A request an entry withdrew stays off; one nobody here owns stays on. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x31, 0x03, 0x02), 0x32ULL); + /* The found word can be empty: only the requests come back. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0, 0x03, 0x01), 0x01ULL); + /* Nothing requested, nothing known. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x10, 0, 0), 0x10ULL); +} + static void pmp_ranges_t6030(struct kunit *test) { u8 table[2 * PMP_PTD_RANGE_SIZE]; @@ -277,6 +287,7 @@ static void pmp_dvfs_words(struct kunit *test) static struct kunit_case apple_pmp_report_cases[] = { KUNIT_CASE(pmp_ranges_t6030), + KUNIT_CASE(pmp_request_word_replay), KUNIT_CASE(pmp_ranges_t8122), KUNIT_CASE(pmp_ranges_t6031), KUNIT_CASE(pmp_ranges_reject_bad_records), diff --git a/drivers/pmdomain/apple/pmp-report-validation.h b/drivers/pmdomain/apple/pmp-report-validation.h index c3e5ad495650b..3165ba33ded3c 100644 --- a/drivers/pmdomain/apple/pmp-report-validation.h +++ b/drivers/pmdomain/apple/pmp-report-validation.h @@ -15,6 +15,15 @@ #define PMP_PTD_RANGE_REQUEST 10 #define PMP_PTD_RANGE_ACK 11 +/* + * The request word for a PMP that is starting: requests made through entries, + * and every other bit as it was found. + */ +static inline u64 apple_pmp_request_word(u64 found, u64 known, u64 shadow) +{ + return (found & ~known) | shadow; +} + /* * A SOC-DEV-DVFS request word: the valid bit, and the level either in the * default five-bit lane at bits 4:0 or, for request classes 0x21 through diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 11178eef8fe7c..fd1c0c0ab0f3c 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -103,8 +103,17 @@ struct apple_pmp_report { /* Requests made through entries: bits requested on, all bits. */ u64 shadow; u64 known; - /* Requests are held in memory until the PMP runs (@defer_requests). */ + /* Bits whose drivers withdraw them at the sleep's commit, with the PMP still running. */ + u64 commit_owned; + /* + * Requests are held in memory until the PMP runs (@defer_requests), and + * while it's stopped for a system sleep. + */ bool defer; + /* The request word when a system sleep stopped the PMP. */ + u64 sleep_request; + /* The last SOC-DEV-DVFS word written to each entry, 0 if none. */ + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; }; @@ -430,7 +439,7 @@ static void apple_pmp_report_ready_work(struct work_struct *work) spin_lock_irqsave(&rep->lock, flags); val = readq(rep->base + offs->tgt_read); - val = (val & ~rep->known) | rep->shadow; + val = apple_pmp_request_word(val, rep->known, rep->shadow); writeq(val, rep->base + offs->tgt_write); want = rep->shadow; rep->defer = false; @@ -444,6 +453,97 @@ static void apple_pmp_report_ready_work(struct work_struct *work) readq(rep->base + offs->status), want); } +/* A report names its PMP where a SoC has more than one kind of link to it. */ +static bool apple_pmp_report_of(const struct apple_pmp_report *rep, const struct device *pmp) +{ + struct device_node *np = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + bool match = !np || np == pmp->of_node; + + of_node_put(np); + return match; +} + +/** + * apple_pmp_report_sleep() - withdraw and hold requests before a system sleep stops the PMP + * @pmp: the PMP device, still running + * @at_commit: the PMP stops at the sleep's commit, so requests whose drivers withdraw them + * there are left set + * + * The devices behind these requests have stopped by now, so the PMP is told before + * it goes. Requests made from here on only update the books, so nothing waits on an + * acknowledgment from a stopped PMP. Bits no entry owns are left alone: nothing here + * knows whether their devices have stopped. + */ +void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 val, withdrawn; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + rep->sleep_request = readq(rep->base + offs->tgt_read); + withdrawn = rep->sleep_request & rep->known & ~(at_commit ? rep->commit_owned : 0); + if (withdrawn) + writeq(rep->sleep_request & ~withdrawn, rep->base + offs->tgt_write); + rep->defer = true; + spin_unlock_irqrestore(&rep->lock, flags); + + if (withdrawn && readq_poll_timeout(rep->base + offs->actual, val, + !(val & withdrawn), 100, 50000)) + dev_warn(rep->dev, "PMP still acknowledges withdrawn requests %#llx (ack %#llx)\n", + withdrawn, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_sleep); + +/** + * apple_pmp_report_wake() - hand the PMP every request after a system sleep + * @pmp: the PMP device, running again + * @lost_power: the sleep reached the hardware, so the report may have lost its + * contents: bits no entry owns come from the word kept at the sleep + * + * The power requests go first and the performance requests after them, the + * order an entry uses when it powers on. + */ +void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ + struct apple_pmp_report *rep; + unsigned long flags; + unsigned int i; + u64 val, want; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + val = lost_power ? rep->sleep_request : readq(rep->base + offs->tgt_read); + writeq(apple_pmp_request_word(val, rep->known, rep->shadow), + rep->base + offs->tgt_write); + for (i = 0; rep->dvfs_shadow && i < offs->dvfs_count; i++) + if (rep->dvfs_shadow[i]) + writeq(rep->dvfs_shadow[i], + rep->base + offs->update + 8 * (offs->dvfs_first + i)); + want = rep->shadow; + rep->defer = false; + spin_unlock_irqrestore(&rep->lock, flags); + + if (want && readq_poll_timeout(rep->base + offs->actual, val, + (val & want) == want, 1000, 1000000)) + dev_warn(rep->dev, "PMP did not acknowledge requests %#llx (ack %#llx)\n", + want, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_wake); + /* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) { @@ -1363,6 +1463,12 @@ static int apple_pmp_report_probe(struct platform_device *pdev) } rep->defer = rep->offsets->defer_requests; + if (rep->offsets->dvfs_count) { + rep->dvfs_shadow = devm_kcalloc(dev, rep->offsets->dvfs_count, + sizeof(*rep->dvfs_shadow), GFP_KERNEL); + if (!rep->dvfs_shadow) + return -ENOMEM; + } INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); @@ -1898,7 +2004,13 @@ EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); */ static void apple_pmp_report_dvfs_write(struct apple_pmp_report *rep, u32 entry, u64 val) { - writeq(val, rep->base + rep->offsets->update + 8 * entry); + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + rep->dvfs_shadow[entry - rep->offsets->dvfs_first] = val; + if (!rep->defer) + writeq(val, rep->base + rep->offsets->update + 8 * entry); + spin_unlock_irqrestore(&rep->lock, flags); } /** @@ -1973,6 +2085,136 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo return ret; } +/** + * apple_pmp_report_entry_request() - set or withdraw one entry's request + * @entry: the report entry's node + * @enable: make the request, or withdraw it + * + * For a driver that switches its device's power itself around a system sleep. The + * PMP may use a device whose request is set, so that driver withdraws the request + * before its power goes and makes it again after its power is back. A withdrawn + * request isn't replayed after the PMP restarts. + * + * Return: 0, also with no report bound or the PMP not running, or the error from + * waiting for its acknowledgment. + */ +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable) +{ + struct device_node *report; + struct apple_pmp_report *rep; + u32 id; + int ret = 0; + + if (of_property_read_u32(entry, "reg", &id)) + return -EINVAL; + + report = of_get_parent(entry); + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != report) + continue; + ret = apple_pmp_report_set_state(rep, id, enable, false); + break; + } + rcu_read_unlock(); + of_node_put(report); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_request); + +static struct apple_pmp_report *apple_pmp_report_of_entry(const struct device_node *entry, + u32 *id) +{ + struct device_node *report; + struct apple_pmp_report *rep, *found = NULL; + + if (of_property_read_u32(entry, "reg", id) || *id >= 64) + return NULL; + report = of_get_parent(entry); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == report) { + found = rep; + break; + } + } + of_node_put(report); + + return found; +} + +/** + * apple_pmp_report_entry_commit_owned() - mark an entry withdrawn by its driver at the commit + * @entry: the report entry's node + * @owned: its driver withdraws it at the sleep's commit, or no longer does + * + * For a device whose stop moves to the sleep's commit: a PMP that stops there too keeps the + * request through apple_pmp_report_sleep(), until apple_pmp_report_entry_withdraw_polled(). + * + * Return: 0, or -ENODEV with no report bound for the entry. + */ +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned) +{ + struct apple_pmp_report *rep; + unsigned long flags; + int ret = -ENODEV; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + spin_lock_irqsave(&rep->lock, flags); + if (owned) + rep->commit_owned |= BIT_ULL(id); + else + rep->commit_owned &= ~BIT_ULL(id); + spin_unlock_irqrestore(&rep->lock, flags); + ret = 0; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_commit_owned); + +/** + * apple_pmp_report_entry_withdraw_polled() - withdraw an entry's request at the sleep's commit + * @entry: the report entry's node + * + * Interrupts off, one CPU: the request goes even with requests held for the sleep, and the + * PMP's acknowledgment is polled for while it runs. One already withdrawn returns at once. + * + * Return: 0, also with no report bound, or -ETIMEDOUT with no acknowledgment. + */ +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 bit, val; + int ret = 0; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + bit = BIT_ULL(id); + spin_lock_irqsave(&rep->lock, flags); + rep->known |= bit; + rep->shadow &= ~bit; + val = readq(rep->base + rep->offsets->tgt_read); + if (val & bit) + writeq(val & ~bit, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + if (apple_pmp_report_running(rep)) + ret = readq_poll_timeout_atomic(rep->base + rep->offsets->actual, val, + !(val & bit), 10, 50000); + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_withdraw_polled); + /* * System sleep. An external display processor's request keeps the PMP's * external display state up while nothing is shown, about 0.25 W in diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 1e306c7bb3f6c..da81dc08e5717 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -1239,6 +1239,24 @@ static void macsmc_power_remove(struct platform_device *pdev) blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); } +/* + * The charge moved while asleep, and an event's uevent that ran before the SMC + * was back found nothing to report. + */ +static void macsmc_power_complete(struct device *dev) +{ + struct macsmc_power *power = dev_get_drvdata(dev); + + if (power->batt) + power_supply_changed(power->batt); + if (power->ac) + power_supply_changed(power->ac); +} + +static const struct dev_pm_ops macsmc_power_pm_ops = { + .complete = pm_sleep_ptr(macsmc_power_complete), +}; + static const struct platform_device_id macsmc_power_id[] = { { .name = "macsmc-power" }, { /* sentinel */ } @@ -1248,6 +1266,7 @@ MODULE_DEVICE_TABLE(platform, macsmc_power_id); static struct platform_driver macsmc_power_driver = { .driver = { .name = "macsmc-power", + .pm = pm_sleep_ptr(&macsmc_power_pm_ops), }, .id_table = macsmc_power_id, .probe = macsmc_power_probe, diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 6043d7ee87a84..709c85948fe67 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -386,7 +386,14 @@ static int apple_pmu_rtc_prepare(struct device *dev) { struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || rtc->io_failed) + if (rtc->io_failed) + return -EBUSY; + /* + * A firmware-driven suspend programs the wake sources itself, and the + * alarm stays armed in the PMU, which stays powered. Only a sleep state + * without either has no way to bring the alarm back. + */ + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !pm_suspend_via_firmware()) return -EBUSY; return 0; } diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 9001a709fb6e7..d144b1adb690e 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -95,6 +95,27 @@ config APPLE_PMGR_DCS_KUNIT_TEST Also verify that boot constraints and concurrent endpoints retain their votes, and that suspend restores the last confirmed request. +config APPLE_CLUSTER_PWRGATE + bool "Apple SoC CPU cluster power gates" + depends on OF + default ARCH_APPLE + help + Sets the power-gate enable of each CPU cluster the device tree + lists, early in boot. Without it, a core that the boot loader + started and PMGR never powered on, such as the boot CPU, only + clock-gates in WFI. + +config APPLE_T8140_SLEEP + bool "Apple T8140 system sleep at EL2" + depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) + depends on HOTPLUG_CPU && SUSPEND + select APPLE_TUNABLE + help + Powers the secondary CPUs off and suspends the system to RAM on an + Apple T8140 whose kernel runs directly at EL2, with no monitor to ask. + It does nothing unless the boot object asks for it through + /chosen/aurora,s2r-mailbox. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX @@ -214,8 +235,9 @@ config APPLE_SEP_KUNIT_TEST mailbox message encoding and decoding, the shared-memory registration limits, the rules that keep a keybag's lookup identity apart from the identity the enclave reports, the format and validation of the - Linux host-state store, and the xART guard's decisions and its list - of Linux's records. The tests touch no hardware. + Linux host-state store, the xART guard's decisions and its list + of Linux's records, and how the fingerprint sensor is put to sleep. + The tests touch no hardware. If unsure, say N. diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 34ede633d2165..b6d60b1705af2 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -13,6 +13,10 @@ apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o +obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o + +obj-$(CONFIG_APPLE_T8140_SLEEP) += t8140-sleep.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 7f446caf407a8..e1520bcdf66ba 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -63,6 +63,7 @@ use kernel::{ atomic::{ Acquire, Atomic, + Relaxed, Release, // }, Arc, @@ -108,12 +109,16 @@ const AFK_OPC_INIT_UNK: u64 = 0x8c; const AFK_OPC_SEND: u64 = 0xa2; const AFK_OPC_START_ACK: u64 = 0x86; const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_SLEEP_ACK: u64 = 0x87; const AFK_OPC_RECV: u64 = 0x85; const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; const AFK_MSG_START: u64 = 0xa3 << 48; const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +/// Stops an endpoint for a system sleep. It keeps its rings, and `AFK_MSG_START` +/// brings it back. +const AFK_MSG_SLEEP: u64 = 0xa4 << 48; const AFK_RB_BLOCK_STEP: usize = 0x40; const EPIC_TYPE_NOTIFY: u32 = 0; const EPIC_CATEGORY_REPORT: u8 = 0x00; @@ -141,6 +146,8 @@ const EPIC_CALL_TIMEOUT_MS: u32 = 5000; /// Bound on the wait for the shutdown acknowledgments of all endpoints /// together, so that a firmware that stopped answering costs one timeout. const AFK_SHUTDOWN_TIMEOUT_MS: u32 = 5000; +/// Bound on the answer to each stop or start around a system sleep. +const AFK_SLEEP_TIMEOUT_MS: u32 = 1000; // The T8140 AOP boots through a second mailbox, its "setup port", besides the // RTKit one. Its management endpoint runs a HELLO / endpoint map / power @@ -207,6 +214,32 @@ mod aop_ring; /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one /// of the endpoints this driver drives. +/// Iterates over `list`, or over every AFK endpoint if `list` is empty. +struct AfkOrder { + list: &'static [u8], + next: usize, +} + +impl AfkOrder { + fn new(list: &'static [u8]) -> AfkOrder { + AfkOrder { list, next: 0 } + } +} + +impl Iterator for AfkOrder { + type Item = u8; + + fn next(&mut self) -> Option { + let at = self.next; + self.next += 1; + if self.list.is_empty() { + (at < AFK_ENDPOINT_COUNT as usize).then(|| AFK_ENDPOINT_START + at as u8) + } else { + self.list.get(at).copied() + } + } +} + fn afk_endpoint_index(ep: u8) -> Option { ep.checked_sub(AFK_ENDPOINT_START) .filter(|i| *i < AFK_ENDPOINT_COUNT) @@ -470,7 +503,7 @@ impl AFKEndpoint { self.recv_get_buf(client.dev.clone(), rtkit, msg)?; } AFK_OPC_INIT_UNK => {} // no-op - AFK_OPC_START_ACK => {} + AFK_OPC_START_ACK | AFK_OPC_SLEEP_ACK => client.sleep_answered(self.index), AFK_OPC_INIT_RX => { if self.rxbuf.is_some() { dev_err!( @@ -1258,6 +1291,21 @@ struct AopData { call_turn: [Mutex<()>; AFK_ENDPOINT_COUNT as usize], #[pin] ep_shutdown: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// Answers to the stop and start of each endpoint around a system sleep. + #[pin] + ep_sleep: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// The order endpoints stop for a system sleep and start after it. Empty + /// means all started ones in endpoint order. + sleep_endpoints: &'static [u8], + wake_endpoints: &'static [u8], + /// No call starts until the system has resumed. + sleeping: Atomic, + /// RTKit reported the firmware crashed. Nothing restarts it short of a reboot. + crashed: Atomic, + /// The endpoints stopped for this sleep, by index. + sleep_stopped: Atomic, + /// The firmware was told the AP is going to sleep. + ap_idle: Atomic, #[pin] hid_listeners: Mutex>, #[pin] @@ -1392,6 +1440,13 @@ impl AopData { }), call_turn <- pin_init::pin_init_array_from_fn(|_| new_mutex!(())), ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + ep_sleep <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + sleep_endpoints: cfg.sleep_endpoints, + wake_endpoints: cfg.wake_endpoints, + sleeping: Atomic::new(false), + crashed: Atomic::new(false), + sleep_stopped: Atomic::new(0), + ap_idle: Atomic::new(false), hid_listeners <- new_mutex!(KVec::new()), report_listeners <- new_mutex!(KVec::new()), source_ring <- new_mutex!(None), @@ -1601,6 +1656,127 @@ impl AopData { ret } + fn sleep_answered(&self, endpoint: u8) { + if let Some(index) = afk_endpoint_index(endpoint) { + self.ep_sleep[index].complete(()); + } + } + + /// Sends `msg` to one endpoint and waits for its answer. + fn sleep_switch(&self, endpoint: u8, msg: u64) -> Result<()> { + let index = afk_endpoint_index(endpoint).ok_or(EINVAL)?; + self.ep_sleep[index].reset(); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + rtk.send_message(endpoint, msg)?; + } + self.ep_sleep[index] + .wait_timeout(AFK_SLEEP_TIMEOUT_MS) + .ok_or(ETIMEDOUT) + } + + /// Tells the firmware about the AP's power state. Waits with the RTKit lock + /// dropped, as the answer shares a worker with messages that take it. + fn ap_power(&self, on: bool) -> Result<()> { + let rtk = (*self.rtkit.lock()) + .as_ref() + .map(|rtk| rtk.as_raw()) + .ok_or(ENODEV)?; + // SAFETY: the handle lives until removal, which holds the device lock + // that system sleep also runs under. + to_result(unsafe { bindings::apple_rtkit_ap_power(rtk, on) }) + } + + /// A crashed firmware hears nothing more, so it's left out of every sleep from then on, + /// rather than refusing them all. + fn crashed_skip(&self) -> bool { + self.crashed.load(Acquire) + } + + /// Puts back what `sleep_enter` did, in the order the firmware expects. + fn sleep_undo(&self) { + if self.crashed_skip() { + self.sleep_stopped.store(0, Relaxed); + self.ap_idle.store(false, Relaxed); + self.sleeping.store(false, Release); + return; + } + let stopped = self.sleep_stopped.xchg(0, Relaxed); + let was_idle = self.ap_idle.load(Relaxed); + if self.ap_idle.xchg(false, Relaxed) { + if let Err(e) = self.ap_power(true) { + dev_err!(self.dev, "Wake notice not answered ({:?})", e); + } + } + let order = AfkOrder::new(self.wake_endpoints); + for endpoint in order { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if stopped & (1 << index) == 0 { + continue; + } + if let Err(e) = self.sleep_switch(endpoint, AFK_MSG_START) { + dev_err!(self.dev, "Endpoint {:#04x} did not restart ({:?})", endpoint, e); + } + } + self.sleeping.store(false, Release); + if was_idle { + dev_dbg!(self.dev, "AP on notice sent and endpoints restarted after a sleep\n"); + } + } + + /// The AOP keeps running through a firmware-driven suspend. It stops its + /// endpoints and hears the AP's power state, then the reverse after a wake, + /// with no firmware restart. Measured on T8140, unverified on other SoCs. + fn sleep_enter(&self) -> Result<()> { + if !self.cpu_started.load(Acquire) { + return Ok(()); + } + if self.crashed_skip() { + dev_warn!(self.dev, "firmware crashed: sleeping without it\n"); + self.sleeping.store(true, Release); + return Ok(()); + } + self.sleeping.store(true, Release); + // Any call in progress finishes first; new ones see `sleeping`. + for turn in &self.call_turn { + drop(turn.lock()); + } + + let mut result = Ok(()); + for endpoint in AfkOrder::new(self.sleep_endpoints) { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if !self.endpoints[index].lock().started { + continue; + } + result = self.sleep_switch(endpoint, AFK_MSG_SLEEP).inspect_err(|e| { + dev_err!(self.dev, "Endpoint {:#04x} did not stop ({:?})", endpoint, e); + }); + if result.is_err() { + break; + } + self.sleep_stopped + .store(self.sleep_stopped.load(Relaxed) | (1 << index), Relaxed); + } + if result.is_ok() { + // Set first: a notice without an answer may still have been taken. + self.ap_idle.store(true, Relaxed); + result = self.ap_power(false).inspect_err(|e| { + dev_err!(self.dev, "Sleep notice not answered ({:?})", e); + }); + } + if result.is_err() { + self.sleep_undo(); + } else { + dev_dbg!(self.dev, "Endpoints stopped and AP idle notice answered for a sleep\n"); + } + result + } + fn patch_bootargs( &self, aop_mmio: &Mmio<'_, Region>, @@ -1915,8 +2091,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -1935,8 +2118,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -2554,6 +2744,7 @@ impl rtkit::Operations for AopData { } fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + data.crashed.store(true, Release); dev_err!(data.dev, "AOP firmware crashed"); } } @@ -2576,6 +2767,10 @@ struct AopHwConfig { epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. setup_port: bool, + /// The order endpoints stop for a system sleep; empty means endpoint order. + sleep_endpoints: &'static [u8], + /// The order they start after one. + wake_endpoints: &'static [u8], /// Shut the co-processor down on unbind and retain its buffers if the /// shutdown cannot be confirmed. quiesce_on_unbind: bool, @@ -2598,6 +2793,8 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2614,6 +2811,8 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2629,6 +2828,8 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2643,6 +2844,8 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2662,6 +2865,8 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: true, required_endpoints: &[], fw_regions: true, @@ -2685,6 +2890,8 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: true, + sleep_endpoints: &[0x2b, 0x23, 0x22, 0x20, 0x21], + wake_endpoints: &[0x20, 0x21, 0x22, 0x2b, 0x23], quiesce_on_unbind: true, required_endpoints: &[0x20, 0x21, 0x22, 0x23, 0x2b], fw_regions: false, @@ -2692,6 +2899,50 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { halt_on_failed_start: false, }; +/// The data of a bound AOP device. +/// +/// # Safety +/// +/// `dev` is an AOP device between the end of probe and the start of removal. +unsafe fn aop_from_dev<'a>(dev: *mut bindings::device) -> &'a AopData { + // SAFETY: by the caller's contract, drvdata is the `AopDriver` probe set. + let driver = unsafe { &*bindings::dev_get_drvdata(dev).cast::() }; + let data: *const dyn AOP = &*driver.0; + // SAFETY: probe made this `Arc` from an `Arc`. + unsafe { &*data.cast::() } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_suspend_late(dev: *mut bindings::device) -> c_int { + // SAFETY: only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: by this function's contract. + match unsafe { aop_from_dev(dev) }.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_resume(dev: *mut bindings::device) -> c_int { + // SAFETY: by this function's contract. + unsafe { aop_from_dev(dev) }.sleep_undo(); + 0 +} + +static AOP_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend_late: Some(aop_suspend_late), + resume: Some(aop_resume), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + kernel::of_device_table!( OF_TABLE, ::IdInfo, @@ -2712,6 +2963,10 @@ impl platform::Driver for AopDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/apple-cluster-pwrgate.c b/drivers/soc/apple/apple-cluster-pwrgate.c new file mode 100644 index 0000000000000..67d493cff8af6 --- /dev/null +++ b/drivers/soc/apple/apple-cluster-pwrgate.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC CPU cluster power gates + * + * Each CPU cluster has a PMGR word whose top bit lets its cores power down + * in WFI. A core PMGR powers on comes up allowed to, but the boot CPU was + * started by the boot ROM, so it only follows the word, which the boot + * loader may leave clear. Set it for every cluster the device tree lists. + */ + +#include +#include +#include +#include +#include +#include + +#define CLUSTER_PWRGATE_EN BIT(31) + +static const struct of_device_id apple_cluster_pwrgate_of_match[] __initconst = { + { .compatible = "apple,cluster-pwrgate" }, + { } +}; + +/* Before the secondaries start. A core picks the word up at its next WFI. */ +static int __init apple_cluster_pwrgate_init(void) +{ + struct device_node *np; + + for_each_matching_node(np, apple_cluster_pwrgate_of_match) { + void __iomem *gate; + int i; + + if (!of_device_is_available(np)) + continue; + + /* One word per cluster. Shared with the sleep code, so not claimed. */ + for (i = 0; (gate = of_iomap(np, i)); i++) { + u32 was = readl(gate); + + writel(was | CLUSTER_PWRGATE_EN, gate); + pr_debug("%pOF: cluster %d power gate %#x, now %#x\n", np, i, was, + readl(gate)); + iounmap(gate); + } + } + + return 0; +} +early_initcall(apple_cluster_pwrgate_init); diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 946c95d7a46d3..644354f5ddb06 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -164,6 +164,7 @@ pub(crate) struct ControlState { retired: [u64; 4], retired_count: u32, entropy: EntropySink, + wake_seen: u8, } impl ControlState { @@ -181,6 +182,7 @@ impl ControlState { retry_at: None, retry_ms: ENTROPY_RETRY_MIN_MS, }, + wake_seen: 0, } } @@ -218,7 +220,7 @@ impl ControlState { } else { tag + 1 }; - if !self.tag_in_flight(tag) && !self.is_retired(tag) { + if tag != proto::TAG_SLEEP && !self.tag_in_flight(tag) && !self.is_retired(tag) { self.slots[idx] = Slot { used: true, tag, @@ -230,6 +232,45 @@ impl ControlState { Err(EBUSY) } + /// Claims the sleep request's own tag, which the pool never hands out. A + /// request still in flight under it means the last one was never answered. + pub(crate) fn alloc_sleep(&mut self) -> Result { + let tag = proto::TAG_SLEEP; + if self.tag_in_flight(tag) { + return Err(EBUSY); + } + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + self.reclaim_retired(tag); + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + Ok(idx) + } + + /// Whether any request is waiting for its reply. + pub(crate) fn requests_in_flight(&self) -> bool { + self.slots.iter().any(|s| s.used) + } + + /// Forget the wake messages seen, before asking for them. + pub(crate) fn wake_reset(&mut self) { + self.wake_seen = 0; + } + + /// A control message that isn't a reply: note it if it's a wake message. + pub(crate) fn wake_note(&mut self, ty: u8) { + if let Some(n) = proto::WAKE_TYPES.iter().position(|&t| t == ty) { + self.wake_seen |= 1 << n; + } + } + + /// All the wake messages have come. + pub(crate) fn wake_complete(&self) -> bool { + self.wake_seen == (1 << proto::WAKE_TYPES.len()) - 1 + } + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { self.slots[idx].reply.take() } @@ -419,6 +460,55 @@ mod tests { } } + #[test] + fn the_pool_never_hands_out_the_sleep_tag() { + let mut state = ControlState::new(); + for _ in 0..POOL_LEN * 2 { + let (idx, tag) = state.alloc().unwrap(); + assert_ne!(tag, proto::TAG_SLEEP); + state.release(idx); + } + } + + #[test] + fn a_sleep_request_gets_its_own_answer() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + assert!(state.requests_in_flight()); + assert_eq!(state.alloc_sleep(), Err(EBUSY)); + let reply = proto::ControlReply { + tag: proto::TAG_SLEEP, + data_lo: 0, + }; + assert!(matches!(state.deliver(reply), Delivery::Matched)); + assert!(state.take_reply(idx).is_some()); + state.release(idx); + assert!(!state.requests_in_flight()); + } + + #[test] + fn a_sleep_request_can_follow_one_that_timed_out() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + state.abandon(idx); + assert!(state.alloc_sleep().is_ok()); + } + + #[test] + fn a_wake_takes_both_messages() { + let mut state = ControlState::new(); + state.wake_reset(); + assert!(!state.wake_complete()); + state.wake_note(0x33); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[0]); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[1]); + assert!(state.wake_complete()); + state.wake_reset(); + assert!(!state.wake_complete()); + } + #[test] fn a_request_is_answered() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f4f674f548c5d..750281bb5c98a 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -46,6 +46,7 @@ #define APPLE_ASC_MBOX_A2I_RECV1 0x818 #define APPLE_ASC_MBOX_I2A_CONTROL 0x114 +#define APPLE_ASC_MBOX_CONTROL_RECV_ENABLE BIT(0) #define APPLE_ASC_MBOX_I2A_SEND0 0x820 #define APPLE_ASC_MBOX_I2A_SEND1 0x828 #define APPLE_ASC_MBOX_I2A_RECV0 0x830 @@ -228,6 +229,8 @@ struct apple_mbox_hw { /* Status word counts RX occupancy and free TX slots; paired transfers. */ bool counted_fifo; bool ap_initializes_mailboxes; + /* No interrupts: the consumer calls apple_mbox_poll() while it expects traffic. */ + bool polled; unsigned int control_full; unsigned int control_empty; @@ -340,14 +343,11 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); -static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +static void apple_mbox_ascwrap_v6_run_off(struct apple_mbox *mbox) { void __iomem *cpu_control; u32 val; - if (!mbox->ascwrap_v6_cpu_running) - return; - if (mbox->hw->is_ascwrap_v8) { u32 state, ack; @@ -377,12 +377,24 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) val = readl(cpu_control); writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); } +} +static void apple_mbox_ascwrap_v6_stopped_locked(struct apple_mbox *mbox) +{ mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + if (!mbox->ascwrap_v6_cpu_running) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); + apple_mbox_ascwrap_v6_stopped_locked(mbox); +} + /* * A failed resume leaves the runtime PM error set, and runtime PM then fails * every later resume with -EINVAL without retrying the power domain. The @@ -486,6 +498,44 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return NULL; + + return (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_control); + +/* No lock and no runtime PM: the registers alone, as apple_mbox_ascwrap_v6_stop_cpu() has them. */ +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs || + !READ_ONCE(mbox->ascwrap_v6_cpu_running)) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu_atomic); + +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) + ret = -ENODEV; + else if (mbox->ascwrap_v6_cpu_running) + apple_mbox_ascwrap_v6_stopped_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_stopped); + static bool apple_mbox_is_g17(const struct apple_mbox *mbox) { return mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX || @@ -704,7 +754,8 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, * it unmasked for them. The handler masks it when it fires, * and removal masks it otherwise. */ - t = wait_for_completion_interruptible_timeout( + /* Not for the freezer's signal: a message half sent can break a handshake. */ + t = wait_for_completion_killable_timeout( &mbox->tx_empty, min(remaining, msecs_to_jiffies(APPLE_MBOX_TX_RECHECK_MS))); if (t < 0) @@ -891,6 +942,12 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) return IRQ_HANDLED; } +bool apple_mbox_send_ready(struct apple_mbox *mbox) +{ + return !(readl_relaxed(mbox->regs + mbox->hw->a2i_control) & mbox->hw->control_full); +} +EXPORT_SYMBOL(apple_mbox_send_ready); + int apple_mbox_poll(struct apple_mbox *mbox) { unsigned long flags; @@ -1015,7 +1072,8 @@ int apple_mbox_start(struct apple_mbox *mbox) goto out_unlock; } - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; out_unlock: @@ -1034,7 +1092,8 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) apple_mbox_v8_rx_stop(mbox); } else { mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1048,6 +1107,25 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +/* + * Bit 0 of the receive control word, read and written back. A client whose + * firmware powers itself down clears it when the firmware has answered, and sets + * it before starting the firmware again. That it gates receiving is inferred. + */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&mbox->rx_lock, flags); + val = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + val = enabled ? val | APPLE_ASC_MBOX_CONTROL_RECV_ENABLE : + val & ~APPLE_ASC_MBOX_CONTROL_RECV_ENABLE; + writel_relaxed(val, mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} +EXPORT_SYMBOL_GPL(apple_mbox_set_recv_enabled); + /* * The mailbox's power domain was off while it stayed started (its consumer * powered its co-processor down and up again around it): set the @@ -1069,13 +1147,15 @@ EXPORT_SYMBOL(apple_mbox_restore); void apple_mbox_hold_rx(struct apple_mbox *mbox) { - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_hold_rx); void apple_mbox_release_rx(struct apple_mbox *mbox) { - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_release_rx); @@ -1116,7 +1196,7 @@ static void apple_mbox_quiesce(void *data) if (was_active) { if (apple_mbox_has_rx_poller(mbox)) apple_mbox_v8_rx_stop(mbox); - else + else if (mbox->irq_recv_not_empty >= 0) disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1136,7 +1216,8 @@ static void apple_mbox_quiesce(void *data) if (mbox->irq_send_empty >= 0) synchronize_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); - synchronize_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + synchronize_irq(mbox->irq_recv_not_empty); spin_lock_irqsave(&mbox->rx_lock, flags); had_rx = mbox->rx; @@ -1411,19 +1492,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; } - mbox->irq_recv_not_empty = - platform_get_irq_byname(pdev, "recv-not-empty"); - if (mbox->irq_recv_not_empty < 0) - return mbox->irq_recv_not_empty; + if (mbox->hw->polled) { + mbox->irq_recv_not_empty = -ENXIO; + mbox->irq_send_empty = -ENXIO; + } else { + mbox->irq_recv_not_empty = + platform_get_irq_byname(pdev, "recv-not-empty"); + if (mbox->irq_recv_not_empty < 0) + return mbox->irq_recv_not_empty; - /* - * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. - * apple_mbox_send then polls the A2I control register instead. - */ - mbox->irq_send_empty = - platform_get_irq_byname_optional(pdev, "send-empty"); - if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return mbox->irq_send_empty; + /* + * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. + * apple_mbox_send then polls the A2I control register instead. + */ + mbox->irq_send_empty = + platform_get_irq_byname_optional(pdev, "send-empty"); + if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) + return mbox->irq_send_empty; + } mbox->rx_util_min = -1; mbox->rx_util_min_applied = -1; @@ -1432,21 +1518,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); - if (!irqname) - return -ENOMEM; + if (mbox->irq_recv_not_empty >= 0) { + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); + if (!irqname) + return -ENOMEM; - if (apple_mbox_is_g17(mbox)) - ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, - apple_mbox_recv_irq, - IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, - irqname, mbox); - else - ret = devm_request_irq(dev, mbox->irq_recv_not_empty, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); - if (ret) - return ret; + if (apple_mbox_is_g17(mbox)) + ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, + apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_NO_SUSPEND, + irqname, mbox); + else + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (ret) + return ret; + } if (mbox->irq_send_empty >= 0) { irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", @@ -1627,6 +1716,24 @@ static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { .has_irq_controls = false, }; +/* The kind of port the secure RTBuddy proxy has, with the AOP setup port's registers. */ +static const struct apple_mbox_hw apple_mbox_secure_rtbuddy_hw = { + .polled = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_T8140_AOP_SETUP_A2I_CONTROL, + .a2i_send0 = APPLE_T8140_AOP_SETUP_A2I_SEND0, + .a2i_send1 = APPLE_T8140_AOP_SETUP_A2I_SEND1, + + .i2a_control = APPLE_T8140_AOP_SETUP_I2A_CONTROL, + .i2a_recv0 = APPLE_T8140_AOP_SETUP_I2A_RECV0, + .i2a_recv1 = APPLE_T8140_AOP_SETUP_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -1750,6 +1857,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6050-j775d-ascwrap-v6", .data = &apple_mbox_t6050_j775d_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, + { .compatible = "apple,secure-rtbuddy-proxy", .data = &apple_mbox_secure_rtbuddy_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* * t6030 gfx-asc mailbox in the boot.bin DTB: it carries no asc-mailbox-v4 fallback so that diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 3b9b47a6d6213..12d40d05b305f 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) #include @@ -39,6 +41,8 @@ struct apple_mcc { unsigned int nr_planes; bool enabled; bool retained; + /* Turned off for a deep sleep, to be turned back on after it. */ + bool sleep_disabled; }; static void apple_mcc_writel(struct apple_mcc *mcc, unsigned int plane, @@ -236,6 +240,43 @@ static void apple_mcc_shutdown(struct platform_device *pdev) apple_mcc_quiesce(mcc); } +/* + * A deep sleep takes the cache's power with the rest of the SoC: stop allocating and write it + * back first, as at shutdown, and allocate again after. Suspend-to-idle keeps it. + */ +static int apple_mcc_suspend_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + + if (!pm_suspend_via_firmware() || !mcc->enabled || mcc->retained) + return 0; + apple_mcc_quiesce(mcc); + mcc->sleep_disabled = true; + + return 0; +} + +static int apple_mcc_resume_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + int ret; + + if (!mcc->sleep_disabled) + return 0; + mcc->sleep_disabled = false; + /* Only a performance setting: a cache not ready after the wake stays off. */ + ret = apple_mcc_enable(mcc); + mcc->enabled = !ret; + if (ret) + dev_warn(dev, "system level cache allocation stays off after the sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_mcc_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_mcc_suspend_noirq, apple_mcc_resume_noirq) +}; + static const struct of_device_id apple_mcc_of_match[] = { { .compatible = "apple,t8132-mcc" }, { .compatible = "apple,t8140-mcc" }, @@ -247,6 +288,7 @@ static struct platform_driver apple_mcc_driver = { .driver = { .name = "apple-mcc", .of_match_table = apple_mcc_of_match, + .pm = pm_sleep_ptr(&apple_mcc_pm_ops), }, .probe = apple_mcc_probe, .remove = apple_mcc_remove, diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 3701f08c18487..a65e5920102f4 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -8,6 +8,10 @@ //! is not frozen either, so entropy requests are held over the same window. //! The notifier itself is in `pm_shim.c`; the capture and entropy handling is //! on [`SepData`]. +//! +//! On a firmware-driven suspend the enclave also saves its state and gates its +//! own power domain when asked, as the sleep powers it down, and has to be +//! started again after a wake. See [`SleepGate`] and `SepData::sleep_enter`. use core::sync::atomic::{AtomicPtr, Ordering}; @@ -23,9 +27,138 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; + fn sep_pm_register( + dev: *mut c_void, + event: unsafe extern "C" fn(bool), + sleep: unsafe extern "C" fn() -> c_int, + wake: unsafe extern "C" fn(), + told: unsafe extern "C" fn() -> bool, + ) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; + fn sep_pm_wait_gated(dev: *mut c_void, timeout_ms: c_uint) -> c_int; + fn sep_pm_clear_dev_disable(dev: *mut c_void) -> c_int; + fn sep_pm_dart_prepare(dev: *mut c_void) -> c_int; + fn sep_pm_dart_restore(dev: *mut c_void); + fn sep_pm_commit_prepare( + dev: *mut c_void, + mbox: *mut c_void, + msg0: u64, + msg1: u32, + reply: u64, + mask: u64, + ) -> c_int; + fn sep_pm_commit_finish(dev: *mut c_void) -> c_int; + pub(crate) static sep_pm_ops: kernel::bindings::dev_pm_ops; +} + +/// How long the enclave gets to gate its domain once it has answered. +const GATE_TIMEOUT_MS: u32 = 5000; + +/// Who may use the mailbox around a firmware-driven suspend. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SleepGate { + /// Anyone. + Open = 0, + /// Replies to what the enclave asked, and the sleep request itself. + Draining = 1, + /// Nobody until the wake: the enclave is gated or about to be. + Told = 2, + /// Nobody: the enclave did not come back. A later sleep may revive it. + Lost = 3, +} + +impl SleepGate { + pub(crate) fn from_u32(v: u32) -> SleepGate { + match v { + 0 => SleepGate::Open, + 1 => SleepGate::Draining, + 2 => SleepGate::Told, + _ => SleepGate::Lost, + } + } + + pub(crate) fn may_request(self) -> bool { + self == SleepGate::Open + } + + pub(crate) fn may_reply(self) -> bool { + matches!(self, SleepGate::Open | SleepGate::Draining) + } + + /// The enclave may be gated, so a wake has to look. + pub(crate) fn needs_wake(self) -> bool { + matches!(self, SleepGate::Told | SleepGate::Lost) + } +} + +/// Whether the enclave gated itself within the bound. +pub(crate) fn wait_gated(dev: &kernel::device::Device) -> bool { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_wait_gated(dev.as_raw().cast(), GATE_TIMEOUT_MS) == 0 } +} + +/// Saves the enclave's DART while it can be read, before the enclave is told. +pub(crate) fn dart_prepare(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_dart_prepare(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Leaves `request` to the sleep driver's last refusal, which sends it with interrupts off +/// and saves the DART just before. Its answer is the message whose `mask` bits read `reply`. +/// False where there's no such step, and the request goes now. +pub(crate) fn leave_for_commit( + dev: &kernel::device::Device, + mbox: *mut kernel::bindings::apple_mbox, + request: kernel::soc::apple::mailbox::Message, + reply: u64, + mask: u64, +) -> bool { + // SAFETY: `dev` is the bound SEP device and `mbox` its mailbox, which outlives the sleep: + // `commit_finish` takes the request back at the resume. + let rc = unsafe { + sep_pm_commit_prepare( + dev.as_raw().cast(), + mbox.cast(), + request.msg0, + request.msg1, + reply, + mask, + ) + }; + rc == 0 +} + +/// After a sleep whose request was left to its last refusal: whether it went out. `None` if +/// none was left. +pub(crate) fn commit_finish(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_commit_finish(dev.as_raw().cast()) }; + if rc < 0 { + None + } else { + Some(rc > 0) + } +} + +/// Puts the enclave's DART back if it lost its state, before the doorbell. +pub(crate) fn dart_restore(dev: &kernel::device::Device) { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_dart_restore(dev.as_raw().cast()) }; +} + +/// Lets a gated enclave start again. Writes only if the bit is set. +pub(crate) fn clear_dev_disable(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_clear_dev_disable(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) } /// Attaches every power domain of a multi-domain SEP node and keeps them on @@ -92,7 +225,41 @@ unsafe extern "C" fn pm_event(entering: bool) { sleep_event(&*sep, entering); } +/// Runs `f` on the registered instance. Only for callbacks that +/// `sep_pm_unregister` waits for or that the bound device excludes. +fn with_sep(none: R, f: impl FnOnce(&SepData) -> R) -> R { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return none; + } + // SAFETY: as in `pm_event`. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + f(&*sep) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's suspend callback. +unsafe extern "C" fn pm_sleep() -> c_int { + with_sep(0, |sep| match sep.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + }) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's resume callback. +unsafe extern "C" fn pm_wake() { + with_sep((), |sep| sep.sleep_exit()); +} + +/// # Safety +/// Called by `pm_shim.c`, whether or not the device suspended. +unsafe extern "C" fn pm_told() -> bool { + with_sep(false, |sep| sep.sleep_gate() == SleepGate::Told) +} + pub(crate) fn register(sep: Arc) -> Result<()> { + let dev: *mut c_void = sep.dev.as_raw().cast(); let ptr = sep.into_foreign(); if SEP @@ -111,7 +278,7 @@ pub(crate) fn register(sep: Arc) -> Result<()> { // SAFETY: the shim stores the callback and registers a notifier block // that lives for the module's lifetime. - let rc = unsafe { sep_pm_register(pm_event) }; + let rc = unsafe { sep_pm_register(dev, pm_event, pm_sleep, pm_wake, pm_told) }; if rc != 0 { let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); if !raw.is_null() { @@ -187,6 +354,33 @@ mod tests { } } + #[test] + fn the_gate_lets_replies_out_until_the_enclave_is_told() { + assert!(SleepGate::Open.may_request()); + assert!(SleepGate::Open.may_reply()); + assert!(!SleepGate::Draining.may_request()); + assert!(SleepGate::Draining.may_reply()); + assert!(!SleepGate::Told.may_request()); + assert!(!SleepGate::Told.may_reply()); + assert!(!SleepGate::Lost.may_request()); + assert!(!SleepGate::Lost.may_reply()); + } + + #[test] + fn only_a_told_or_lost_enclave_needs_waking() { + assert!(!SleepGate::Open.needs_wake()); + assert!(!SleepGate::Draining.needs_wake()); + assert!(SleepGate::Told.needs_wake()); + assert!(SleepGate::Lost.needs_wake()); + } + + #[test] + fn the_gate_round_trips() { + for gate in [SleepGate::Open, SleepGate::Draining, SleepGate::Told, SleepGate::Lost] { + assert_eq!(SleepGate::from_u32(gate as u32), gate); + } + } + #[test] fn going_to_sleep_holds_entropy_then_captures() { let calls = Calls::new(); diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 76846ad5aaac3..f7ca2e711e6be 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -3,18 +3,41 @@ * System sleep for the SEP driver: the notifier only forwards "about to sleep" * and "back from sleep" to Rust, which ends any Touch ID capture before the * system suspends (see pm.rs), and the enclave's power domains are kept on - * across sleep. + * across sleep. The device's suspend and resume callbacks are the firmware + * suspend's: the enclave is told, and gates its own domain, whose state word is + * read here. On T8140 at EL2 the sleep driver sends the request, as its last + * refusal. */ #include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include #include #include "shim.h" +/* The state word of a PMGR power domain, as pmgr-pwrstate has it. */ +#define SEP_PS_DEV_DISABLE BIT(10) + static void (*sep_pm_event)(bool entering); +static int (*sep_pm_sleep)(void); +static void (*sep_pm_wake)(void); +static bool (*sep_pm_told)(void); +/* In the device's .suspend, the only caller that can leave the request to the sleep. */ +static bool sep_pm_suspending; static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) @@ -46,15 +69,375 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(void (*event)(bool entering)) +/* The enclave's own domain is the first one its node lists. */ +static int sep_pm_domain(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + + *regmap = syscon_node_to_regmap(np->parent); + ret = PTR_ERR_OR_ZERO(*regmap); + if (!ret) + ret = of_property_read_u32(np, "reg", offset); + of_node_put(np); + + return ret; +} + +/* + * Returns 0 once the enclave has gated itself, which shows as DEV_DISABLE, and + * -ENOENT if there is no domain to read. The actual state is no guide: a running + * enclave idles at 0 with auto-enable on. + */ +int sep_pm_wait_gated(void *ptr, unsigned int timeout_ms) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + return regmap_read_poll_timeout(regmap, offset, val, + val & SEP_PS_DEV_DISABLE, + 1000, timeout_ms * USEC_PER_MSEC); +} + +int sep_pm_clear_dev_disable(void *ptr) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + ret = regmap_read(regmap, offset, &val); + if (ret || !(val & SEP_PS_DEV_DISABLE)) + return ret; + + return regmap_write(regmap, offset, val & ~SEP_PS_DEV_DISABLE); +} + +/* The enclave's DART, the first one its node lists, with a reference held. */ +static struct device *sep_pm_dart(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + + np = of_parse_phandle(dev->of_node, "iommus", 0); + if (!np) + return NULL; + pdev = of_find_device_by_node(np); + of_node_put(np); + + return pdev ? &pdev->dev : NULL; +} + +int sep_pm_dart_prepare(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + int ret; + + if (!dart) + return -ENODEV; + ret = apple_dart_sleep_prepare(dart); + put_device(dart); + + return ret; +} + +void sep_pm_dart_restore(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + + if (dart) { + apple_dart_sleep_restore(dart); + put_device(dart); + } +} + +#define SEP_PM_POLL_US 20 +#define SEP_PM_TIMEOUT_US (5000 * USEC_PER_MSEC) + +/* + * The sleep request, sent by the sleep driver as its very last refusal, with interrupts off: + * an enclave that gated doesn't come back with its power kept, so a sleep refused anywhere + * before must find it never told. The driver hands over the request and its answer's match + * at .suspend, with its receive interrupt held. + */ +static struct sep_pm_commit { + struct device *dev; + struct device *dart; + struct apple_mbox *mbox; + void __iomem *ps; + struct apple_mbox_msg request; + u64 reply, reply_mask; + void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); + void *cookie; + bool registered, sent, answered; + unsigned int answer_us; + int gate_ret; + u32 state; +} sep_pm_commit; + +/* The answer stops here. Anything else goes to the driver, which reads it after the wake. */ +static void sep_pm_commit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct sep_pm_commit *c = cookie; + + if (c->sent && (msg.msg0 & c->reply_mask) == c->reply) { + c->answered = true; + return; + } + if (c->rx) + c->rx(mbox, msg, c->cookie); +} + +static void sep_pm_commit_take_rx(struct sep_pm_commit *c, bool take) +{ + unsigned long flags; + + spin_lock_irqsave(&c->mbox->rx_lock, flags); + if (take) { + c->rx = c->mbox->rx; + c->cookie = c->mbox->cookie; + c->mbox->rx = sep_pm_commit_rx; + c->mbox->cookie = c; + } else { + c->mbox->rx = c->rx; + c->mbox->cookie = c->cookie; + } + spin_unlock_irqrestore(&c->mbox->rx_lock, flags); +} + +/* 0 once the enclave has gated. A sleep refused after it was sent finds it told. */ +static int sep_pm_commit_tell(void *data) +{ + struct sep_pm_commit *c = data; + unsigned int us; + int ret = 0; + + c->sent = false; + c->answered = false; + c->answer_us = 0; + c->gate_ret = -EINVAL; + + sep_pm_commit_take_rx(c, true); + /* What was already waiting first. */ + apple_mbox_poll(c->mbox); + for (us = 0; !apple_mbox_send_ready(c->mbox); us += SEP_PM_POLL_US) { + if (us >= SEP_PM_TIMEOUT_US) { + ret = -ETIMEDOUT; + goto out; + } + udelay(SEP_PM_POLL_US); + } + /* Its DART is saved, with macOS's writes, right before the request, as at .suspend. */ + ret = apple_dart_sleep_prepare(c->dart); + if (ret) + goto out; + ret = apple_mbox_send(c->mbox, c->request, true); + if (ret) { + /* Not a real sleep: this only drops the saved copy. */ + apple_dart_sleep_restore(c->dart); + goto out; + } + c->sent = true; + /* Never read a gated enclave's mailbox: that stalls the CPU for good. */ + while (!c->answered && !(readl(c->ps) & SEP_PS_DEV_DISABLE) && + c->answer_us < SEP_PM_TIMEOUT_US) { + apple_mbox_poll(c->mbox); + if (!c->answered) { + udelay(SEP_PM_POLL_US); + c->answer_us += SEP_PM_POLL_US; + } + } +out: + sep_pm_commit_take_rx(c, false); + if (!c->sent) + return ret; + if (c->answered) + apple_mbox_set_recv_enabled(c->mbox, false); + c->gate_ret = readl_poll_timeout_atomic(c->ps, c->state, c->state & SEP_PS_DEV_DISABLE, + SEP_PM_POLL_US, SEP_PM_TIMEOUT_US); + + return c->gate_ret; +} + +/* + * From .suspend, with the enclave drained and its receive interrupt held: leaves the + * request in @msg0 and @msg1 to the sleep driver's last refusal. Its answer is the message + * whose @mask bits read @reply. -EOPNOTSUPP where there's no such step. + */ +int sep_pm_commit_prepare(void *ptr, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask) +{ + struct sep_pm_commit *c = &sep_pm_commit; + struct device *dev = ptr; + struct device_node *np; + struct resource res; + u32 offset; + int ret; + + if (!READ_ONCE(sep_pm_suspending)) + return -EOPNOTSUPP; + if (c->registered) + return -EBUSY; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + ret = of_property_read_u32(np, "reg", &offset); + if (!ret) + ret = np->parent ? of_address_to_resource(np->parent, 0, &res) : -EINVAL; + of_node_put(np); + if (ret) + return ret; + if (offset + 4 > resource_size(&res)) + return -EINVAL; + + c->dart = sep_pm_dart(dev); + if (!c->dart || !dev_get_drvdata(c->dart)) { + ret = -ENODEV; + goto err; + } + c->ps = ioremap(res.start + offset, 4); + if (!c->ps) { + ret = -ENOMEM; + goto err; + } + c->dev = dev; + c->mbox = mbox; + c->request.msg0 = msg0; + c->request.msg1 = msg1; + c->reply = reply; + c->reply_mask = mask; + c->sent = false; + ret = apple_sleep_enclave_register(dev, sep_pm_commit_tell, c); + if (ret) + goto err; + c->registered = true; + + return 0; + +err: + if (c->ps) + iounmap(c->ps); + c->ps = NULL; + if (c->dart) + put_device(c->dart); + c->dart = NULL; + return ret; +} + +/* After the sleep, refused or not: 1 if the request went out, 0 if not, <0 if none was left. */ +int sep_pm_commit_finish(void *ptr) +{ + struct sep_pm_commit *c = &sep_pm_commit; + + if (!c->registered || c->dev != ptr) + return -ENOENT; + apple_sleep_enclave_unregister(c->dev); + c->registered = false; + if (c->sent) + dev_dbg(c->dev, "sleep request at the last refusal: %s in %u us, gate %d, state %#x\n", + c->answered ? "answered" : "no answer", c->answer_us, c->gate_ret, + c->state); + iounmap(c->ps); + c->ps = NULL; + put_device(c->dart); + c->dart = NULL; + + return c->sent; +} + +/* + * Only a firmware-driven suspend tells the enclave anything, so every other sleep + * stays as it was. Refusing is safe: the PM core runs the resume callback of a + * device whose suspend had passed when a later one failed, and sleep_enter undoes + * what it did if its own step fails. + */ +static int __maybe_unused sep_pm_suspend(struct device *dev) +{ + int (*sleep)(void) = READ_ONCE(sep_pm_sleep); + int ret; + + if (!pm_suspend_via_firmware() || !sleep) + return 0; + + WRITE_ONCE(sep_pm_suspending, true); + ret = sleep(); + WRITE_ONCE(sep_pm_suspending, false); + + return ret; +} + +/* The last step before the platform's own: the domain has to be down by now. */ +static int __maybe_unused sep_pm_suspend_noirq(struct device *dev) +{ + bool (*told)(void) = READ_ONCE(sep_pm_told); + int ret; + + if (!pm_suspend_via_firmware() || !told || !told()) + return 0; + + ret = sep_pm_wait_gated(dev, 5000); + if (ret == -ENOENT) + return 0; + if (ret) { + dev_err(dev, "the enclave did not gate its power domain: %d\n", ret); + } else { + struct regmap *regmap; + u32 offset, val = 0; + + if (!sep_pm_domain(dev, ®map, &offset)) + regmap_read(regmap, offset, &val); + dev_dbg(dev, "the enclave gated its power domain, state %#x\n", val); + } + + return ret; +} + +static int __maybe_unused sep_pm_resume(struct device *dev) +{ + void (*wake)(void) = READ_ONCE(sep_pm_wake); + + if (wake) + wake(); + + return 0; +} + +const struct dev_pm_ops sep_pm_ops = { + .suspend = pm_sleep_ptr(sep_pm_suspend), + .resume = pm_sleep_ptr(sep_pm_resume), + .suspend_noirq = pm_sleep_ptr(sep_pm_suspend_noirq), +}; + +int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)) { int ret; WRITE_ONCE(sep_pm_event, event); + WRITE_ONCE(sep_pm_sleep, sleep); + WRITE_ONCE(sep_pm_wake, wake); + WRITE_ONCE(sep_pm_told, told); ret = register_pm_notifier(&sep_pm_nb); - if (ret) + if (ret) { WRITE_ONCE(sep_pm_event, NULL); - return ret; + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); + return ret; + } + + return 0; } /* Returns only once no notifier call is still running. */ @@ -62,6 +445,9 @@ void sep_pm_unregister(void) { unregister_pm_notifier(&sep_pm_nb); WRITE_ONCE(sep_pm_event, NULL); + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); } static void sep_pm_release_domains(void *data) diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 41091755a77c6..93a8aa9123757 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -1,15 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include #include #include +#include +#include #include +#include #include +#include +#include +#include /* Join removal and serialize the firmware's untagged power reply stream. */ static DEFINE_MUTEX(pmp_lock); static const void *pmp_data; static struct device *pmp_device; +/* The firmware is stopped for a system sleep. */ +static bool pmp_asleep; int apple_pmp_register(struct device *dev, const void *data) { @@ -66,6 +75,8 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; /* The matched native report must be ready; never use an elapsed-time fallback. */ ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) @@ -104,9 +115,148 @@ int apple_pmp_t6030_gpu_ready(void) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); if (ret) return ret; return apple_pmp_report_set_t6030_gfx(pmp_device, true); } EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); + +/* + * The PMP's idle is a point of no return: an idled PMP doesn't come back with its power kept, + * and only a real sleep restarts it. Where the sleep driver can take them, its idle + * and stop move to the sleep's last step, after every refusal: a refused sleep then finds it + * still running, and the GPU's request, withdrawn at the commit, reaches a running PMP. + */ +static struct apple_pmp_commit_stop pmp_commit; +static bool pmp_commit_registered, pmp_commit_used; + +/* Interrupts off, one CPU: what apple_pmp_sleep() does, through pointers taken before. */ +static int apple_pmp_commit_stop_fn(void *unused) +{ + struct apple_pmp_commit_stop *c = &pmp_commit; + unsigned int us; + size_t i; + int ret; + + ret = apple_rtkit_sleep_idle_polled(c->rtk, c->control, &us); + apple_rtkit_mailbox_recv_off(c->rtk); + for (i = 0; c->sram && i < c->count; i++) + c->words[i] = readl_relaxed(c->sram + 4 * i); + /* Told to idle, it can't carry on whatever it answered: the sleep goes on. */ + return ret ? 1 : 0; +} + +/* + * Retained J713 power votes have no teardown. On T8140 an unbind leaves the firmware running, + * and a commit with it running reset the SoC: its sleep registration is what refuses + * then, every sleep until reboot, since a rebind can't register again. + */ +bool apple_pmp_keep_bound(void) +{ + return apple_pmp_display_retained_board() || of_machine_is_compatible("apple,t8140"); +} + +/* Whether the sleep's last step takes the PMP: registered with the sleep driver, once. */ +static bool apple_pmp_commit_available(struct device *dev) +{ + void __iomem *asc; + int idx; + + if (pmp_commit_registered) + return true; + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return false; + if (apple_sleep_coproc_register(dev, asc, false, apple_pmp_commit_stop_fn, NULL)) { + iounmap(asc); + return false; + } + pmp_commit_registered = true; + + return true; +} + +static int apple_pmp_commit_prepare(struct device *dev) +{ + int ret; + + ret = apple_pmp_sleep_prepare(pmp_data, &pmp_commit); + if (ret) + return ret; + apple_sleep_coproc_set_ready(dev, true); + pmp_commit_used = true; + dev_dbg(dev, "idle and stop left for the sleep's commit\n"); + + return 0; +} + +static int apple_pmp_suspend_late(struct device *dev) +{ + bool at_commit; + int ret; + + if (!pm_suspend_via_firmware()) + return 0; + /* Retained power votes have no teardown. */ + if (apple_pmp_display_retained_board()) + return -EBUSY; + + guard(mutex)(&pmp_lock); + if (pmp_device != dev || pmp_asleep) + return 0; + at_commit = apple_pmp_commit_available(dev); + apple_pmp_report_sleep(dev, at_commit); + ret = at_commit ? apple_pmp_commit_prepare(dev) : apple_pmp_sleep(pmp_data); + if (ret) { + apple_pmp_report_wake(dev, false); + return dev_err_probe(dev, ret, "firmware did not go idle for sleep\n"); + } + pmp_asleep = true; + + return 0; +} + +static int apple_pmp_resume_early(struct device *dev) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_asleep) + return 0; + /* Left for the commit: if the sleep was refused before it, the firmware never idled. */ + if (pmp_commit_used) { + bool stopped = apple_sleep_coproc_stopped(dev); + + pmp_commit_used = false; + apple_sleep_coproc_set_ready(dev, false); + if (!pm_resume_via_firmware() && !stopped) { + pmp_asleep = false; + apple_pmp_report_wake(dev, true); + dev_info(dev, "sleep refused before the commit: firmware never stopped\n"); + return 0; + } + } + /* A stopped firmware doesn't restart with its power kept: the next real sleep does. */ + if (!pm_resume_via_firmware()) { + dev_warn(dev, "sleep refused: firmware stays stopped until a real sleep\n"); + return 0; + } + pmp_asleep = false; + ret = apple_pmp_wake(pmp_data); + if (ret) + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + else + dev_dbg(dev, "firmware back from sleep\n"); + /* Requests go back either way: consumers read them from the report too. */ + apple_pmp_report_wake(dev, true); + + return 0; +} + +const struct dev_pm_ops apple_pmp_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmp_suspend_late, apple_pmp_resume_early) +}; diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 3a9f970bf4893..ff2c1bf973e04 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -71,6 +71,7 @@ const BOOTARGS_UUID_MAGIC: u32 = 0x6469_7575; // "uuid" const BOOTARGS_UUID_VERSION: u32 = 5; const BOOTARGS_OFFSET: usize = 0x22c; const BOOTARGS_SIZE: usize = 0x230; +const ASC_CPU_CONTROL_OFFSET: usize = 0x44; const PMP_ENDPOINT: u8 = 0x20; const PM_COMMAND_MARKER: u64 = pmp_power::MARKER; const PM_COMMAND_SHIFT: u32 = pmp_power::SHIFT; @@ -120,11 +121,19 @@ unsafe impl AsBytes for IovaTableEntry {} // SAFETY: TODO: unsafe impl FromBytes for IovaTableEntry {} +/// A range of the firmware's SRAM as it was when the firmware stopped. +struct SramCopy { + offset: usize, + words: KVVec, +} + struct PmpState { iova_table: Option>, allocs: KVec, value_buf: Option, ioreg_entries: KVec, + /// Held from a system sleep's start until its end. + sram: Option, } impl PmpState { @@ -134,6 +143,7 @@ impl PmpState { allocs: KVec::with_capacity(10, GFP_KERNEL)?, value_buf: None, ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + sram: None, }) } fn find_alloc(&self, addr: u64) -> Option { @@ -169,10 +179,12 @@ struct PmpData { display: AtomicPtr, /// The first failure seen before or after the bridge was published. display_error: AtomicI32, + /// See [`PmpHwConfig::sram_backup`]. + sram_backup: Option<(usize, usize)>, } impl PmpData { - fn new(dev: &platform::Device>) -> Result> { + fn new(dev: &platform::Device>, cfg: &PmpHwConfig) -> Result> { let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = pmp_req.iomap_sized::()?.into_devres()?; let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; @@ -189,6 +201,7 @@ impl PmpData { asc_mmio, display: AtomicPtr::new(core::ptr::null_mut()), display_error: AtomicI32::new(0), + sram_backup: cfg.sram_backup, } ), GFP_KERNEL, @@ -438,6 +451,112 @@ impl PmpData { ) }; } + /// The handle, to wait on the firmware with the lock free: its messages take that lock. + fn rtkit_handle(&self) -> Result<*mut bindings::apple_rtkit> { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + Ok(rtk.as_raw()) + } + /// The ASC's CPU control register. + fn asc_cpu_control(&self) -> Result<*mut kernel::ffi::c_void> { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let asc_mmio = self.asc_mmio.access(bound_dev)?.as_view(); + Ok(MmioBackend::as_ptr(asc_mmio) + .cast::() + .wrapping_add(ASC_CPU_CONTROL_OFFSET) + .cast()) + } + /// Room for what SRAM needs kept, taken first so that running out refuses a sleep cleanly. + fn sram_alloc(&self, sram_size: usize) -> Result> { + let Some((offset, size)) = self.sram_backup else { + return Ok(None); + }; + if !pmp_bootargs::word_range_valid(offset, size, sram_size) { + return Err(EINVAL); + } + let mut words = KVVec::new(); + words.resize(size / 4, 0u32, GFP_KERNEL)?; + Ok(Some(SramCopy { offset, words })) + } + /// Stops the firmware for a system sleep, then keeps what its restart needs from SRAM. + fn sleep(&self) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + let mut copy = self.sram_alloc(io.size())?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + if let Err(error) = to_result(unsafe { bindings::apple_rtkit_sleep_stop(rtk, control) }) { + // SAFETY: as above. + if to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }).is_err() { + self.fail_display(error); + } + return Err(error); + } + if let Some(copy) = copy.as_mut() { + for (idx, word) in copy.words.iter_mut().enumerate() { + // The range was checked against the mapping when it was allocated. + *word = io.try_read32(copy.offset + idx * 4).unwrap_or(0); + } + } + self.state.lock().sram = copy; + Ok(()) + } + /// For a stop at the commit: the RTKit handle, the CPU control and the SRAM copy's room, + /// whose buffer stays where it is once moved into the state. + fn sleep_prepare(&self, stop: &mut bindings::apple_pmp_commit_stop) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let size = pmp_mmio.as_view().relaxed().size(); + let mut copy = self.sram_alloc(size)?; + stop.rtk = self.rtkit_handle()?; + stop.control = self.asc_cpu_control()?; + stop.sram = core::ptr::null_mut(); + stop.words = core::ptr::null_mut(); + stop.count = 0; + if let Some(copy) = copy.as_mut() { + stop.sram = MmioBackend::as_ptr(pmp_mmio) + .cast::() + .wrapping_add(copy.offset) + .cast(); + stop.words = copy.words.as_mut_ptr(); + stop.count = copy.words.len(); + } + self.state.lock().sram = copy; + Ok(()) + } + /// Writes SRAM back after a sleep that took its power, while the firmware is stopped. + fn sram_restore(&self) -> Result { + let Some(copy) = self.state.lock().sram.take() else { + return Ok(()); + }; + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + for (idx, word) in copy.words.iter().enumerate() { + io.try_write32(*word, copy.offset + idx * 4)?; + } + // The firmware mustn't start ahead of writes still on their way. + let _ = io.try_read32(copy.offset); + dev_dbg!(self.dev, "SRAM restored: {} bytes\n", copy.words.len() * 4); + Ok(()) + } + /// Brings the firmware back after a system sleep. It keeps its endpoints and everything the + /// host gave it, so nothing here repeats what it asked for at probe. + fn wake(&self) -> Result { + self.sram_restore()?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }) + } fn recv_message(&self, msg: u64) -> Result<()> { let pm_command = (msg >> PM_COMMAND_SHIFT) & 0xff; if msg & PM_COMMAND_MARKER != 0 && (pm_command == 0x0f || pm_command == 0x10) { @@ -611,6 +730,58 @@ pub unsafe extern "C" fn apple_pmp_send_power_command( 0 } +/// Stops the firmware for a system sleep. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.sleep() { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Takes what the sleep's last step needs to stop the firmware with interrupts off, and the room +/// for the SRAM it keeps. Nothing is stopped here. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver, and `stop` to writable memory. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep_prepare( + data: *const core::ffi::c_void, + stop: *mut bindings::apple_pmp_commit_stop, +) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + // SAFETY: The caller passes a valid pointer. + let stop = unsafe { &mut *stop }; + match data.sleep_prepare(stop) { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Brings the firmware back after a system sleep. A firmware that doesn't come back disables +/// further power commands. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_wake(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.wake() { + Ok(()) => 0, + Err(error) => { + data.fail_display(error); + error.to_errno() + } + } +} + struct NoBuffer; impl rtkit::Buffer for NoBuffer { fn iova(&self) -> Result { @@ -659,6 +830,9 @@ struct PmpHwConfig { /// When the firmware needs system endpoint 7 (CoreAnalytics) started /// and given its buffer before it acknowledges the IOP power state. coreanalytics: Ep7, + /// The range of the firmware's SRAM, as (offset, size) in bytes, that a system sleep keeps + /// and writes back. `None` where no sleep has been traced. + sram_backup: Option<(usize, usize)>, } /// Whether a PMP firmware needs endpoint 7 to finish its RTKit boot. @@ -676,6 +850,7 @@ const HW_CFG_DEFAULT: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; /// T8140: the DART translates 42 bits and its DMA window starts at 1 TiB. @@ -683,6 +858,7 @@ const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: Some((0x3c000, 0x5c000)), }; /// T6041: the DART window starts at 1 TiB; the boot argument header is @@ -692,6 +868,7 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: true, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 @@ -701,6 +878,7 @@ const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T6050: the DART window starts at 1 TiB (ADT dart-pmp0 vm-base) and spans @@ -709,12 +887,14 @@ const HW_CFG_T6050: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::RetainedBoard, + sram_backup: None, }; impl Drop for PmpDriver { @@ -742,9 +922,14 @@ impl platform::Driver for PmpDriver { type Data<'bound> = PmpDriver; fn suppress_bind_attrs() -> bool { - // Retained J713 power votes have no teardown path. - // SAFETY: read-only board identity check; other boards remain unchanged. - unsafe { bindings::apple_pmp_display_retained_board() } + // Retained J713 power votes have no teardown path, and an unbound T8140 PMP still runs. + // SAFETY: read-only machine identity checks; other machines remain unchanged. + unsafe { bindings::apple_pmp_keep_bound() } + } + + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pmp-export.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(bindings::apple_pmp_pm_ops) }) } const OF_ID_TABLE: Option> = Some(&OF_TABLE); @@ -760,7 +945,7 @@ impl platform::Driver for PmpDriver { // yet. unsafe { pdev.dma_set_mask_and_coherent(mask)? }; } - let data = PmpData::new(pdev)?; + let data = PmpData::new(pdev, cfg)?; let pmp_mmio = data.pmp_mmio.access(pdev.as_ref())?.as_view(); let asc_mmio = data.asc_mmio.access(pdev.as_ref())?.as_view(); let node = dev.fwnode().ok_or(EIO)?; diff --git a/drivers/soc/apple/pmp_bootargs.rs b/drivers/soc/apple/pmp_bootargs.rs index c085a323fee47..bd2abbc94029b 100644 --- a/drivers/soc/apple/pmp_bootargs.rs +++ b/drivers/soc/apple/pmp_bootargs.rs @@ -7,6 +7,11 @@ pub(super) fn region_valid(offset: usize, size: usize, sram_size: usize) -> bool size != 0 && offset <= sram_size && size <= sram_size - offset } +/// A range of SRAM that can be copied as 32-bit words. +pub(super) fn word_range_valid(offset: usize, size: usize, sram_size: usize) -> bool { + region_valid(offset, size, sram_size) && offset % 4 == 0 && size % 4 == 0 +} + fn header(bytes: &[u8]) -> Option<(u32, usize)> { let key = u32::from_le_bytes(bytes.get(..4)?.try_into().ok()?); let size = u32::from_le_bytes(bytes.get(4..8)?.try_into().ok()?) as usize; @@ -69,6 +74,15 @@ mod tests { assert!(!region_valid(1, usize::MAX, usize::MAX)); } + #[test] + fn a_word_range_is_aligned_and_inside_the_window() { + assert!(word_range_valid(0x3c000, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c002, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x5c002, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x64004, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0, 0xa0000)); + } + #[test] fn larger_sram_uses_its_actual_extent() { assert!(region_valid(0x90000, 0x1000, 0x100000)); diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 8fc7278550d9a..1bdea86a4c51e 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -90,6 +90,32 @@ pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; pub(crate) const TAG_ENTROPY: u8 = 0xE7; +/// The tag of the sleep request, kept out of the pool. +pub(crate) const TAG_SLEEP: u8 = 0x08; + +/// The sleep request's answer, in the bits [`SLEEP_REPLY_MASK`] keeps: the control endpoint, +/// its tag and the reply type. +pub(crate) const SLEEP_REPLY: u64 = (EP_CONTROL as u64) + | ((TAG_SLEEP as u64) << MSG_TAG_SHIFT) + | ((CONTROL_REPLY_TYPE as u64) << MSG_TYPE_SHIFT); +pub(crate) const SLEEP_REPLY_MASK: u64 = (1 << MSG_PARAM_SHIFT) - 1; + +// The answer in the trace, 0x0000_0000_0001_0800. +static_assert!(0x0000_0000_0001_0800 & SLEEP_REPLY_MASK == SLEEP_REPLY); + +/// How long the enclave gets to answer the sleep request. +pub(crate) const SLEEP_TIMEOUT_MS: u32 = 5000; + +/// The first message macOS sends a gated enclave after a sleep. The contents +/// vary from sleep to sleep, so it is taken to be a doorbell. This is one of them. +pub(crate) const WAKE_KICK: Message = Message { + msg0: 0xffff_ffff_a482_9303, + msg1: 0, +}; + +/// The control messages a woken enclave sends without being asked, in order. +pub(crate) const WAKE_TYPES: [u8; 2] = [0x0d, 0x10]; + pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; pub(crate) const TAG_POOL_FIRST: u8 = 0x01; @@ -97,6 +123,7 @@ pub(crate) const TAG_POOL_LAST: u8 = 0x7e; static_assert!(TAG_POOL_FIRST > 0); static_assert!(TAG_POOL_LAST < TAG_ENTROPY); +static_assert!(TAG_SLEEP >= TAG_POOL_FIRST && TAG_SLEEP <= TAG_POOL_LAST); pub(crate) struct ControlOp { ty: u8, @@ -161,6 +188,19 @@ pub(crate) fn op_get_entropy() -> ControlOp { } } +/// Asks the enclave to save its state and gate its own power domain. +pub(crate) fn op_sleep() -> ControlOp { + ControlOp { + ty: 0x0c, + param: 0x02, + data: 0, + expect_reply: true, + timeout_ms: SLEEP_TIMEOUT_MS, + reserved_tag: false, + name: c"SLEEP", + } +} + pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { Message { msg0: u64::from(EP_CONTROL) @@ -282,6 +322,34 @@ mod tests { assert_eq!(f.data_lo, 0x1234_5678); } + #[test] + fn sleep_request_matches_the_trace() { + let msg = encode_control(&op_sleep(), TAG_SLEEP); + assert_eq!(msg.msg0, 0x0000_0000_020c_0800); + assert_eq!(msg.msg1, 0); + } + + #[test] + fn its_answer_matches_the_trace() { + let reply = Message { + msg0: 0x0000_0000_0001_0800, + msg1: 0, + }; + let f = decode(&reply); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, CONTROL_REPLY_TYPE); + assert_eq!(f.tag, TAG_SLEEP); + } + + #[test] + fn wake_messages_match_the_trace() { + for (raw, ty) in [(0x000d_0000u64, WAKE_TYPES[0]), (0x0010_0000u64, WAKE_TYPES[1])] { + let f = decode(&Message { msg0: raw, msg1: 0 }); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, ty); + } + } + #[test] fn only_entropy_uses_the_reserved_tag() { assert!(op_get_entropy().uses_reserved_tag()); diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 07b5840da8fde..498d1413e14ef 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -39,6 +39,10 @@ struct apple_rtkit_helper { * it does not list must not replace storage the firmware still owns. */ bool strict_buffers; + /* The firmware was told the AP is going to sleep. */ + bool ap_idle; + /* It was told, gave no answer, and wasn't told otherwise. */ + bool idle_unanswered; void __iomem *asc_base; @@ -380,16 +384,97 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); } +static int apple_rtkit_helper_ap_power(struct apple_rtkit_helper *helper, bool on) +{ + int ret = apple_rtkit_ap_power(helper->rtk, on); + + if (ret) + dev_err(helper->dev, "AP power notice (%s) not answered: %d\n", + on ? "on" : "idle", ret); + /* + * Nothing is sent to undo an unanswered idle notice, which macOS never does: + * .complete looks whether a late answer came, and only then sends AP on. + */ + if (ret && !on) + helper->idle_unanswered = true; + return ret; +} + +static int apple_rtkit_helper_ap_idle(struct apple_rtkit_helper *helper) +{ + int ret; + + if (helper->ap_idle) + return 0; + ret = apple_rtkit_helper_ap_power(helper, false); + if (ret) + return ret; + helper->ap_idle = true; + dev_dbg(helper->dev, "AP idle notice answered for a sleep\n"); + return 0; +} + +static void apple_rtkit_helper_ap_on(struct apple_rtkit_helper *helper) +{ + if (!helper->ap_idle) + return; + helper->ap_idle = false; + if (!apple_rtkit_helper_ap_power(helper, true)) + dev_dbg(helper->dev, "AP on notice answered after a sleep\n"); +} + +/* + * macOS tells this coprocessor first of all. Told before any driver stops its own, + * an unanswered notice refuses a sleep that has stopped nothing yet. + */ +static int __maybe_unused apple_rtkit_helper_prepare(struct device *dev) +{ + if (!pm_suspend_via_firmware()) + return 0; + return apple_rtkit_helper_ap_idle(dev_get_drvdata(dev)); +} + +/* A sleep refused before this device's .suspend never calls its .resume. */ +static void __maybe_unused apple_rtkit_helper_complete(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + + if (helper->idle_unanswered) { + bool late = !apple_rtkit_is_running(helper->rtk); + + helper->idle_unanswered = false; + dev_info(dev, "after the unanswered idle notice: %s\n", + late ? "its answer came late, sending AP on" : + "still no answer, nothing sent"); + helper->ap_idle = late; + } + apple_rtkit_helper_ap_on(helper); +} + static int __maybe_unused apple_rtkit_helper_suspend(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + /* + * A firmware-driven suspend keeps this coprocessor running. It hears the + * AP's power state and nothing else, so an adopted session stays live. + * That is measured on T8140 and unverified on other SoCs. + */ + if (pm_suspend_via_firmware()) + return apple_rtkit_helper_ap_idle(helper); + /* Only suspend-to-idle has a verified live-session retention contract. */ if (helper->adopted && pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; return 0; } +static int __maybe_unused apple_rtkit_helper_resume(struct device *dev) +{ + apple_rtkit_helper_ap_on(dev_get_drvdata(dev)); + return 0; +} + static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); @@ -399,7 +484,10 @@ static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) } static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { + .prepare = pm_sleep_ptr(apple_rtkit_helper_prepare), + .complete = pm_sleep_ptr(apple_rtkit_helper_complete), .suspend = pm_sleep_ptr(apple_rtkit_helper_suspend), + .resume = pm_sleep_ptr(apple_rtkit_helper_resume), .freeze = pm_sleep_ptr(apple_rtkit_helper_freeze), .poweroff = pm_sleep_ptr(apple_rtkit_helper_freeze), }; diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index fb365ebfe1921..7551fe6aca285 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -63,6 +63,17 @@ struct apple_rtkit { * hand out kdebug buffers, and poll the mailbox while waiting. */ bool sync_syslog_ack; + /* Echo each OSLOG channel-0 counter, as macOS does. */ + bool oslog_echo; + /* A counter queued for the receive work and not yet echoed. */ + u64 oslog_owed; + /* Device interrupts are off: poll the mailbox while waiting. */ + bool poll_waits; + /* The firmware is restarting with its buffers and rings still in place. */ + bool warm_restart; + u32 warm_last_mask; + /* A port that only answers the co-processor, see apple_rtkit_port_handshake(). */ + bool port_only; DECLARE_KFIFO(sync_ack_fifo, struct apple_mbox_msg, 64); struct work_struct sync_ack_work; @@ -93,6 +104,8 @@ struct apple_rtkit { /* Session inherited from the bootloader: INIT geometry never observed. */ bool syslog_inherited; bool crashlog_inherited; + /* The crash log buffer is the firmware's own, mapped here when it crashed. */ + bool crashlog_placed; bool oslog_inherited; /* A crash reported before the crashlog buffer was mapped: the log is diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 65faedc3da557..8e324534f8e3e 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -7,10 +7,13 @@ #include "rtkit-internal.h" #include +#include +#include #include #include #include #include +#include enum { APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */ @@ -45,7 +48,7 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, - /* Sent by some co-processors once idle after an IOP IDLE request. */ + /* Sent by some co-processors once idle after an IOP IDLE request, argument 0. */ APPLE_RTKIT_MGMT_IDLE_NOTIFY = 0xc, }; @@ -63,6 +66,9 @@ enum { #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0) +#define APPLE_RTKIT_POLLED_STEP_US 10 +#define APPLE_RTKIT_POLLED_LOOPS 20000 + #define APPLE_RTKIT_CRASHLOG_CRASH 1 #define APPLE_RTKIT_BUFFER_REQUEST 1 @@ -117,6 +123,14 @@ enum { #define APPLE_RTKIT_CA_EVENT 2 #define APPLE_RTKIT_CA_EVENT_ACK 3 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +/* From the CPU control register. Both bits wait for the firmware's last message to be read. */ +#define APPLE_ASC_CPU_STATUS 4 +#define APPLE_ASC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_ASC_CPU_STATUS_IDLE BIT(5) +#define APPLE_ASC_CPU_STATUS_DOWN (APPLE_ASC_CPU_STATUS_STOPPED | APPLE_ASC_CPU_STATUS_IDLE) + #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 @@ -302,7 +316,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) * request early crashlog startup, need the endpoint before the map is * acknowledged: G17P can request its crash DVA right after. */ - if (apple_rtkit_early_crashlog(rtk) && base == 0 && + if (!rtk->warm_restart && apple_rtkit_early_crashlog(rtk) && base == 0 && test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) { ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); if (ret) { @@ -312,6 +326,15 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + /* macOS's secure side sends each part back unchanged. */ + if (rtk->port_only) { + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, msg); + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) { + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + } + return; + } if (rtk->retained_wake) { apple_rtkit_management_rx_epmap_retained(rtk, msg, base, bitmap); return; @@ -323,11 +346,29 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) else reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; + /* + * macOS's reply to a restarted DCP echoes the map, and adds bit 1 to the + * first part: 0x51d becomes 0x51f. It starts no endpoints. Measured under + * aura, 3 of 3. Why bit 1 isn't known, but ep 1 is the crash log's. + */ + if (rtk->warm_restart) { + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) + reply |= bitmap & rtk->warm_last_mask; + else + reply |= bitmap | BIT_ULL(1); + } + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (rtk->warm_restart) { + rtk->warm_restart = false; + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + return; + } if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) dev_info(rtk->dev, "RTKit: crashlog endpoint not in the endpoint map\n"); @@ -444,7 +485,7 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) apple_rtkit_management_rx_ap_pwr_ack(rtk, msg); break; case APPLE_RTKIT_MGMT_IDLE_NOTIFY: - dev_dbg(rtk->dev, "RTKit: co-processor idle\n"); + dev_dbg(rtk->dev, "RTKit: co-processor idle: 0x%llx\n", msg); break; default: dev_warn( @@ -569,10 +610,14 @@ static void apple_rtkit_free_buffer(struct apple_rtkit *rtk, if (bfr->size == 0) return; - if (rtk->ops->shmem_destroy) + if (bfr == &rtk->crashlog_buffer && rtk->crashlog_placed) { + memunmap(bfr->buffer); + rtk->crashlog_placed = false; + } else if (rtk->ops->shmem_destroy) { rtk->ops->shmem_destroy(rtk->cookie, bfr); - else if (bfr->buffer) + } else if (bfr->buffer) { dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova); + } bfr->buffer = NULL; bfr->iomem = NULL; @@ -804,6 +849,46 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, len, DMA_FROM_DEVICE); } +/* + * The buffer a crashing firmware placed at its own address, mapped if its IOMMU + * translates every page of it to one contiguous range, and acknowledged as a placed + * buffer is. The firmware doesn't run again until a reboot. + */ +static int apple_rtkit_crashlog_map(struct apple_rtkit *rtk, u64 msg) +{ + struct apple_rtkit_shmem *bfr = &rtk->crashlog_buffer; + struct iommu_domain *domain = iommu_get_domain_for_dev(rtk->dev); + dma_addr_t iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + size_t size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; + phys_addr_t phys, page; + size_t off; + void *va; + + if (!domain || !iova || !size) + return -ENODEV; + phys = iommu_iova_to_phys(domain, iova); + for (off = 0; phys && off < size; off += PAGE_SIZE) { + page = iommu_iova_to_phys(domain, iova + off); + if (page != phys + off) { + dev_err(rtk->dev, "RTKit: crash log at %pad isn't contiguous at +%#zx\n", + &iova, off); + return -EINVAL; + } + } + if (!phys) + return -ENOENT; + va = memremap(phys, size, MEMREMAP_WB); + if (!va) + return -ENOMEM; + bfr->buffer = va; + bfr->size = size; + bfr->iova = iova; + bfr->is_mapped = true; + rtk->crashlog_placed = true; + + return apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_CRASHLOG, msg, NULL, false); +} + #define APPLE_RTKIT_CRASHLOG_LATE_POLL_MS 250 #define APPLE_RTKIT_CRASHLOG_LATE_POLLS 20 @@ -887,6 +972,27 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } + /* + * A firmware that never announced this endpoint asks for its buffer only when + * it crashes, so there's no handshake to take the request for. Report the + * crash now, as the log may never come. A client that sets such a buffer up + * itself takes the path below, which also reads a log written later. + */ + if (!rtk->crashlog_buffer.size && !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints) && + !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { + int ret = apple_rtkit_crashlog_map(rtk, msg); + + dev_err(rtk->dev, "RTKit: co-processor has crashed: crash log buffer at %#llx %s (%d)\n", + FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), + ret ? "not mapped" : "mapped, waiting for its log", ret); + if (!rtk->crashed) { + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + } + return; + } + if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) @@ -938,6 +1044,9 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) "RTKit: Couldn't allocate crashlog shadow buffer\n"); } + /* Its driver heard of the crash when the buffer was asked for. */ + if (rtk->crashed) + return; rtk->crashed = true; if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); @@ -1116,6 +1225,22 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); int err; + /* + * A firmware counts its log writes and holds the next counter until the last is + * echoed, and with one outstanding takes no AP power notice: a DCP after a warm + * restart, the MTP and the SMC after about eight sleeps. macOS echoes every one. + */ + if (type == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo) { + /* A polled idle that came first echoed it already. */ + if (cmpxchg(&rtk->oslog_owed, msg, 0) != msg) + return; + dev_dbg(rtk->dev, "RTKit: oslog counter %016llx, echoed\n", msg); + err = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, msg, NULL, false); + if (err) + dev_err(rtk->dev, "RTKit: oslog echo failed: %d\n", err); + return; + } + /* Preserve the tested Neo/M5 and synchronous-log no-ACK policy. */ if (type == APPLE_RTKIT_OSLOG_LOG && (rtk->sync_syslog_ack || of_machine_is_compatible("apple,t8140") || @@ -1236,12 +1361,17 @@ static void apple_rtkit_rx_work(struct work_struct *work) kfree(rtk_work); } -static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +/* An OSLOG counter this firmware waits to see echoed. */ +static bool apple_rtkit_oslog_owed_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return ep == APPLE_RTKIT_EP_OSLOG && + FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg) == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo; +} + +static bool apple_rtkit_notification_msg(u8 ep, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - if (!rtk->sync_syslog_ack) - return false; if (ep == APPLE_RTKIT_EP_SYSLOG) return type == APPLE_RTKIT_SYSLOG_LOG; if (ep == APPLE_RTKIT_EP_IOREPORT) @@ -1250,6 +1380,11 @@ static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) return false; } +static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return rtk->sync_syslog_ack && apple_rtkit_notification_msg(ep, msg); +} + static void apple_rtkit_sync_ack_work(struct work_struct *work) { struct apple_rtkit *rtk = @@ -1320,6 +1455,8 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, if (!work) return; + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) + WRITE_ONCE(rtk->oslog_owed, msg.msg0); work->rtk = rtk; work->ep = ep; work->msg = msg.msg0; @@ -1365,6 +1502,82 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +static int apple_rtkit_poll_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->epmap_completion)) + return rtk->boot_result; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return try_wait_for_completion(&rtk->epmap_completion) ? rtk->boot_result : -ETIME; +} + +static int apple_rtkit_port_power(struct apple_rtkit *rtk, unsigned int state, + unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + reinit_completion(&rtk->ap_pwr_ack_completion); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state)); + if (ret) + return ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->ap_pwr_ack_completion)) + return 0; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return -ETIME; +} + +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + rtk->port_only = true; + + return apple_rtkit_poll_handshake(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_map); + +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + /* The co-processor starts its side of this port itself. */ + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_INIT, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_on); + +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + int ret; + + ret = apple_rtkit_port_map(rtk, timeout_ms); + if (ret) + return ret; + + return apple_rtkit_port_on(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_handshake); + +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_IDLE, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_idle); + void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { apple_mbox_hold_rx(rtk->mbox); @@ -1442,6 +1655,10 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) set_bit(system_endpoints[i], rtk->endpoints); + /* apple_rtkit_wake() must not wait for an EPMAP that already happened. */ + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->syslog_inherited = true; @@ -1584,6 +1801,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, rtk->handshake_timeout_ms = APPLE_RTKIT_HANDSHAKE_TIMEOUT_MS; rtk->sync_syslog_ack = of_property_read_bool(dev->of_node, "apple,sync-syslog-ack"); + /* As macOS does; a driver whose firmware shouldn't get them turns it off. */ + rtk->oslog_echo = of_machine_is_compatible("apple,t8140"); INIT_KFIFO(rtk->sync_ack_fifo); INIT_WORK(&rtk->sync_ack_work, apple_rtkit_sync_ack_work); @@ -1613,6 +1832,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + rtk->warm_last_mask = U32_MAX; + rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { @@ -1667,15 +1888,18 @@ static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c); /* - * A retained wake is not interrupted by a signal to its caller: it has + * A handshake given up halfway can leave a firmware started that nothing stops. The + * freezer signals every user task as a sleep begins, so only a fatal signal ends the + * wait, and during a system sleep not even that. Nor does a retained wake's: it has * already started the co-processor's CPU, and nothing would undo that. */ static long apple_rtkit_wait_timeout(struct apple_rtkit *rtk, struct completion *c, unsigned long timeout) { - if (READ_ONCE(rtk->retained_wake)) + if (READ_ONCE(rtk->retained_wake) || pm_suspend_target_state == PM_SUSPEND_MEM) return wait_for_completion_timeout(c, timeout); - return wait_for_completion_interruptible_timeout(c, timeout); + + return wait_for_completion_killable_timeout(c, timeout); } /* @@ -1689,7 +1913,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) long t; int ret; - if (!rtk->sync_syslog_ack) + if (!rtk->sync_syslog_ack && !rtk->poll_waits) return apple_rtkit_wait_for_completion(rtk, c); timeout = jiffies + msecs_to_jiffies(rtk->handshake_timeout_ms); @@ -1722,10 +1946,9 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted_with_buffers); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c) { - unsigned long timeout = msecs_to_jiffies(rtk->handshake_timeout_ms); long t; - t = apple_rtkit_wait_timeout(rtk, c, timeout); + t = apple_rtkit_wait_timeout(rtk, c, msecs_to_jiffies(rtk->handshake_timeout_ms)); if (t < 0) return t; else if (t == 0) @@ -1741,6 +1964,63 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, } EXPORT_SYMBOL_GPL(apple_rtkit_set_handshake_timeout); +/* + * Expect HELLO from a firmware being restarted warm: it keeps the buffers it was + * given and its rings, so unlike apple_rtkit_reinit() nothing is freed, and + * endpoints aren't started again. The caller then sets the run bit and calls + * apple_rtkit_boot(). + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk) +{ + unsigned long flags; + int ret; + + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + + reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + + rtk->crashed = false; + rtk->warm_restart = true; + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; + kfifo_reset(&rtk->sync_ack_fifo); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + WRITE_ONCE(rtk->shutting_down, false); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + ret = apple_mbox_start(rtk->mbox); + if (ret) { + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + } else { + apple_mbox_set_recv_enabled(rtk->mbox, true); + } + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_warm_restart); + +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask) +{ + rtk->warm_last_mask = last_mask; +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_warm_epmap_mask); + +int apple_rtkit_port_restart(struct apple_rtkit *rtk) +{ + int ret = apple_rtkit_warm_restart(rtk); + + rtk->warm_restart = false; + rtk->port_only = true; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_restart); + int apple_rtkit_reinit(struct apple_rtkit *rtk) { unsigned long flags; @@ -1846,7 +2126,8 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power, + void (*before_ap)(void *data), void *data) { int ret; @@ -1894,16 +2175,27 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) if (rtk->boot_result) return rtk->boot_result; - return early_ap_power ? 0 : - apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (early_ap_power) + return 0; + + if (before_ap) + before_ap(data); + + return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); } int apple_rtkit_boot(struct apple_rtkit *rtk) { - return apple_rtkit_boot_common(rtk, false); + return apple_rtkit_boot_common(rtk, false, NULL, NULL); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data) +{ + return apple_rtkit_boot_common(rtk, false, before_ap, data); +} +EXPORT_SYMBOL_GPL(apple_rtkit_boot_with); + int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { if (rtk->crashed) @@ -1948,26 +2240,66 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poweroff); -int apple_rtkit_idle(struct apple_rtkit *rtk) +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data) { int ret; /* if OFF is used here the co-processor will not wake up again */ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: AP idle not acknowledged: %d\n", ret); return ret; + } + + if (after_ap) + after_ap(data); ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: IOP idle not acknowledged: %d\n", ret); return ret; + } rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE; return 0; } +EXPORT_SYMBOL_GPL(apple_rtkit_idle_with); + +int apple_rtkit_idle(struct apple_rtkit *rtk) +{ + return apple_rtkit_idle_with(rtk, NULL, NULL); +} EXPORT_SYMBOL_GPL(apple_rtkit_idle); +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_off); + +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on) +{ + return apple_rtkit_set_ap_power_state(rtk, on ? APPLE_RTKIT_PWR_STATE_ON : + APPLE_RTKIT_PWR_STATE_IDLE); +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power); + +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on) +{ + int ret; + + rtk->poll_waits = true; + ret = apple_rtkit_ap_power(rtk, on); + rtk->poll_waits = false; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power_noirq); + int apple_rtkit_quiesce(struct apple_rtkit *rtk) { int ret; @@ -2012,7 +2344,7 @@ static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) if (ret) return ret; - return apple_rtkit_boot_common(rtk, early_ap_power); + return apple_rtkit_boot_common(rtk, early_ap_power, NULL, NULL); } int apple_rtkit_wake(struct apple_rtkit *rtk) @@ -2027,6 +2359,218 @@ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); +static int apple_rtkit_sleep_idle(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + u32 status; + int ret; + + ret = apple_rtkit_idle(rtk); + if (ret) + return ret; + + ret = readl_poll_timeout(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == APPLE_ASC_CPU_STATUS_DOWN, + 100, 5000); + if (ret) + dev_err(rtk->dev, "RTKit: idle, but not stopped: status %#x\n", status); + + return ret; +} + +/* With interrupts off: room in the FIFO, then the message. */ +static bool apple_rtkit_polled_send(struct apple_rtkit *rtk, struct apple_mbox_msg msg) +{ + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return false; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + + return !apple_mbox_send(rtk->mbox, msg, true); +} + +/* + * Only what an idle needs: the state acks, notifications answered as they come, and OSLOG + * counters echoed, without which the firmware takes no AP power notice. + */ +static void apple_rtkit_polled_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct apple_rtkit *rtk = cookie; + u8 ep = msg.msg1; + + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) { + if (!apple_rtkit_polled_send(rtk, msg)) + dev_err(rtk->dev, "RTKit: oslog echo failed while idling\n"); + return; + } + if (ep == APPLE_RTKIT_EP_MGMT) { + u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg.msg0); + unsigned int state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg.msg0); + + if (type == APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK) + rtk->iop_power_state = state; + else if (type == APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK) + rtk->ap_power_state = state; + } else if (apple_rtkit_notification_msg(ep, msg.msg0) && apple_mbox_send_ready(mbox)) { + apple_mbox_send(mbox, msg, true); + } +} + +static int apple_rtkit_polled_state(struct apple_rtkit *rtk, u8 type, unsigned int *state, + unsigned int *us) +{ + struct apple_mbox_msg msg = { + .msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type) | + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_IDLE), + .msg1 = APPLE_RTKIT_EP_MGMT, + }; + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return -ETIMEDOUT; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + apple_mbox_send(rtk->mbox, msg, true); + + for (i = 0; i < APPLE_RTKIT_POLLED_LOOPS; i++) { + apple_mbox_poll(rtk->mbox); + if (READ_ONCE(*state) == APPLE_RTKIT_PWR_STATE_IDLE) + return 0; + udelay(APPLE_RTKIT_POLLED_STEP_US); + *us += APPLE_RTKIT_POLLED_STEP_US; + } + + return -ETIMEDOUT; +} + +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us) +{ + unsigned long flags; + u64 owed; + u32 status; + int ret; + + *us = 0; + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_polled_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + /* What's already waiting is answered first, or the firmware may not go idle. */ + apple_mbox_poll(rtk->mbox); + /* Then a counter left for the receive work, which can't run now. */ + owed = xchg(&rtk->oslog_owed, 0); + if (owed) { + struct apple_mbox_msg echo = { .msg0 = owed, .msg1 = APPLE_RTKIT_EP_OSLOG }; + + if (!apple_rtkit_polled_send(rtk, echo)) + dev_err(rtk->dev, "RTKit: oslog echo failed before the idle\n"); + } + + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + &rtk->ap_power_state, us); + if (!ret) + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + &rtk->iop_power_state, us); + if (!ret) + ret = readl_poll_timeout_atomic(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == + APPLE_ASC_CPU_STATUS_DOWN, + APPLE_RTKIT_POLLED_STEP_US, 5000); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_idle_polled); + +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_recv_off); + +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + ret = apple_rtkit_sleep_idle(rtk, cpu_control); + if (ret) + return ret; + + apple_rtkit_mailbox_off(rtk); + + /* Twice, each time from what it reads back. */ + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_stop); + +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + /* + * No real sleep: a firmware that idled doesn't come back from that with its power kept, + * and stays down until the next sleep restarts it. One whose idle failed still runs. + */ + if (!pm_resume_via_firmware()) + return (readl(cpu_control) & APPLE_ASC_CPU_CONTROL_RUN) && + (rtk->iop_power_state & 0xff) == APPLE_RTKIT_PWR_STATE_ON ? 0 : -EAGAIN; + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return); + +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret = apple_rtkit_warm_restart(rtk); + + if (ret) + return ret; + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_power_cycled_return); + +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo) +{ + rtk->oslog_echo = echo; + dev_dbg(rtk->dev, "RTKit: OSLOG counters %s\n", echo ? "echoed" : "dropped"); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_oslog_echo); + +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + if (!pm_resume_via_firmware()) + return -EOPNOTSUPP; + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return_start); + /* * Wake a co-processor whose CPU was stopped after apple_rtkit_idle(), with * its firmware, endpoints and shared buffers kept (and its power domain @@ -2059,7 +2603,7 @@ int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), voi ret = start(ctx); if (ret) goto out; - ret = apple_rtkit_boot_common(rtk, false); + ret = apple_rtkit_boot_common(rtk, false, NULL, NULL); out: WRITE_ONCE(rtk->retained_wake, false); if (ret) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index f9e79924e53a6..4fa31a0e54f47 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #define APPLE_SART_MAX_ENTRIES 16 @@ -90,6 +91,15 @@ struct apple_sart { unsigned long protected_entries; unsigned long used_entries; + /* The registers as they stood before a firmware-driven suspend */ + struct { + u8 flags; + phys_addr_t paddr; + size_t size; + } saved[APPLE_SART_MAX_ENTRIES]; + bool saved_valid; + /* saved[] still holds the last sleep's entries, for apple_sart_replay() */ + bool replay_valid; }; static void sart0_get_entry(struct apple_sart *sart, int index, u8 *flags, @@ -398,6 +408,98 @@ static void apple_sart_shutdown(struct platform_device *pdev) } } +/* + * A SART in a domain that a firmware-driven suspend powers off comes back empty, + * and its coprocessor can't reach its firmware or buffers until the entries are + * back. This includes the bootloader's, which Linux otherwise never writes. + */ +static int apple_sart_suspend_late(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + int i; + + sart->saved_valid = false; + sart->replay_valid = false; + if (pm_suspend_target_state != PM_SUSPEND_MEM || !pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) + sart->ops->get_entry(sart, i, &sart->saved[i].flags, + &sart->saved[i].paddr, &sart->saved[i].size); + sart->saved_valid = true; + + return 0; +} + +static int apple_sart_resume_noirq(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + unsigned int restored = 0; + int i; + + if (!sart->saved_valid || !pm_resume_via_firmware()) + return 0; + sart->saved_valid = false; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + /* An unused entry stays as the power left it, whatever that reads. */ + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + /* Every one in use, the same or not: macOS rewrites 0 to 7 at every wake. */ + sart->ops->set_entry(sart, i, sart->saved[i].flags, + sart->saved[i].paddr >> sart->ops->paddr_shift, + sart->saved[i].size >> sart->ops->size_shift); + restored++; + } + dev_dbg(dev, "%u entries restored after sleep\n", restored); + sart->replay_valid = true; + + return 0; +} + +/* + * A coprocessor's driver calls this right before its run bit. macOS writes the entries back + * there, config first, then address, then size, so this does the same on SARTv3. Unused + * entries stay as they read. + */ +int apple_sart_replay(struct apple_sart *sart) +{ + int i, bad = 0; + + if (sart->ops != &sart_ops_v3 || !sart->replay_valid) + return -EOPNOTSUPP; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + phys_addr_t paddr; + size_t size; + u8 flags; + + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + writel(sart->saved[i].flags, sart->regs + APPLE_SART3_CONFIG(i)); + writel(sart->saved[i].paddr >> APPLE_SART3_PADDR_SHIFT, + sart->regs + sart->ops->paddr_reg + 4 * i); + writel(sart->saved[i].size >> APPLE_SART3_SIZE_SHIFT, + sart->regs + sart->ops->size_reg + 4 * i); + sart->ops->get_entry(sart, i, &flags, &paddr, &size); + if (flags == sart->saved[i].flags && paddr == sart->saved[i].paddr && + size == sart->saved[i].size) + continue; + bad++; + dev_warn(sart->dev, "entry %d reads %#x %pa+%#zx, wrote %#x %pa+%#zx\n", + i, flags, &paddr, size, sart->saved[i].flags, &sart->saved[i].paddr, + sart->saved[i].size); + } + + return bad; +} +EXPORT_SYMBOL_GPL(apple_sart_replay); + +static const struct dev_pm_ops apple_sart_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_sart_suspend_late, NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, apple_sart_resume_noirq) +}; + static const struct of_device_id apple_sart_of_match[] = { { .compatible = "apple,t6050-sart", @@ -423,6 +525,7 @@ static struct platform_driver apple_sart_driver = { .driver = { .name = "apple-sart", .of_match_table = apple_sart_of_match, + .pm = pm_sleep_ptr(&apple_sart_pm_ops), }, .probe = apple_sart_probe, .shutdown = apple_sart_shutdown, diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index bed228b4d1ca3..5536de938e5ba 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,21 @@ MODULE_PARM_DESC(sensor_spi_mode, struct sep_sensor_lifetime { /* Serializes sensor operations with this device's resource teardown. */ struct mutex lock; + /* The power last written; a failed write counts as on. */ + bool powered; + /* Set across a system sleep: bus and power requests are refused. */ + bool asleep; }; +/* The power line or rails and the bus, with no settling delays. */ +struct sep_sensor_hw { + int (*set_power)(int on); + int (*send)(const void *tx, size_t len); +}; + +/* Puts the sensor in its low-power state; macOS sends it before a sleep. */ +static const u8 sep_sensor_cmd_sleep[] = { 0x80, 0x31, 0x00, 0x07, 0x00, 0x00, 0x00 }; + static DEFINE_MUTEX(sep_sensor_registry_lock); static struct sep_sensor_lifetime *sep_active_sensor; @@ -58,6 +72,7 @@ static struct sep_sensor_lifetime *sep_lock_sensor(void) } static void sep_sensor_irq_teardown_locked(void); +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len); static struct spi_device *sep_spi; static struct gpio_desc *sep_power; @@ -328,6 +343,52 @@ static void sep_release_power(void) sep_power_source = SEP_POWER_NONE; } +static int sep_sensor_set_power_now(int on) +{ + if (sep_power_source == SEP_POWER_SMC_RAILS) + return sep_smc_set_power(on); + if (!sep_power) + return -ENODEV; + gpiod_set_value_cansleep(sep_power, on); + return 0; +} + +static const struct sep_sensor_hw sep_hw = { + .set_power = sep_sensor_set_power_now, + .send = sep_sensor_xfer_tx_locked, +}; + +static int sep_sensor_write_power(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw, int on) +{ + int rc = hw->set_power(on); + + /* A failed write may have taken effect. */ + sensor->powered = rc || on; + return rc; +} + +/* + * As macOS does for a sleep: the sleep command, then the power off. The next + * bring-up's power cycle powers the sensor again, so a resume does not. + * Returns the power-off's error. + */ +static int sep_sensor_sleep_locked(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw) +{ + WRITE_ONCE(sensor->asleep, true); + if (!sensor->powered) + return 0; + /* Best effort: only the power-off matters. */ + hw->send(sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + return sep_sensor_write_power(sensor, hw, 0); +} + +static int sep_sensor_awake_locked(struct sep_sensor_lifetime *sensor) +{ + return READ_ONCE(sensor->asleep) ? -ESHUTDOWN : 0; +} + #define SEP_CS_TIMING_SOFTWARE 0 #define SEP_CS_TIMING_HOOK_BYPASSED 1 #define SEP_CS_TIMING_HARDWARE 2 @@ -462,6 +523,35 @@ static void sep_sensor_remove(struct spi_device *spi) mutex_unlock(&sep_sensor_registry_lock); } +static int sep_sensor_suspend(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + bool was_on; + int rc; + + mutex_lock(&sensor->lock); + was_on = sensor->powered; + rc = sep_sensor_sleep_locked(sensor, &sep_hw); + mutex_unlock(&sensor->lock); + /* Failing the sleep would keep the machine awake; the next bring-up recovers. */ + if (rc) + dev_warn(dev, "sep sensor: power-off for sleep failed (%d)\n", rc); + else if (was_on) + dev_dbg(dev, "sep sensor: off for sleep\n"); + return 0; +} + +/* No lock: a capture still waiting on the data-ready line may hold it. */ +static int sep_sensor_resume(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + + WRITE_ONCE(sensor->asleep, false); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(sep_sensor_pm_ops, sep_sensor_suspend, sep_sensor_resume); + /* * Its own compatible, deliberately not spidev's: capture must stay in the * kernel, and a spidev node would leak an image to userspace. No @@ -485,6 +575,7 @@ static struct spi_driver sep_sensor_driver = { .driver = { .name = "apple-mesa", .of_match_table = sep_sensor_of_match, + .pm = pm_sleep_ptr(&sep_sensor_pm_ops), }, .id_table = sep_sensor_spi_ids, .probe = sep_sensor_probe, @@ -528,26 +619,16 @@ static int sep_sensor_cs_timing_mode_locked(void) } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -static int sep_sensor_power_cycle_locked(void) +static int sep_sensor_power_cycle_locked(struct sep_sensor_lifetime *sensor) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(0); - - if (rc) - return rc; - msleep(SEP_SENSOR_OFF_DELAY_MS); - rc = sep_smc_set_power(1); - if (rc) - return rc; - msleep(SEP_SENSOR_ON_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; + int rc = sep_sensor_write_power(sensor, &sep_hw, 0); - gpiod_set_value_cansleep(sep_power, 0); + if (rc) + return rc; msleep(SEP_SENSOR_OFF_DELAY_MS); - gpiod_set_value_cansleep(sep_power, 1); + rc = sep_sensor_write_power(sensor, &sep_hw, 1); + if (rc) + return rc; msleep(SEP_SENSOR_ON_DELAY_MS); return 0; } @@ -588,24 +669,13 @@ static const char *sep_sensor_firmware_name_locked(unsigned int index) } /* Powers the sensor on/off, holding the hardware settling delay. */ -static int sep_sensor_power_locked(int on) +static int sep_sensor_power_locked(struct sep_sensor_lifetime *sensor, int on) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(on ? 1 : 0); + int rc = sep_sensor_write_power(sensor, &sep_hw, on ? 1 : 0); - if (rc) - return rc; - msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; - - gpiod_set_value_cansleep(sep_power, on ? 1 : 0); - if (on) - msleep(SEP_SENSOR_ON_DELAY_MS); - else - msleep(SEP_SENSOR_OFF_DELAY_MS); + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); return 0; } @@ -846,7 +916,7 @@ int sep_sensor_power_cycle(void) if (!sensor) return -ENODEV; - result = sep_sensor_power_cycle_locked(); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_cycle_locked(sensor); mutex_unlock(&sensor->lock); return result; } @@ -894,7 +964,7 @@ int sep_sensor_power(int on) if (!sensor) return -ENODEV; - result = sep_sensor_power_locked(on); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_locked(sensor, on); mutex_unlock(&sensor->lock); return result; } @@ -962,7 +1032,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_locked(tx, rx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_locked(tx, rx, len); mutex_unlock(&sensor->lock); return result; } @@ -974,7 +1044,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_tx_locked(tx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_tx_locked(tx, len); mutex_unlock(&sensor->lock); return result; } @@ -986,7 +1056,13 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + result = sep_sensor_awake_locked(sensor) ?: + sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); mutex_unlock(&sensor->lock); return result; } + +/* Like the module's Rust tests, which need KUnit built in. */ +#if IS_ENABLED(CONFIG_APPLE_SEP_KUNIT_TEST) && IS_BUILTIN(CONFIG_KUNIT) +#include "sensor_shim_test.c" +#endif diff --git a/drivers/soc/apple/sensor_shim_test.c b/drivers/soc/apple/sensor_shim_test.c new file mode 100644 index 0000000000000..a0374e65bbf5f --- /dev/null +++ b/drivers/soc/apple/sensor_shim_test.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* KUnit tests for the fingerprint sensor's system sleep. No hardware is touched. */ + +#include +#include + +struct sep_fake_hw { + /* 's' for a send, '0' and '1' for power writes, in order. */ + char calls[8]; + unsigned int ncalls; + u8 sent[16]; + size_t sent_len; + int send_rc; + int power_rc; +}; + +static struct sep_fake_hw *sep_fake(void) +{ + return kunit_get_current_test()->priv; +} + +static void sep_fake_note(struct sep_fake_hw *f, char call) +{ + if (f->ncalls < ARRAY_SIZE(f->calls)) + f->calls[f->ncalls++] = call; +} + +static int sep_fake_send(const void *tx, size_t len) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, 's'); + f->sent_len = min(len, sizeof(f->sent)); + memcpy(f->sent, tx, f->sent_len); + return f->send_rc; +} + +static int sep_fake_set_power(int on) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, on ? '1' : '0'); + return f->power_rc; +} + +static const struct sep_sensor_hw sep_fake_hw = { + .set_power = sep_fake_set_power, + .send = sep_fake_send, +}; + +static int sep_sensor_test_init(struct kunit *test) +{ + test->priv = kunit_kzalloc(test, sizeof(struct sep_fake_hw), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, test->priv); + return 0; +} + +static void sep_sensor_sleep_sends_the_command_then_cuts_power(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_EQ(test, f->sent_len, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_MEMEQ(test, f->sent, sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_leaves_an_unpowered_sensor_alone(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = false }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_EXPECT_EQ(test, f->ncalls, 0U); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_cuts_power_when_the_command_fails(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->send_rc = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_FALSE(test, sensor.powered); +} + +static void sep_sensor_sleep_counts_a_failed_power_off_as_on(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_requests_wait_for_the_resume(struct kunit *test) +{ + struct sep_sensor_lifetime sensor = { .powered = true }; + struct device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + sep_sensor_sleep_locked(&sensor, &sep_fake_hw); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), -ESHUTDOWN); + + dev_set_drvdata(dev, &sensor); + KUNIT_EXPECT_EQ(test, sep_sensor_resume(dev), 0); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + /* The resume powers nothing: the next bring-up does. */ + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, ((struct sep_fake_hw *)test->priv)->ncalls, 2U); +} + +static void sep_sensor_power_writes_are_tracked(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = {}; + + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), 0); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), 0); + KUNIT_EXPECT_FALSE(test, sensor.powered); + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + sensor.powered = false; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); +} + +/* Framed like the driver's other commands: 0x80, opcode, then the length at 3. */ +static void sep_sensor_sleep_command_is_framed(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, sizeof(sep_sensor_cmd_sleep), (size_t)7); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[0], (u8)0x80); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[1], (u8)0x31); + KUNIT_EXPECT_EQ(test, (size_t)get_unaligned_le16(&sep_sensor_cmd_sleep[3]), + sizeof(sep_sensor_cmd_sleep)); +} + +static struct kunit_case sep_sensor_sleep_cases[] = { + KUNIT_CASE(sep_sensor_sleep_sends_the_command_then_cuts_power), + KUNIT_CASE(sep_sensor_sleep_leaves_an_unpowered_sensor_alone), + KUNIT_CASE(sep_sensor_sleep_cuts_power_when_the_command_fails), + KUNIT_CASE(sep_sensor_sleep_counts_a_failed_power_off_as_on), + KUNIT_CASE(sep_sensor_requests_wait_for_the_resume), + KUNIT_CASE(sep_sensor_power_writes_are_tracked), + KUNIT_CASE(sep_sensor_sleep_command_is_framed), + {} +}; + +static struct kunit_suite sep_sensor_sleep_suite = { + .name = "apple_sep_sensor_sleep", + .init = sep_sensor_test_init, + .test_cases = sep_sensor_sleep_cases, +}; + +kunit_test_suite(sep_sensor_sleep_suite); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index a2bd2d5dfb4c7..41cc5cda7b48f 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -61,8 +61,10 @@ use kernel::{ sync::{ aref::ARef, atomic::{ + Acquire, Atomic, - Relaxed, // + Relaxed, + Release, // }, barrier::{ dma_mb, @@ -643,6 +645,12 @@ struct SepData { // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, + // A `pm::SleepGate`. + sleep_gate: Atomic, + + // The mailbox's receive interrupt is held off for the enclave's sleep. + sleep_rx_held: Atomic, + #[pin] ool_xarm: Mutex>, @@ -1066,6 +1074,8 @@ impl SepData { sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), + sleep_gate: Atomic::new(pm::SleepGate::Open as u32), + sleep_rx_held: Atomic::new(false), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), @@ -1321,7 +1331,276 @@ impl SepData { if self.shutting_down.load(Relaxed) { return Err(ENODEV); } - Self::ring(self.mbox.lock().as_ref().ok_or(ENODEV)?, msg) + // Checked with the mailbox lock held: a sender can wait for it for longer + // than the system takes to tell the enclave it's going to sleep. + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_request() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + /// As [`Self::send`], for an answer to something the enclave asked: it + /// waits for these before it can answer the sleep request. + fn send_reply(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_reply() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + pub(crate) fn sleep_gate(&self) -> pm::SleepGate { + pm::SleepGate::from_u32(self.sleep_gate.load(Acquire)) + } + + /// Under the mailbox lock, so that a sender holding it finishes first. + fn set_sleep_gate(&self, gate: pm::SleepGate) { + let _mbox = self.mbox.lock(); + self.sleep_gate.store(gate as u32, Release); + } + + /// Asks the enclave to go to sleep. The request is sent past the gate. + fn sleep_request(&self) -> Result<()> { + let op = proto::op_sleep(); + let tag = proto::TAG_SLEEP; + let idx = self.control.lock().alloc_sleep()?; + let msg = proto::encode_control(&op, tag); + + let sent = match self.mbox.lock().as_ref() { + Some(mbox) => Self::ring(mbox, msg), + None => Err(ENODEV), + }; + if let Err(e) = sent { + self.control.lock().abandon(idx); + return Err(e); + } + match self.control_wait(&op, idx, tag)? { + Some(_) => Ok(()), + None => Err(ETIMEDOUT), + } + } + + /// A firmware-driven suspend: tell the enclave, which answers, saves its + /// state and gates its own domain, and read and send nothing more until the + /// wake. + /// A failure leaves the enclave as it was if it can. + pub(crate) fn sleep_enter(&self) -> Result<()> { + match self.sleep_gate() { + // Gated already, or never coming back: nothing to tell it. + pm::SleepGate::Lost => return Ok(()), + pm::SleepGate::Open => {} + _ => return Err(EBUSY), + } + // Only an enclave that came up is told anything. The sleep goes on without one that didn't. + if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { + dev_warn!( + self.dev, + "the enclave did not finish coming up, so it isn't told of this sleep\n" + ); + return Ok(()); + } + + self.set_sleep_gate(pm::SleepGate::Draining); + let idle = control::wait_idle( + proto::CONTROL_TIMEOUT_MS, + 10, + || self.control.lock().requests_in_flight(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_err!(self.dev, "sleep refused: a request is still waiting for its reply\n"); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(EBUSY); + } + + if self.sleep_leave_for_commit() { + return Ok(()); + } + + if let Err(e) = pm::dart_prepare(&self.dev) { + dev_err!(self.dev, "sleep refused: its DART could not be saved ({:?})\n", e); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(e); + } + + match self.sleep_request() { + Ok(()) => { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.set_recv_enabled(false); + } + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Told); + dev_dbg!(self.dev, "the enclave answered the sleep request\n"); + Ok(()) + } + Err(e) => { + dev_err!(self.dev, "sleep refused: no answer to the sleep request ({:?})\n", e); + // A request that timed out may still have been taken. + self.set_sleep_gate(pm::SleepGate::Told); + self.sleep_exit(); + Err(e) + } + } + } + + /// Where the sleep driver takes it, the request goes out as its very last refusal, with + /// interrupts off: a gated enclave comes back only through a real sleep, so any earlier + /// refusal has to find it never told. Its answer is polled there, so the receive + /// interrupt is held from now on. The gate stays at draining until the wake. + fn sleep_leave_for_commit(&self) -> bool { + let request = proto::encode_control(&proto::op_sleep(), proto::TAG_SLEEP); + let left = match self.mbox.lock().as_ref() { + Some(mbox) => pm::leave_for_commit( + &self.dev, + mbox.as_raw(), + request, + proto::SLEEP_REPLY, + proto::SLEEP_REPLY_MASK, + ), + None => false, + }; + if left { + self.sleep_hold_rx(); + dev_dbg!(self.dev, "the sleep request is left for the sleep's last refusal\n"); + } + left + } + + /// Holds off the receive interrupt, whose handler reads the mailbox. + fn sleep_hold_rx(&self) { + if !self.sleep_rx_held.xchg(true, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.hold_rx(); + } + } + } + + fn sleep_release_rx(&self) { + if self.sleep_rx_held.xchg(false, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.release_rx(); + } + } + } + + /// Back from a sleep, or a suspend that was aborted. The enclave has kept + /// its state and takes no new endpoint setup. + /// + /// Reading the mailbox of a gated enclave can stall the CPU for good, so only + /// the power cycle of a real sleep ends one. A sleep refused after the request, + /// which only the request itself can do where the sleep driver sends it, leaves + /// the enclave asleep, and requests fail until the next real sleep or a reboot. + /// If it never gated it was never stopped, and a doorbell would reach an + /// endpoint: ask it something instead. + pub(crate) fn sleep_exit(&self) { + // Left to the sleep's last refusal: told only if it went out. + match pm::commit_finish(&self.dev) { + Some(true) => self.set_sleep_gate(pm::SleepGate::Told), + Some(false) => self.sleep_release_rx(), + None => {} + } + let gate = self.sleep_gate(); + if gate == pm::SleepGate::Draining { + self.set_sleep_gate(pm::SleepGate::Open); + return; + } + if !gate.needs_wake() { + return; + } + // SAFETY: only reads a flag. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if !real && gate == pm::SleepGate::Lost { + return; + } + let answered = if real { + self.sleep_kick() + } else if pm::wait_gated(&self.dev) { + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + dev_warn!( + self.dev, + "the enclave stays asleep until the next real sleep or a reboot\n" + ); + return; + } else { + dev_warn!(self.dev, "the enclave never gated its power domain\n"); + self.sleep_probe() + }; + match answered { + Ok(()) => { + self.set_sleep_gate(pm::SleepGate::Open); + // One line per sleep it was told of: nothing else says it came back. + dev_info!(self.dev, "the enclave is back from its sleep request\n"); + } + Err(e) => { + dev_err!(self.dev, "the enclave did not come back ({:?}); it stays unused\n", e); + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + } + } + } + + /// Starts a gated enclave and waits for the messages that say it's up. + fn sleep_kick(&self) -> Result<()> { + pm::clear_dev_disable(&self.dev)?; + pm::dart_restore(&self.dev); + self.sleep_release_rx(); + self.control.lock().wake_reset(); + match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, proto::WAKE_KICK)?; + } + None => return Err(ENODEV), + } + let mut remaining = time::msecs_to_jiffies(proto::CONTROL_TIMEOUT_MS); + let mut guard = self.control.lock(); + loop { + if guard.wake_complete() { + return Ok(()); + } + if remaining == 0 { + return Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + /// Whether an enclave that was told and never gated still answers. + fn sleep_probe(&self) -> Result<()> { + pm::dart_restore(&self.dev); + self.sleep_release_rx(); + let op = proto::op_nop(0); + let (idx, tag) = self.control.lock().alloc()?; + let msg = proto::encode_control(&op, tag); + let sent = match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, msg) + } + None => Err(ENODEV), + }; + if let Err(e) = sent { + self.control.lock().abandon(idx); + return Err(e); + } + match self.control_wait(&op, idx, tag)? { + Some(_) => Ok(()), + None => Err(ETIMEDOUT), + } } /// Puts `msg` in the mailbox once the enclave can see what the CPU wrote @@ -1347,6 +1626,16 @@ impl SepData { return Err(e); } + self.control_wait(op, idx, tag) + } + + /// Waits for the answer to the request sent under `idx` and `tag`. + fn control_wait( + &self, + op: &proto::ControlOp, + idx: usize, + tag: u8, + ) -> Result> { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { @@ -1484,7 +1773,7 @@ impl SepData { } let mbox = self.mbox.lock(); let mbox = mbox.as_ref().ok_or(ENODEV)?; - if !self.control.lock().entropy_may_send() { + if !self.sleep_gate().may_request() || !self.control.lock().entropy_may_send() { return Err(EAGAIN); } Self::ring(mbox, msg) @@ -1915,7 +2204,7 @@ impl SepData { ); } let msg = crate::xarm::encode_xarm_reply(reply); - let _ = self.send(msg); + let _ = self.send_reply(msg); } fn release_deferred_query(&self) { @@ -2627,6 +2916,8 @@ impl SepData { fn on_control(&self, msg: Message, f: proto::Fields) { if f.ty != proto::CONTROL_REPLY_TYPE { + self.control.lock().wake_note(f.ty); + self.control_wq.notify_all(); return; } @@ -2979,6 +3270,11 @@ impl platform::Driver for SepDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pm_shim.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(pm::sep_pm_ops) }) + } + fn probe<'bound>( pdev: &'bound platform::Device>, _info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index d58ae573fbbad..a3e0ca6e91bab 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -19,15 +19,24 @@ #include struct apple_sep_fv_new_file_key; +struct dev_pm_ops; void sep_cancel_work_sync(void *work); void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void (*event)(bool entering)); +int sep_pm_register(void *dev, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); +int sep_pm_wait_gated(void *dev, unsigned int timeout_ms); +int sep_pm_clear_dev_disable(void *dev); +int sep_pm_dart_prepare(void *dev); +void sep_pm_dart_restore(void *dev); +int sep_pm_commit_prepare(void *dev, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask); +int sep_pm_commit_finish(void *dev); +extern const struct dev_pm_ops sep_pm_ops; /* -- hwrng_shim.c ------------------------------------------------------- */ diff --git a/drivers/soc/apple/t8140-sleep.c b/drivers/soc/apple/t8140-sleep.c new file mode 100644 index 0000000000000..a24d395b8837d --- /dev/null +++ b/drivers/soc/apple/t8140-sleep.c @@ -0,0 +1,3083 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T8140 system sleep + * + * On T8140 a kernel can run directly at EL2 behind the CTXR execute window + * (Documentation/arch/arm64/ctxr-execmem.rst), with no monitor below it. So no + * PSCI answers a request to power a CPU off or to suspend the system, and the + * kernel does both itself. + * + * iBoot locks every core's reset vector to the boot object. A core that comes + * back from a power-off therefore restarts inside the boot object (aura), and + * that has to send it back here. The boot object opts in by publishing a mailbox + * in /chosen/aurora,s2r-mailbox, which also gives its spin-table CPUs the parking + * spin table (arch/arm64/kernel/cpu_ops.c). Without it this driver does nothing: + * + * - A secondary core goes back through its spin-table release address, which + * the boot object keeps, into secondary_holding_pen. + * - The boot core takes cpu_resume from the mailbox: EL2, MMU off. + * + * The register addresses are in the device tree. The ADT has every one of them, + * as /arm-io/pmgr's or /arm-io/wdt's reg entries. Register offsets and order not + * otherwise explained here were measured from a macOS sleep under aura. + * + * What the kernel does in suspend-to-RAM is what macOS does in its last 14 ms, + * with the order kept. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Must match aura's struct aura_s2r_mailbox. */ +#define T8140_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ + +struct t8140_s2r_mailbox { + u64 magic; /* aura, before the kernel starts */ + u64 boot_mpidr; /* aura: the only core that may take resume_pa */ + u64 resume_pa; /* kernel, armed before a commit; aura clears it */ + u64 resumes; /* aura: resume_pa taken this many times */ +}; + +/* PS word */ +#define PS_AUTO_ENABLE BIT(28) +#define PS_DEV_DISABLE BIT(10) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_OFF_TIMEOUT_US 5000 + +#define PS_DISP_CPU 0x10000 +#define PS_DISP_FE 0x318 +#define PS_DISP_SYS 0x280 +#define PS_DISPEXT0_CPU 0x328 +#define PS_DISPEXT0_FE 0x320 +#define PS_DISPEXT0_SYS 0x288 +#define PS_SIO_CPU 0x2a8 +#define PS_SEP 0xc00 +#define PS_ATC0_USB_AON 0x3e0 +#define PS_ATC0_COMMON 0x3e8 +#define PS_ATC0_USB 0x3f0 + +/* CPU start block, as aura's smp_start_cpu() uses it */ +#define CPU_START_STOP 0x0 +#define CPU_START_SYS 0x4 +#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) + +/* Power-gate page: reads back this value already. Purpose unknown. */ +#define PWRGATE_ON 0x80000000 +#define PWRGATE_WAKE_0 0x000 +#define PWRGATE_WAKE_1 0x040 +#define PWRGATE_CPM_0 0x060 +#define PWRGATE_CPM_1 0x068 +#define PWRGATE_DISP_0 0xc8c +#define PWRGATE_DISP_1 0xcd0 + +/* Three clock muxes macOS writes before the commit. */ +#define CLK_MUX_PARKED 0x80100000 +#define CLK_MUX_34 0x0 +#define CLK_MUX_38 0x4 +#define CLK_MUX_3C 0x8 + +/* Cluster control: only these three words, the next one is locked. */ +#define CLUSTER_E 0x0 +#define CLUSTER_P 0x4 +#define CLUSTER_AVE 0x8 +#define CLUSTER_ON 0xf +/* Set when a complex's up sequence ends, and cleared by writing 1. */ +#define CLUSTER_UP_DONE BIT(5) +#define CLUSTER_E_READY BIT(5) +#define CLUSTER_TIMEOUT_US 5000 + +/* CLLT: names from nothing. Bits measured cleared by macOS before the commit. */ +#define CLLT_0 0x30 +#define CLLT_1 0x34 +#define CLLT_2 0x3c +#define CLLT_0_CLEAR GENMASK(7, 0) +#define CLLT_1_CLEAR GENMASK(7, 0) +#define CLLT_2_CLEAR GENMASK(17, 16) + +/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ +#define WDT_WD0_COUNT 0x00 +#define WDT_WD0_CTL 0x0c +#define WDT_WD1_COUNT 0x10 +#define WDT_WD2_COUNT 0x20 +#define WDT_WD2_ALARM 0x28 +#define WDT_WD2_CTL 0x2c +#define WDT_TICKS_PER_S 24000000 +/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ +#define WDT_SLEEP_SECS 5 +/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ +#define WDT_SUSPEND_SECS 120 + +#define AON_AP_DEADLINE 0x08 + +/* Wake sources: a mask, so 0 enables. Status words are sticky. */ +#define WAKE_MASK(word) (0x08 + (word) * 4) +#define WAKE_STATUS(word) (0x28 + (word) * 4) +#define WAKE_WORDS 8 +/* Word 7 has 21 sources. Measured: macOS reads 0x001fef77 there. */ +#define WAKE_WORD7_ALL GENMASK(20, 0) +#define WAKE_SOURCES_MAX 32 + +/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ +#define SPMI_STATUS 0x00 +#define SPMI_STATUS_RX_EMPTY BIT(24) +#define SPMI_CMD 0x04 +#define SPMI_REPLY 0x08 +#define SPMI_NOTIFY_ENABLE 0x40 +#define SPMI_NOTIFY_ENABLE_SLEEP 0x2000001 +#define SPMI_NOTIFY_REPLY BIT(0) +#define SPMI_NOTIFY_ACK 0x80 +#define SPMI_CMD_ACTIVE BIT(15) +#define SPMI_OP_EXT_WRITE(n) ((n) - 1) +#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) +#define SPMI_OP_ZERO_WRITE 0x80 +#define SPMI_OP_SLEEP 0x11 +#define SPMI_OP_WAKEUP 0x13 +#define SPMI_POLL_US 20000 + +/* A port controller on SPMI: registers go through a page select, then a data window. */ +#define HPM_REG_SELECT 0x00 +#define HPM_REG_DATA 0x20 +#define HPM_REG_DATA_CMD 0xa0 +#define HPM_PAGE_CMD1 0x08 +#define HPM_PAGE_DATA1 0x09 +#define HPM_CMD_SSPS 0x53505353 +#define HPM_CMD_COLD_RESET 0x64696147 +#define HPM_SSPS_S3 3 +#define HPM_RESET_MS 1000 + +/* A PMU register write on the sleep's SPMI bus */ +#define SPMI_CMD_WRITE (BIT(15) | (0xe << 8) | 0x30) +#define PMU_BOOT_STAGE 0xf701 +#define PMU_BOOT_STAGE_SLEEP 0x50 + +/* Three of the memory controller's channels, and what macOS steps down on each. */ +#define MCC_CHANNELS 3 +#define MCC_ENABLES 0x4 +#define MCC_LANES 0x8 +#define MCC_LANES_MASK 0xc +#define MCC_LANE_STROBE(n) BIT(16 + 4 * (n)) +#define MCC_LANE(n) BIT(4 * (n)) + +/* Two words macOS zeroes just before the power gates. */ +#define PTD_A 0x0 +#define PTD_B 0x8 + +/* Perf-state words, each like DCS's. The SoC's is first. */ +#define CLK_PERF_SOC 0x0 +#define SOC_SLEEP_STATE 3 + +/* An always-on word that a live debug link holds up. macOS has it off. */ +#define PS_AON_DEBUG_SWITCH 0xb0 + +/* The PMC's voter agents, each with a word holding its floor for every rail. */ +#define PMC_VOTER_AGENTS 4 +#define PMC_VOTER_STRIDE 0x4000 + +/* DCS: the low nibble asks for a state, and [11:8] is the state it has reached. */ +#define DCS_STATE GENMASK(11, 8) +#define DCS_BUSY BIT(31) +#define DCS_SLEEP_STATE 4 +#define DCS_STEP_STATE 6 +#define DCS_TIMEOUT_US 5000 + +/* Word 0 of the wake block. macOS clears it at boot, and sets it only for a device's DDR use. */ +#define WAKE_SLP_DDR 0x00 + +/* + * AMCC error enables. Any error enabled and pending at the commit turns the + * sleep around within 20 ms with no error handler to service it. Linux has none. + * Measured: with macOS's handler hidden, as Linux's position is, 6 of 6 sleeps + * held with both of these off, and bit 25 alone still bounced. Their status + * stays set; only the enables matter. A deep sleep sets them again, so this has + * to be done before every commit. + */ +#define AMCC_WORD0_ENABLE 0x0 +#define AMCC_WORD1_ENABLE 0x8 +#define AMCC_WORD0_DSID_AGE_OUT BIT(25) +#define AMCC_WORD1_UNEXP_RT_HIT BIT(8) + +/* A coprocessor's ASC */ +#define ASC_CPU_CONTROL 0x44 +#define ASC_CPU_CONTROL_RUN BIT(4) +#define ASC_CPU_STATUS 0x48 + +#define PS_GPIO 0x120 +#define PS_ANS 0x278 +#define PS_APCIE_GP 0x258 +#define PS_APCIE_SYS_GP 0x2a0 +#define PS_APCIE_ST 0x310 +#define PS_APCIE_SYS_ST 0x3a0 +#define PS_APCIE_PHY_SW 0x3c0 + +/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ +#define APCIE_AXI_EN0 0x104 +#define APCIE_AXI_EN1 0x108 +#define APCIE_CLK_CTRL 0x0 +#define APCIE_CLK_BUSY BIT(31) +#define APCIE_CLK_RUN BIT(5) +#define APCIE_CLK_OFF BIT(1) +#define APCIE_CLK_OFF2 BIT(2) +#define APCIE_CLK_START 0x4 +#define APCIE_CLK_START_GO BIT(31) +#define APCIE_PHY_COMMON 0x4000 +#define APCIE_PHY_CTRL 0x8000 +#define APCIE_PHY_CTRL_CLK0REQ BIT(0) +#define APCIE_PHY_CTRL_CLK1REQ BIT(1) +#define APCIE_PHY_CTRL_CLK0ACK BIT(2) +#define APCIE_PHY_CTRL_CLK1ACK BIT(3) +#define APCIE_PHY_CTRL_RESET BIT(4) +#define APCIE_PHY_CTRL_UP BIT(27) +#define APCIE_PHY_RUN 0x8004 +#define APCIE_PHY_RUN_EN BIT(0) +#define APCIE_PHY_RUN_EN2 BIT(4) +#define APCIE_PHY_READY 0x8008 +#define APCIE_PHY_IP_PLL_CTL 0x90 +#define APCIE_PHY_IP_PLL_HOLD BIT(16) +#define APCIE_PORT_INTSTAT 0x100 +#define APCIE_PORT_INTMSK 0x104 +#define APCIE_PORT_INT_LINK GENMASK(7, 4) +#define APCIE_PORT_L2 0x88 +#define APCIE_PORT_L2_ENTER 0x11 +#define APCIE_PORT_L2_BUSY BIT(0) +#define APCIE_PORT_PERST_GUARD 0x13c +#define APCIE_PORT_LINKSTS 0x208 +#define APCIE_PORT_LINKSTS_UP BIT(0) +#define APCIE_PORT_APPCLK 0x800 +#define APCIE_PORT_APPCLK_EN BIT(0) +#define APCIE_PORT_APPCLK_CGDIS BIT(8) +#define APCIE_PORT_PERST 0x82c +#define APCIE_PORT_PHY_LINK BIT(31) +#define APCIE_PORT_PHY_EN0 BIT(9) +#define APCIE_PORT_PHY_EN1 BIT(10) +#define APCIE_ENGINE_CTRL 0x0 +#define APCIE_ENGINE_IRQ_MASK 0x8 +#define APCIE_ENGINE_STOP BIT(0) +#define APCIE_FABRIC_STOP 0x4 +#define APCIE_PORT_STOP_US 5000 +#define APCIE_RC_ZERO 0x04 +#define APCIE_RC_READY_START 0x50 +#define APCIE_RC_READY_CFG 0x54 +#define APCIE_RC_READY_STAT 0x58 +#define APCIE_POLL_US 50000 + +/* The coprocessors the last mile stops. Each driver idles its own first. */ +#define SLEEP_COPROCS 4 + +static struct t8140_sleep_coproc { + struct device *dev; + void __iomem *asc; + bool pcie_ans; + int (*last_check)(void *data); + int (*last_stop)(void *data); + void *data; + bool ready; + bool stopped; + u32 status; + /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ + bool first; + u32 ps; +} t8140_sleep_coprocs[SLEEP_COPROCS]; +static DEFINE_MUTEX(t8140_sleep_coprocs_lock); + +/* The enclave's sleep request, left to the last refusal. Under t8140_sleep_coprocs_lock. */ +static struct { + struct device *dev; + int (*tell)(void *data); + void *data; +} t8140_sleep_enclave; + +/* + * Written with bits 0 and 63 by every core right before its last WFI, measured + * from macOS. This is not CYC_OVRD, which is undefined on this SoC. + */ +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) + +/* The names of ADT pmgr/ap-wake-sources: for the log, and to tell the SMC's apart */ +static const struct { + u32 id; + const char *name; +} t8140_wake_names[] = { + { 0, "AOP.AICWakeupAP" }, { 5, "AOP.CPUWakeupAP" }, + { 42, "AOP.OutboxNotEmpty" }, { 62, "AOP.SPMI0.SW3" }, + { 66, "AOP.SPMI1.SW3" }, { 70, "ATC0_PMGR_USB_WAKEUP" }, + { 80, "MTP.DOCK.CHANNELS.AP0.IRQ" }, { 99, "MTP.OUTBOX.N.EMPTY_IRQ" }, + { 109, "NUB.APWakeupTime" }, { 110, "NUB.APWatchdog" }, + { 125, "NUB.GPIO.IRQ6" }, { 131, "NUB.SPMI0.SW3" }, + { 136, "NUB.SPMI1.SW3" }, { 140, "NUB.WakeupAP" }, + { 145, "SMC.CPUWakeupAP" }, { 165, "SMC.OutboxNotEmpty" }, + { 211, "AOP.TrustedOutboxNotEmpty" }, { 227, "AOP2.OutboxNotEmpty" }, + { 231, "AOP2.TrustedOutboxNotEmpty" }, { 236, "AOP2.CPUWakeupAP" }, +}; + +#define T8140_DISPLAY_COUNT 2 + +/* CPU clusters: 0 is E, 1 is P. ADT /arm-io/pmgr reg[5], [7], [9] and [14], [16], [18]. */ +#define T8140_CLUSTERS 2 + +/* cpm-dvfm */ +#define DVFM_CTL 0x178 +#define DVFM_WORD_0 0x140 +#define DVFM_WORD_1 0x160 +#define DVFM_WORD_2 0x190 +#define DVFM_WORD_3 0x198 +#define DVFM_ENABLE BIT_ULL(63) + +/* In macOS's order, on the way down. */ +static const u32 t8140_dvfm_words[] = { + DVFM_WORD_0, DVFM_WORD_1, DVFM_WORD_2, DVFM_WORD_3, +}; + +struct t8140_sleep { + struct device *dev; + void __iomem *ps; + void __iomem *pwrgate; + void __iomem *cpu_start; + void __iomem *cllt; + void __iomem *cluster; + void __iomem *commit; + void __iomem *wdt; + void __iomem *aon_misc; + void __iomem *wake; + void __iomem *amcc[4]; + void __iomem *pmc_dcp; + void __iomem *spmi; + void __iomem *pmc_voters; + void __iomem *dcs; + void __iomem *ps_aon; + void __iomem *spmi_a0; + void __iomem *hpm_spmi; + /* The port controller whose cold reset ends the debug link. */ + struct device_node *hpm_np; + u64 siq_cfg; + u32 hpm_sid; + int usb_down_ret; + /* Cut, or found already down, for this sleep. */ + bool usb_down_ran; + void __iomem *clk_mux; + void __iomem *mcc[MCC_CHANNELS]; + void __iomem *ptd; + void __iomem *clk_perf; + void __iomem *cluster_dcs; + u32 ps_on_saved[10]; + u32 ps_off_saved[64]; + u32 ps_aon_saved[4]; + u32 ps_aon_auto_saved[2]; + u32 spmi_saved[2]; + bool matched; + void __iomem *dvfm[T8140_CLUSTERS]; + struct t8140_s2r_mailbox *mbox; + + u32 wake_enable[WAKE_SOURCES_MAX]; + int wake_enable_count; + u32 wake_saved[WAKE_WORDS]; + /* The sources this sleep left enabled. */ + u32 wake_armed[WAKE_WORDS]; + + struct { + struct device_node *np; + void __iomem *asc; + bool bound; + } display[T8140_DISPLAY_COUNT]; + + /* Filled in for the report after a return. */ + u32 wake_status[WAKE_WORDS]; + u64 dvfm_ctl[T8140_CLUSTERS]; + u64 dvfm_saved[T8140_CLUSTERS][ARRAY_SIZE(t8140_dvfm_words)]; + bool dvfm_parked; + u64 resumes_before; + u64 resumes_after; + int cluster_e_ret; + int cluster_ave_ret; + int cluster_ave_commit_ret; + unsigned int ps_rewoken; + unsigned int ps_rewake_timeouts; + u64 cntkctl; + int cpus_stop_ret; + int ans_off_ret; + int pcie_ans_on_ret; + void __iomem *apcie_axi; + void __iomem *apcie_cio3pll; + void __iomem *apcie_clkgen; + void __iomem *apcie_phy; + void __iomem *apcie_rc; + bool apcie_port_stopped; + /* A real sleep left the radio's port down, until its host sets it up again */ + bool apcie_port_down; + void __iomem *apcie_port; + void __iomem *apcie_port_phy; + void __iomem *apcie_engine; + void __iomem *apcie_fabric; + struct apple_tunable *apcie_axi_tunable; + struct apple_tunable *apcie_cio3pll_tunable; + struct apple_tunable *apcie_clkgen_tunable; + struct apple_tunable *apcie_phy_tunable; + void __iomem *apcie_phy_ip; + struct apple_tunable *apcie_phy_ip_pll; + struct apple_tunable *apcie_phy_ip_lanes; + bool pcie_ans_stopped; + /* The words t8140_sleep_lower_stray() lowered, one bit each */ + u8 stray_lowered; + /* What must be bound for a sleep, and what must have stopped by the last step */ + struct t8140_sleep_gate { + const char *name; + /* Compared only, never dereferenced: nodes outlive the gate. */ + const struct device_node *np; + void __iomem *asc; + bool bound; + bool must_stop; + bool sep; + u32 ps; + } gate[16]; + unsigned int gate_count; + /* The AP pins, which a deep sleep resets */ + void __iomem *pins; + u32 *pins_saved; + unsigned long *pins_skip; + unsigned int npins; + unsigned int pin_irq_groups; + bool pins_saved_valid; + bool pins_lost; + /* The radio port's PERST# among them, or -1 */ + int radio_perst; + bool radio_perst_low; + bool returned; + /* This sleep's wake came from the SMC's sources alone. */ + bool smc_woke_alone; +}; + +static struct t8140_sleep *t8140_sleep; + +static struct t8140_sleep_coproc *t8140_sleep_coproc_find(struct device *dev) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (t8140_sleep_coprocs[i].dev == dev) + return &t8140_sleep_coprocs[i]; + + return NULL; +} + +/* A registered coprocessor its driver has readied, whose run bit the last step clears. */ +static bool t8140_sleep_coproc_ready_node(const struct device_node *np) +{ + int i; + + for (i = 0; np && i < SLEEP_COPROCS; i++) + if (t8140_sleep_coprocs[i].dev && t8140_sleep_coprocs[i].ready && + t8140_sleep_coprocs[i].dev->of_node == np) + return true; + return false; +} + +static bool t8140_usb_asleep; + +void apple_sleep_usb_set_asleep(bool asleep) +{ + WRITE_ONCE(t8140_usb_asleep, asleep); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_asleep); + +static unsigned long t8140_usb_wake_ports; + +void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ + WRITE_ONCE(t8140_usb_wake_ports, ports); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); + +bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return port < BITS_PER_LONG && (READ_ONCE(t8140_usb_wake_ports) & BIT(port)); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); + +static bool t8140_debug_link_cut; + +/* Only the debug port's controller took the cold reset. */ +static bool t8140_debug_port_is(struct device *dev) +{ + return t8140_sleep && t8140_sleep->hpm_np && dev->of_node == t8140_sleep->hpm_np; +} + +bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return t8140_debug_port_is(dev) && xchg(&t8140_debug_link_cut, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_was_cut); + +bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return t8140_debug_port_is(dev) && READ_ONCE(t8140_debug_link_cut); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_peek); + +/* The radio's host set its port up again after a real sleep: the next commit may stop it. */ +void apple_sleep_radio_port_up(void) +{ + if (t8140_sleep) + WRITE_ONCE(t8140_sleep->apcie_port_down, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_port_up); + +#define PIN_REG(n) (4 * (n)) +#define PIN_DATA BIT(0) +#define PIN_LOCK BIT(21) +#define PIN_CTL 0xc48 +#define PIN_IRQ(g, w) (0x800 + 0x40 * (g) + 4 * (w)) + +/* Before any device suspends, with the pins' domain on. Skipped pins aren't read. */ +static void t8140_pins_save(struct t8140_sleep *s) +{ + unsigned int pin; + + s->pins_lost = false; + s->pins_saved_valid = false; + if (!s->pins) + return; + for (pin = 0; pin < s->npins; pin++) + if (!test_bit(pin, s->pins_skip)) + s->pins_saved[pin] = readl(s->pins + PIN_REG(pin)); + s->pins_saved_valid = true; +} + +/* + * A deep sleep resets every pin, and the pin controller's driver puts back only the + * ones it configured. macOS puts all of them back right before ANS's run bit, and + * ANS's link doesn't come up without that: 1 written at 0xc48, every interrupt + * group's pending bits cleared, then each pin written back as it reads, then as it + * was, its lock bit last. + */ +void apple_sleep_restore_pins(void) +{ + struct t8140_sleep *s = t8140_sleep; + unsigned int pin, grp, w, changed = 0; + + if (!s || !s->pins_lost) + return; + s->pins_lost = false; + + writel(1, s->pins + PIN_CTL); + for (grp = 0; grp < s->pin_irq_groups; grp++) + for (w = 0; w < DIV_ROUND_UP(s->npins, 32); w++) + writel(U32_MAX, s->pins + PIN_IRQ(grp, w)); + for (pin = 0; pin < s->npins; pin++) { + void __iomem *reg = s->pins + PIN_REG(pin); + u32 want = s->pins_saved[pin], cur; + + if (test_bit(pin, s->pins_skip)) + continue; + cur = readl(reg); + changed += cur != want; + writel(cur, reg); + if (want & PIN_LOCK) + writel(want & ~PIN_LOCK, reg); + writel(want, reg); + } + dev_info(s->dev, "%u of %u AP pins changed by the sleep, all put back\n", changed, + s->npins - bitmap_weight(s->pins_skip, s->npins)); +} +EXPORT_SYMBOL_GPL(apple_sleep_restore_pins); + +/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data) +{ + struct t8140_sleep_coproc *c; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&t8140_sleep_coprocs_lock); + if (t8140_sleep_coproc_find(dev)) + return -EEXIST; + c = t8140_sleep_coproc_find(NULL); + if (!c) + return -ENOSPC; + c->asc = asc; + c->pcie_ans = pcie_ans; + c->last_check = NULL; + c->last_stop = last_stop; + c->data = data; + c->ready = false; + c->stopped = false; + c->first = false; + c->ps = 0; + c->dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); + +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data) +{ + struct t8140_sleep_coproc *c; + struct device_node *np; + u32 ps = 0; + int ret; + + if (!last_stop) + return -EINVAL; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np) { + of_property_read_u32(np, "reg", &ps); + of_node_put(np); + } + ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); + if (ret) + return ret; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (c) { + c->first = true; + c->ps = ps; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); + +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (!c) + return -ENOENT; + c->last_check = last_check; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); + +int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), void *data) +{ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&t8140_sleep_coprocs_lock); + if (t8140_sleep_enclave.dev) + return -EBUSY; + t8140_sleep_enclave.tell = tell; + t8140_sleep_enclave.data = data; + t8140_sleep_enclave.dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); + +void apple_sleep_enclave_unregister(struct device *dev) +{ + guard(mutex)(&t8140_sleep_coprocs_lock); + if (t8140_sleep_enclave.dev == dev) + memset(&t8140_sleep_enclave, 0, sizeof(t8140_sleep_enclave)); +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); + +void apple_sleep_coproc_unregister(struct device *dev) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (c) + memset(c, 0, sizeof(*c)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); + +void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + if (!c) + return; + c->ready = ready; + c->stopped = false; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); + +bool apple_sleep_coproc_stopped(struct device *dev) +{ + struct t8140_sleep_coproc *c; + + guard(mutex)(&t8140_sleep_coprocs_lock); + c = t8140_sleep_coproc_find(dev); + return c && c->stopped; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); + +/* + * The last thing that can fail on the way to the commit: no CPU but this one is up and + * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only + * through a real sleep, so every refusal has to be behind this, and nothing after it may + * go back. In the order measured on macOS, on ANS itself: whatever its driver does last, + * status and control read, then 0 written twice. + */ +static void t8140_sleep_coproc_run_off(struct t8140_sleep_coproc *c) +{ + /* Its own last stop cleared its run bits. */ + if (!c->asc) { + c->stopped = true; + return; + } + c->status = readl(c->asc + ASC_CPU_STATUS); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + c->stopped = true; +} + +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +static bool t8140_sleep_coproc_lowers(u32 ps) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (t8140_sleep_coprocs[i].dev && t8140_sleep_coprocs[i].ready && + t8140_sleep_coprocs[i].first && t8140_sleep_coprocs[i].ps == ps) + return true; + return false; +} + +/* The last refusals, each coprocessor's own: nothing has stopped yet. */ +static int t8140_sleep_coprocs_check(struct t8140_sleep *s) +{ + int i, ret; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->stopped || !c->last_check) + continue; + ret = c->last_check(c->data); + if (ret) { + dev_err(s->dev, "refused: %s: %d\n", dev_name(c->dev), ret); + return ret; + } + } + + return 0; +} + +/* + * The other cores stop at the commit, and one left awake there keeps the engine waiting + * until WD2 resets the SoC. Each answers here, before the display goes off, and waits for the + * stop or a refusal. + */ +static int t8140_sleep_cpus_hold(struct t8140_sleep *s) +{ + int ret = spin_table_hold_parked(); + + if (ret) + dev_err(s->dev, "refused: a CPU didn't answer the hold: %d\n", ret); + + return ret; +} + +/* Last of all the refusals: an enclave that gated comes back only through a real sleep. */ +static int t8140_sleep_enclave_tell(struct t8140_sleep *s) +{ + int ret; + + if (!t8140_sleep_enclave.tell) + return 0; + ret = t8140_sleep_enclave.tell(t8140_sleep_enclave.data); + if (ret) + dev_err(s->dev, "refused: the enclave didn't take the sleep request: %d\n", ret); + + return ret; +} + +/* + * In two passes, after the last refusal: from the first stop only a real sleep restarts the + * GPU, so a failure here goes on to the commit. Those registered first stop whole first, + * before macOS's off words drop under them: the GPU withdraws its PMP request there, and the + * PMP has to see that before its own stop. The rest after the match. + */ +static void t8140_sleep_coprocs_stop(struct t8140_sleep *s, bool first) +{ + int i; + + if (first) + s->pcie_ans_stopped = false; + for (i = 0; i < SLEEP_COPROCS; i++) { + struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != first) + continue; + s->pcie_ans_stopped |= c->pcie_ans; + if (c->stopped) + continue; + if (c->last_stop) + c->last_stop(c->data); + t8140_sleep_coproc_run_off(c); + } + dsb(sy); +} + +/* MPIDR is cluster 0 cores 0-3 at 0.0.n and cluster 1 cores 0-1 at 1.1.n. */ +static unsigned int t8140_cpu_cluster(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 2); +} + +static unsigned int t8140_cpu_core(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 0); +} + +static u32 t8140_cpu_start_bit(u64 mpidr) +{ + return BIT(4 * t8140_cpu_cluster(mpidr) + t8140_cpu_core(mpidr)); +} + +static void t8140_deep_wfi_arm(void) +{ + /* A timer left running ends the WFI, and costs power until reset. */ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_ENABLE); +} + +/* A core's last instructions: nothing brings it back but a reset. */ +static void __noreturn t8140_deep_wfi(void) +{ + t8140_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void t8140_cpu_stop(u64 mpidr) +{ + writel(t8140_cpu_start_bit(mpidr), t8140_sleep->cpu_start + CPU_START_STOP); + dsb(sy); +} + +static void t8140_cpu_off(u64 mpidr) +{ + t8140_deep_wfi(); +} + +static int t8140_cpu_on(u64 mpidr) +{ + struct t8140_sleep *s = t8140_sleep; + unsigned int cluster = t8140_cpu_cluster(mpidr); + + writel(t8140_cpu_start_bit(mpidr), s->cpu_start + CPU_START_SYS); + writel(BIT(t8140_cpu_core(mpidr)), s->cpu_start + CPU_START_CLUSTER(cluster)); + + return 0; +} + +static const struct spin_table_power_ops t8140_cpu_power_ops = { + .cpu_stop = t8140_cpu_stop, + .cpu_off = t8140_cpu_off, + .cpu_on = t8140_cpu_on, +}; + +/* + * As macOS lowers a domain: AUTO_ENABLE off first if it's set, then target 0. + * Writing the flags clears them. Values measured, for instance ANS 0x1f0020ff, + * 0x0f0020ff, 0x0f0023f0. + */ +static u32 t8140_ps_lower_start(struct t8140_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + + return val; +} + +static void t8140_ps_lower_finish(struct t8140_sleep *s, u32 offset, u32 val) +{ + writel((val & ~PS_TARGET) | PS_FLAGS, s->ps + offset); +} + +static void t8140_ps_lower(struct t8140_sleep *s, u32 offset) +{ + t8140_ps_lower_finish(s, offset, t8140_ps_lower_start(s, offset)); +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Measured + * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. + */ +static int t8140_ps_raise_start(struct t8140_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 now; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + + return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, + PS_OFF_TIMEOUT_US); +} + +static void t8140_ps_raise_finish(struct t8140_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, + reg); +} + +static int t8140_ps_raise(struct t8140_sleep *s, u32 offset) +{ + int ret = t8140_ps_raise_start(s, offset); + + t8140_ps_raise_finish(s, offset); + + return ret; +} + +static int t8140_ps_wait_off(struct t8140_sleep *s, u32 offset) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, !FIELD_GET(PS_ACTUAL, val), + 10, PS_OFF_TIMEOUT_US); +} + +static bool t8140_ps_on(struct t8140_sleep *s, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)); +} + +/* + * After a refused sleep, the sleep engine's own restart of a firmware's CPU whose domain shares + * a power switch and can't gate alone, as SIO's: each of the device's own domains reset through + * target 4 and 0, released with the domain off, then raised again, its DARTs' state kept. The + * caller has cleared the run bit, and starts the firmware after. + */ +#define PS_BUSY BIT(11) +#define PS_RESET BIT(31) +#define PS_ENGINE_RESET_4 0x8f000244 +#define PS_ENGINE_RESET_0 0x80000000 +#define ASC_CPU_STATUS_READY BIT(1) +#define ENGINE_RESET_PS 4 +#define ENGINE_RESET_DARTS 2 + +static int t8140_engine_reset_wait(struct t8140_sleep *s, u32 offset, u32 actual) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, + FIELD_GET(PS_ACTUAL, val) == actual && !(val & PS_BUSY), + 10, PS_OFF_TIMEOUT_US); +} + +/* A domain of the PMGR block mapped here as "ps", and its offset in it. */ +static bool t8140_ps_domain(struct t8140_sleep *s, struct device_node *pd, u32 *offset) +{ + int idx = of_property_match_string(s->dev->of_node, "reg-names", "ps"); + struct device_node *parent = of_get_parent(pd); + struct resource ps, res; + bool ours; + + ours = idx >= 0 && parent && !of_address_to_resource(s->dev->of_node, idx, &ps) && + !of_address_to_resource(parent, 0, &res) && res.start == ps.start && + !of_property_read_u32(pd, "reg", offset); + of_node_put(parent); + + return ours; +} + +/* The @index-th DART in @np's iommus, a reference held, or NULL past the last. */ +static struct device *t8140_iommus_dart(struct device_node *np, int index) +{ + struct of_phandle_args args; + struct platform_device *pdev; + + if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, &args)) + return NULL; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + + return pdev ? &pdev->dev : NULL; +} + +int apple_sleep_engine_reset(struct device *dev) +{ + struct t8140_sleep *s = t8140_sleep; + struct device *darts[ENGINE_RESET_DARTS] = {}; + u32 ps[ENGINE_RESET_PS], saved[ENGINE_RESET_PS], val; + struct of_phandle_iterator it; + void __iomem *asc; + int i, n = 0, idx, err, ret = 0; + + if (!s) + return -ENODEV; + of_for_each_phandle(&it, err, dev->of_node, "power-domains", NULL, 0) { + if (!t8140_ps_domain(s, it.node, &val)) + continue; + if (n == ENGINE_RESET_PS) { + of_node_put(it.node); + return -E2BIG; + } + ps[n++] = val; + } + if (!n) + return -ENODEV; + + idx = of_property_match_string(dev->of_node, "reg-names", "coproc"); + if (idx < 0) + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return -ENODEV; + + for (i = 0; i < ENGINE_RESET_DARTS; i++) { + darts[i] = t8140_iommus_dart(dev->of_node, i); + if (!darts[i]) + break; + apple_dart_state_save(darts[i]); + } + + for (i = 0; i < n && !ret; i++) { + saved[i] = readl(s->ps + ps[i]); + writel(PS_ENGINE_RESET_4, s->ps + ps[i]); + ret = t8140_engine_reset_wait(s, ps[i], 4); + if (!ret) { + writel(PS_ENGINE_RESET_0, s->ps + ps[i]); + ret = t8140_engine_reset_wait(s, ps[i], 0); + } + } + if (!ret) { + usleep_range(1000, 2000); + for (i = 0; i < n; i++) + writel(0, s->ps + ps[i]); + for (i = n - 1; i >= 0 && !ret; i--) { + writel(PS_TARGET, s->ps + ps[i]); + ret = t8140_engine_reset_wait(s, ps[i], FIELD_MAX(PS_ACTUAL)); + if (!ret) + writel(saved[i] & ~(PS_FLAGS | PS_RESET | PS_ACTUAL), + s->ps + ps[i]); + } + } + if (!ret) + ret = readl_poll_timeout(asc + ASC_CPU_STATUS, val, val & ASC_CPU_STATUS_READY, 10, + 100 * USEC_PER_MSEC); + + for (i = 0; i < ENGINE_RESET_DARTS && darts[i]; i++) { + apple_dart_state_restore(darts[i]); + put_device(darts[i]); + } + iounmap(asc); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_sleep_engine_reset); + +/* The order macOS does it in, across clusters. */ +static void t8140_dvfm_park(struct t8140_sleep *s) +{ + int cluster, word; + + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) { + s->dvfm_ctl[cluster] = readq(s->dvfm[cluster] + DVFM_CTL); + for (word = 0; word < ARRAY_SIZE(t8140_dvfm_words); word++) + s->dvfm_saved[cluster][word] = + readq(s->dvfm[cluster] + t8140_dvfm_words[word]); + } + + writeq(0, s->dvfm[0] + DVFM_CTL); + writeq(0, s->dvfm[1] + DVFM_CTL); + for (word = 0; word < 2; word++) + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + t8140_dvfm_words[word]); + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + t8140_dvfm_words[word]); + s->dvfm_parked = true; +} + +static void t8140_dvfm_unpark(struct t8140_sleep *s) +{ + int cluster, word; + + if (!s->dvfm_parked) + return; + s->dvfm_parked = false; + + writeq(s->dvfm_ctl[0], s->dvfm[0] + DVFM_CTL); + writeq(s->dvfm_ctl[1], s->dvfm[1] + DVFM_CTL); + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + t8140_dvfm_words[word]); + for (word = 1; word >= 0; word--) + for (cluster = 0; cluster < T8140_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + t8140_dvfm_words[word]); +} + +/* + * What XNU polls just before its complex words, and panics on if one is on, besides the + * display's, ANS's and ATC0's, which the commit lowers itself. Every device has suspended by + * the last step, so one still on has no user: it's lowered there, and a sleep with one that + * won't go off is refused. They come back on after the sleep or a refusal, for genpd's books. + */ +static const struct { + const char *name; + u32 offset; +} t8140_stray_words[] = { + { "AVD_SYS", 0x270 }, { "VENC0_SYS", 0x240 }, { "VENC1_SYS", 0x248 }, + { "GFX", 0x1f0 }, { "JPG", 0x230 }, +}; + +static void t8140_sleep_stray_restore(struct t8140_sleep *s) +{ + int i; + + for (i = ARRAY_SIZE(t8140_stray_words) - 1; i >= 0; i--) + if (s->stray_lowered & BIT(i)) + t8140_ps_raise(s, t8140_stray_words[i].offset); + s->stray_lowered = 0; +} + +static int t8140_sleep_lower_stray(struct t8140_sleep *s) +{ + int i, ret = 0; + + s->stray_lowered = 0; + for (i = 0; i < ARRAY_SIZE(t8140_stray_words); i++) { + u32 off = t8140_stray_words[i].offset; + + if (!t8140_ps_on(s, off)) + continue; + /* Its driver's own last stop lowers it, at the commit. */ + if (t8140_sleep_coproc_lowers(off)) { + dev_dbg(s->dev, "%s: left on for its driver's last stop\n", + t8140_stray_words[i].name); + continue; + } + t8140_ps_lower(s, off); + s->stray_lowered |= BIT(i); + if (t8140_ps_wait_off(s, off)) { + dev_err(s->dev, "refused: %s won't go off: %#x\n", + t8140_stray_words[i].name, readl(s->ps + off)); + ret = -EBUSY; + } else { + dev_info(s->dev, "%s was on at the last step, lowered\n", + t8140_stray_words[i].name); + } + } + if (ret) + t8140_sleep_stray_restore(s); + + return ret; +} + +/* + * T8140's coprocessors that must be bound for a sleep, the ones that must also have stopped + * by the last step (a commit with the PMP and SIO still running reset the SoC 5.5 s later), + * and the domains macOS has off by then. Taken in process context, since the checks run with + * interrupts off. A driver whose stop failed has already failed the suspend, naming itself, + * so what's left is whether it's bound at all, and what the hardware shows. @ps is the CPU's + * own domain where a driver may power it off, as the DCPs' at DPMS off. + */ +static const struct { + const char *path; + bool must_stop; + u32 ps; +} t8140_gate_coprocs[] = { + { "/soc/dcp@412e00000", true, PS_DISP_CPU }, + { "/soc/dcp@416e00000", true, PS_DISPEXT0_CPU }, + { "/soc/pmp@300500000", true }, { "/soc/dma-controller@385e00000", true }, + { "/soc/nvme@3cdcc0000" }, { "/soc/mtp@314600000" }, + { "/soc/aop@310c00000" }, { "/soc/smc@30c600000" }, + { "/soc/sep@282600000" }, { "/soc/gpu@482600000" }, +}; + +static const struct { + const char *name; + u32 offset; +} t8140_gate_off[] = { + { "ane_sys", 0x290 }, { "isp_sys", 0x268 }, { "prores", 0x250 }, + { "avd_sys", 0x270 }, { "jpg", 0x230 }, { "venc0_sys", 0x240 }, +}; + +static void t8140_sleep_gate_prepare(struct t8140_sleep *s) +{ + int i; + + s->gate_count = 0; + for (i = 0; i < ARRAY_SIZE(t8140_gate_coprocs) && s->gate_count < ARRAY_SIZE(s->gate); + i++) { + struct device_node *c = of_find_node_by_path(t8140_gate_coprocs[i].path); + struct platform_device *pdev = c ? of_find_device_by_node(c) : NULL; + struct t8140_sleep_gate *g; + + if (!c) { + dev_warn(s->dev, "%s isn't in this tree\n", t8140_gate_coprocs[i].path); + continue; + } + g = &s->gate[s->gate_count++]; + g->name = c->name; + g->np = c; + g->bound = pdev && device_is_bound(&pdev->dev); + g->sep = of_node_name_eq(c, "sep"); + g->must_stop = t8140_gate_coprocs[i].must_stop; + g->ps = t8140_gate_coprocs[i].ps; + if (g->must_stop && !g->asc) { + int idx = of_property_match_string(c, "reg-names", "coproc"); + + if (idx < 0) + idx = of_property_match_string(c, "reg-names", "asc"); + if (idx >= 0) + g->asc = of_iomap(c, idx); + } + if (pdev) + put_device(&pdev->dev); + of_node_put(c); + } +} + +/* + * Names everything wrong, so one boot shows the whole list. A coprocessor that must stop and + * still runs refuses the sleep; the rest only warns, as a real sleep is the only full test. + */ +static bool t8140_sleep_gate_failed(struct t8140_sleep *s) +{ + int i, bad = 0, running = 0; + + for (i = 0; i < s->gate_count; i++) { + struct t8140_sleep_gate *g = &s->gate[i]; + + if (!g->bound) { + dev_err(s->dev, "%s isn't bound\n", g->name); + bad++; + /* Nobody told it of the sleep; a commit with it running never returns. */ + if (g->sep && t8140_ps_on(s, PS_SEP)) { + dev_err(s->dev, "sep runs with no driver bound: %#x\n", + readl(s->ps + PS_SEP)); + running++; + } + continue; + } + /* A CPU whose domain is off isn't running, and its registers aren't there. */ + if (g->ps && !t8140_ps_on(s, g->ps)) + continue; + if (g->must_stop && (!g->asc || + ((readl(g->asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN) && + !t8140_sleep_coproc_ready_node(g->np)))) { + dev_err(s->dev, "%s didn't stop: control %#x status %#x\n", g->name, + g->asc ? readl(g->asc + ASC_CPU_CONTROL) : 0, + g->asc ? readl(g->asc + ASC_CPU_STATUS) : 0); + bad++; + running++; + } + /* Told later, as the last refusal: see t8140_sleep_enclave_tell(). */ + if (g->sep && t8140_ps_on(s, PS_SEP) && !t8140_sleep_enclave.tell) { + dev_err(s->dev, "sep hasn't gated itself: %#x\n", readl(s->ps + PS_SEP)); + bad++; + running++; + } + } + + for (i = 0; i < ARRAY_SIZE(t8140_gate_off); i++) { + if (t8140_ps_on(s, t8140_gate_off[i].offset)) { + dev_err(s->dev, "%s is on: %#x\n", t8140_gate_off[i].name, + readl(s->ps + t8140_gate_off[i].offset)); + bad++; + } + } + + if (running) { + dev_err(s->dev, "refused: %d coprocessors that must stop or gate are still running\n", + running); + return true; + } + if (bad) + dev_warn(s->dev, "going ahead with %d problems\n", bad); + else + dev_info(s->dev, "%u coprocessors up and stopped\n", s->gate_count); + + return false; +} + +static void t8140_wake_unmask(u32 *val, u32 id) +{ + if (id / 32 < WAKE_WORDS) + val[id / 32] &= ~BIT(id % 32); +} + +/* + * Mask every source but the ones asked for. These words read all zero until + * something writes them, which leaves every source enabled, listed in the ADT or + * not. macOS reads all ones outside the ADT's ap-wake-sources (measured). The AOP's + * source 0 is masked with the rest: that held under macOS with its handler hidden, + * as did leaving it enabled with the AMCC enables off. The two together are unmeasured. + */ +static void t8140_wake_masks(struct t8140_sleep *s, u32 *val) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + val[i] = i == WAKE_WORDS - 1 ? WAKE_WORD7_ALL : U32_MAX; + for (i = 0; i < s->wake_enable_count; i++) + t8140_wake_unmask(val, s->wake_enable[i]); +} + +/* The last checks, then the stray words lowered: what has stopped may not return until a wake. */ +static int t8140_sleep_refuse(struct t8140_sleep *s) +{ + bool refused = false, idled = false; + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct t8140_sleep_coproc *c = &t8140_sleep_coprocs[i]; + + if (!c->dev) + continue; + if (c->ready) { + idled |= c->pcie_ans; + } else { + /* Nothing may lower a coprocessor's domain under its driver. */ + dev_err(s->dev, "refused: %s is running and its driver has not quiesced it\n", + dev_name(c->dev)); + refused = true; + } + } + + if (num_online_cpus() != 1 || + (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { + dev_err(s->dev, "refused: wrong CPU, or other CPUs online\n"); + refused = true; + } + + /* Only a driver takes the display to where its domain can be lowered. */ + if (!s->display[0].bound) { + dev_err(s->dev, "refused: no driver bound to the display coprocessor\n"); + refused = true; + } + + /* ANS's link needs these after a real power loss. */ + if (idled && !s->apcie_phy_tunable) { + dev_err(s->dev, "refused: no tunables to bring ANS's PCIe block back\n"); + refused = true; + } + + /* Only ANS's driver can stop it, and it doesn't come back without a real sleep. */ + if (!idled && (t8140_ps_on(s, PS_ANS) || t8140_ps_on(s, PS_APCIE_SYS_GP))) { + dev_err(s->dev, "refused: ANS or PCIe is on with no driver to stop it\n"); + refused = true; + } + + /* The engine waits for DISP_CPU with no timeout, so this must never slip. */ + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + u32 ps = i ? PS_DISPEXT0_CPU : PS_DISP_CPU; + + if (s->display[i].bound && t8140_ps_on(s, ps) && + (readl(s->display[i].asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + dev_err(s->dev, "refused: a bound display coprocessor is still running\n"); + refused = true; + } + } + + if (t8140_sleep_gate_failed(s)) + refused = true; + + if (refused) + return -EBUSY; + + return t8140_sleep_lower_stray(s); +} + +static void t8140_wake_arm(struct t8140_sleep *s) +{ + u32 val[WAKE_WORDS]; + int i; + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_saved[i] = readl(s->wake + WAKE_MASK(i)); + t8140_wake_masks(s, val); + for (i = 0; i < WAKE_WORDS; i++) { + writel(val[i], s->wake + WAKE_MASK(i)); + s->wake_armed[i] = ~val[i]; + } +} + +static void t8140_wake_restore(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + writel(s->wake_saved[i], s->wake + WAKE_MASK(i)); +} + +/* + * The sleep engine waits on DISP_CPU with no timeout, so the display domains + * must be off. A coprocessor's driver stops its firmware first, and nothing here + * talks to the firmware. Otherwise no driver is bound, and the bootloader left the + * firmware running: on the DCP, stop its CPU cold. That's unproven. The coprocessor + * restarts from its reset vector after any sleep, and the engine itself clears the + * DCP's run bit on the way down. + * + * Domain order and values as macOS's: DCPEXT then DCP, CPU, front end, system. + * The rest of the display's domains go off with these, by nothing Linux or macOS + * writes. The coprocessors' secure mailboxes are not used here. Whether that + * holds is open. + */ +static int t8140_display_off(struct t8140_sleep *s) +{ + static const u32 domains[T8140_DISPLAY_COUNT][3] = { + { PS_DISP_CPU, PS_DISP_FE, PS_DISP_SYS }, + { PS_DISPEXT0_CPU, PS_DISPEXT0_FE, PS_DISPEXT0_SYS }, + }; + int i, ret = 0; + + for (i = T8140_DISPLAY_COUNT - 1; i >= 0; i--) { + void __iomem *asc = s->display[i].asc; + + /* The engine doesn't wait for DISPEXT0. */ + if (i == 1 && !s->display[i].bound) + continue; + + if (t8140_ps_on(s, domains[i][0]) && asc && + (readl(asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + writel(0, asc + ASC_CPU_CONTROL); + writel(0, asc + ASC_CPU_CONTROL); + } + + /* Reads as a count of the DCPs powered, and macOS ends at 0. */ + writel(i, s->pmc_dcp); + if (i == 0) { + if (readl(s->ps + PS_DISP_CPU) & PS_AUTO_ENABLE) + writel(readl(s->ps + PS_DISP_CPU) & + ~(PS_AUTO_ENABLE | PS_FLAGS), s->ps + PS_DISP_CPU); + writel(3, s->pwrgate + PWRGATE_DISP_0); + writel(0, s->pwrgate + PWRGATE_DISP_1); + } + t8140_ps_lower(s, domains[i][0]); + t8140_ps_lower(s, domains[i][1]); + t8140_ps_lower(s, domains[i][2]); + } + + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + int err; + + if (i == 1 && !s->display[i].bound) + continue; + + err = t8140_ps_wait_off(s, domains[i][0]); + + if (err) { + dev_err(s->dev, "refused: display domain %#x stayed on: %#x\n", + domains[i][0], readl(s->ps + domains[i][0])); + ret = err; + } + } + + return ret; +} + +/* One command on the port controller's bus, polled: no other CPU, and interrupts are off. */ +static int t8140_hpm_cmd(struct t8140_sleep *s, u32 op, u32 extra, const u32 *in, u32 *out) +{ + void __iomem *b = s->hpm_spmi; + u32 status, reply; + int ret; + + while (!(readl(b + SPMI_STATUS) & SPMI_STATUS_RX_EMPTY)) + readl(b + SPMI_REPLY); + + writel(extra << 16 | SPMI_CMD_ACTIVE | s->hpm_sid << 8 | op, b + SPMI_CMD); + if (in) + writel(*in, b + SPMI_CMD); + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, SPMI_POLL_US); + if (ret) + return ret; + reply = readl(b + SPMI_REPLY); + writel(1, b + SPMI_NOTIFY_ACK); + if (out) { + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, + SPMI_POLL_US); + if (ret) + return ret; + *out = readl(b + SPMI_REPLY); + } + + /* A zero write's reply carries the value written. */ + return op == SPMI_OP_ZERO_WRITE || (reply & 0xff) == op ? 0 : -EIO; +} + +static int t8140_hpm_select(struct t8140_sleep *s, u32 page) +{ + u32 val = ~0U; + int i, ret; + + ret = t8140_hpm_cmd(s, SPMI_OP_ZERO_WRITE, page << 8, NULL, NULL); + for (i = 0; !ret && i < 2000; i++) { + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_READ(1), HPM_REG_SELECT, NULL, &val); + if (!ret && (val & 0xff) == page) + return 0; + udelay(10); + } + + return ret ?: -ETIMEDOUT; +} + +/* + * What macOS sends the other port controller about a second before a sleep, measured + * in three runs: system power state 3, then the bus's sleep command. + */ +static int t8140_hpm_sleep(struct t8140_sleep *s) +{ + u32 data = HPM_SSPS_S3, cmd = HPM_CMD_SSPS, val = ~0U; + int i, ret; + + ret = t8140_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = t8140_hpm_select(s, HPM_PAGE_DATA1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_WRITE(1), HPM_REG_DATA_CMD, &data, NULL); + if (!ret) + ret = t8140_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA_CMD, &cmd, NULL); + /* Selected afresh for each read, as macOS does. */ + for (i = 0; !ret && val && i < 100; i++) { + udelay(1000); + ret = t8140_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); + } + if (!ret && val) + ret = -ETIMEDOUT; + /* Asleep either way: a failure here only means it wasn't told first. */ + t8140_hpm_cmd(s, SPMI_OP_SLEEP, 0, NULL, NULL); + + return ret; +} + +/* + * The debug link stays up with the ATC0 domains off, and it holds the debug domains + * active, which macOS has idle. The only known way to end it is the port controller's + * cold reset. To the SMC that's a cable pulled: it queues the port's events and records + * them as wake reasons. So it has to come before the SMC's AP idle notice, not after, or + * SMC.OutboxNotEmpty is up at the commit. tipd rearms the link after a refusal. + */ +static bool t8140_sleep_usb_down(struct t8140_sleep *s) +{ + u32 cmd = HPM_CMD_COLD_RESET; + u32 enable; + int ret; + + if (s->usb_down_ran) + return false; + s->usb_down_ran = true; + + if (!s->hpm_spmi || !FIELD_GET(PS_ACTUAL, readl(s->ps_aon + PS_AON_DEBUG_SWITCH))) + return false; + + dev_info(s->dev, "debug link: cutting it for the sleep\n"); + enable = readl(s->hpm_spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + ret = t8140_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = t8140_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = t8140_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA, &cmd, NULL); + if (!ret) { + WRITE_ONCE(t8140_debug_link_cut, true); + mdelay(HPM_RESET_MS); + ret = t8140_hpm_sleep(s); + } + writel(1, s->hpm_spmi + SPMI_NOTIFY_ACK); + writel(enable, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + s->usb_down_ret = ret; + + return true; +} + +bool apple_sleep_debug_link_cut(void) +{ + return t8140_sleep ? t8140_sleep_usb_down(t8140_sleep) : false; +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); + +/* + * macOS takes the wake for a lid closing during the sleep as a dark one: every driver wakes, + * the display stays off, and the root domain sleeps again as soon as it sees the lid closed. + */ +void apple_sleep_lid_closed_wake(void) +{ + struct t8140_sleep *s = t8140_sleep; + + if (!s || !s->smc_woke_alone) + return; + dev_info(s->dev, "the lid closed during the sleep: dark wake\n"); + pm_set_dark_wake(); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_wake); + +/* + * The radio's link and the engine that starts it, in macOS's order, measured on one + * sleep: the link into L2, the engine stopped, PERST# asserted, the PHY's clocks off. + * macOS does it 40 ms before ANS's idle. Here it follows, so that a refused sleep finds + * the link untouched. A link that's down or a radio that's gone only makes the poll time + * out, and the power goes next either way. + */ +static void t8140_apcie_port_stop(struct t8140_sleep *s) +{ + void __iomem *port = s->apcie_port; + void __iomem *phy = s->apcie_port_phy; + u32 val; + + writel(readl(port + APCIE_PORT_INTMSK) | APCIE_PORT_INT_LINK, port + APCIE_PORT_INTMSK); + writel(APCIE_PORT_INT_LINK, port + APCIE_PORT_INTSTAT); + + writel(readl(port + APCIE_PORT_APPCLK) | APCIE_PORT_APPCLK_CGDIS, port + APCIE_PORT_APPCLK); + /* A loader can leave the port ready with the link never trained. */ + if (readl(port + APCIE_PORT_LINKSTS) & APCIE_PORT_LINKSTS_UP) { + writel(APCIE_PORT_L2_ENTER, port + APCIE_PORT_L2); + readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), + 10, APCIE_PORT_STOP_US); + } + /* The radio keeps its power and firmware, in reset until its host releases it. */ + if (s->radio_perst >= 0 && t8140_ps_on(s, PS_GPIO)) { + void __iomem *reg = s->pins + PIN_REG(s->radio_perst); + + val = readl(reg); + writel(s->radio_perst_low ? val & ~PIN_DATA : val | PIN_DATA, reg); + } + writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); + + if (s->apcie_engine && s->apcie_fabric) { + writel(0, s->apcie_engine + APCIE_ENGINE_IRQ_MASK); + writel(3, s->apcie_fabric + APCIE_FABRIC_STOP); + writel(readl(s->apcie_engine + APCIE_ENGINE_CTRL) | APCIE_ENGINE_STOP, + s->apcie_engine + APCIE_ENGINE_CTRL); + } + udelay(1000); + + /* What this bit does isn't known. macOS sets it around PERST#. */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(0, port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), port + APCIE_PORT_PERST_GUARD); + + writel(readl(phy) & ~APCIE_PORT_PHY_EN1, phy); + writel(readl(phy) & ~APCIE_PORT_PHY_EN0, phy); + writel(readl(phy) | APCIE_PHY_CTRL_RESET, phy); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK0REQ, phy); + readl_poll_timeout_atomic(phy, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK1REQ, phy); + + writel(readl(port + APCIE_PORT_APPCLK) & ~APCIE_PORT_APPCLK_EN, port + APCIE_PORT_APPCLK); + writel(readl(port + APCIE_PORT_INTSTAT), port + APCIE_PORT_INTSTAT); +} + +static void t8140_apcie_port_stop_once(struct t8140_sleep *s) +{ + /* Only with the port set up, by the loader or by its host after a wake, or it aborts. */ + if (s->apcie_port_stopped || s->apcie_port_down || !s->apcie_port || !s->apcie_port_phy || + !t8140_ps_on(s, PS_APCIE_GP) || !t8140_ps_on(s, PS_APCIE_PHY_SW)) + return; + s->apcie_port_stopped = true; + t8140_apcie_port_stop(s); +} + +/* A poll here only paces the steps: the power goes next whatever it reads. */ +static void t8140_apcie_phy_off(struct t8140_sleep *s) +{ + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + + writel(readl(ctrl) | APCIE_PHY_CTRL_RESET, ctrl); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK1REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK1ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK0REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_UP, ctrl); + writel(readl(run) & ~APCIE_PHY_RUN_EN2, run); + writel(readl(run) & ~APCIE_PHY_RUN_EN, run); +} + +static void t8140_apcie_clk_off(struct t8140_sleep *s) +{ + void __iomem *clk = s->apcie_clkgen + APCIE_CLK_CTRL; + u32 val; + + writel(readl(clk) & ~APCIE_CLK_RUN, clk); + writel(readl(clk) | APCIE_CLK_OFF, clk); + readl_poll_timeout_atomic(clk, val, !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + writel(readl(clk) | APCIE_CLK_OFF2, clk); +} + +/* + * ANS's own link runs on this block, and macOS takes it down with the domains. Its + * order, measured on one sleep, in the same microseconds as the power words. A block's + * registers fault with its domain off, and a loader may have left any of these off. + */ +static void t8140_sleep_lower_apcie(struct t8140_sleep *s) +{ + u32 val; + + t8140_apcie_port_stop_once(s); + /* The PHY's registers need both domains on. */ + if (s->apcie_phy && t8140_ps_on(s, PS_APCIE_PHY_SW) && t8140_ps_on(s, PS_APCIE_SYS_ST)) + t8140_apcie_phy_off(s); + t8140_ps_lower(s, PS_APCIE_PHY_SW); + + if (s->apcie_clkgen && t8140_ps_on(s, PS_APCIE_SYS_ST)) + t8140_apcie_clk_off(s); + t8140_ps_lower(s, PS_APCIE_SYS_ST); + t8140_ps_lower(s, PS_APCIE_ST); + + val = t8140_ps_lower_start(s, PS_APCIE_SYS_GP); + if (s->apcie_axi && t8140_ps_on(s, PS_APCIE_SYS_GP)) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + t8140_ps_lower_finish(s, PS_APCIE_SYS_GP, val); + + t8140_ps_lower(s, PS_APCIE_GP); + t8140_ps_lower(s, PS_ANS); +} + +/* Read-modify-write in table order, with no write where nothing changes. */ +static void t8140_tunable_apply_changed(void __iomem *regs, const struct apple_tunable *t) +{ + size_t i; + + for (i = 0; i < t->sz; i++) { + void __iomem *reg = regs + t->values[i].offset; + u32 old = readl(reg); + u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); + + if (val != old) + writel(val, reg); + } +} + +/* + * The power loss takes the PHY's PLL and lane settings with it, and ANS's link can't + * train without them. Applied as macOS does: the PLL table, then the lanes, then the + * PLL's hold released. That takes about the settling time the PHY needs here. + */ +static void t8140_apcie_phy_ip_apply(struct t8140_sleep *s) +{ + void __iomem *ctl = s->apcie_phy_ip + APCIE_PHY_IP_PLL_CTL; + + t8140_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); + t8140_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); + writel(readl(ctl) & ~APCIE_PHY_IP_PLL_HOLD, ctl); +} + +/* + * Brought up from power-off in macOS's order, measured on one sleep: all of it follows + * the domains on the waking core, well before ANS runs. Whether ANS needs each step is + * inferred. Nothing else does this if no host driver is bound. + */ +static int t8140_sleep_raise_apcie(struct t8140_sleep *s) +{ + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + int ret; + + ret = t8140_ps_raise(s, PS_ANS); + ret |= t8140_ps_raise(s, PS_APCIE_GP); + ret |= t8140_ps_raise_start(s, PS_APCIE_SYS_GP); + if (!ret) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) | BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) | BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + t8140_ps_raise_finish(s, PS_APCIE_SYS_GP); + ret |= t8140_ps_raise(s, PS_APCIE_ST); + ret |= t8140_ps_raise(s, PS_APCIE_SYS_ST); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_axi, s->apcie_axi_tunable); + apple_tunable_apply(s->apcie_cio3pll, s->apcie_cio3pll_tunable); + apple_tunable_apply(s->apcie_clkgen, s->apcie_clkgen_tunable); + writel(readl(s->apcie_clkgen + APCIE_CLK_START) | APCIE_CLK_START_GO, + s->apcie_clkgen + APCIE_CLK_START); + /* Both off bits: the lowering sets them, and only a power-off clears OFF. */ + writel(readl(s->apcie_clkgen + APCIE_CLK_CTRL) & ~(APCIE_CLK_OFF2 | APCIE_CLK_OFF), + s->apcie_clkgen + APCIE_CLK_CTRL); + ret = readl_poll_timeout_atomic(s->apcie_clkgen + APCIE_CLK_CTRL, val, + !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + if (ret) + return ret; + + ret = t8140_ps_raise(s, PS_APCIE_PHY_SW); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_phy, s->apcie_phy_tunable); + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK0REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK1REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) & ~APCIE_PHY_CTRL_RESET, ctrl); + + writel(readl(run) | APCIE_PHY_RUN_EN, run); + t8140_apcie_phy_ip_apply(s); + writel(readl(run) | APCIE_PHY_RUN_EN2, run); + ret = readl_poll_timeout_atomic(s->apcie_phy + APCIE_PHY_READY, val, val & BIT(0), 1, + APCIE_POLL_US); + if (ret) + return ret; + writel(readl(s->apcie_phy + APCIE_PHY_COMMON) | BIT(0), s->apcie_phy + APCIE_PHY_COMMON); + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + + writel(0, s->apcie_rc + APCIE_RC_ZERO); + writel(0x140, s->apcie_rc + APCIE_RC_READY_CFG); + writel(1, s->apcie_rc + APCIE_RC_READY_START); + + return readl_poll_timeout_atomic(s->apcie_rc + APCIE_RC_READY_STAT, val, val & BIT(0), 1, + APCIE_POLL_US); +} + +/* About 4 ms after the other cores stop on macOS: deepest first, ANS last. */ +static void t8140_sleep_lower_io(struct t8140_sleep *s) +{ + /* + * USB's complex, deepest first: macOS has it off here, and its drivers don't lower it. + * A host that sleeps in place keeps ATC0_USB_AON, as macOS's does. + */ + t8140_ps_lower(s, PS_ATC0_USB); + t8140_ps_lower(s, PS_ATC0_COMMON); + if (!READ_ONCE(t8140_usb_asleep)) + t8140_ps_lower(s, PS_ATC0_USB_AON); + if (s->pcie_ans_stopped) { + t8140_sleep_lower_apcie(s); + s->ans_off_ret = t8140_ps_wait_off(s, PS_ANS); + } +} + +static void t8140_sleep_set_wd2(struct t8140_sleep *s, unsigned int secs) +{ + writel(0, s->wdt + WDT_WD2_CTL); + writel(0, s->wdt + WDT_WD2_COUNT); + writel(secs * WDT_TICKS_PER_S, s->wdt + WDT_WD2_ALARM); + writel(1, s->wdt + WDT_WD2_CTL); +} + +/* + * Every power word in the ADT, read just before ANS's stop so a run shows the whole chip. + */ +static const u32 t8140_ps_all[] = { + 0x0, 0x8, 0x10, 0x18, 0x30, 0x38, 0x50, 0x58, 0x60, 0x100, 0x108, 0x110, 0x118, + 0x120, 0x128, 0x130, 0x138, 0x140, 0x148, 0x150, 0x158, 0x160, 0x168, 0x170, 0x178, + 0x180, 0x188, 0x190, 0x198, 0x1a0, 0x1a8, 0x1b0, 0x1b8, 0x1c0, 0x1c8, 0x1d0, 0x1d8, + 0x1e0, 0x1e8, 0x1f0, 0x1f8, 0x200, 0x208, 0x210, 0x218, 0x220, 0x228, 0x230, 0x238, + 0x240, 0x248, 0x250, 0x258, 0x260, 0x268, 0x270, 0x278, 0x280, 0x288, 0x290, 0x298, + 0x2a0, 0x2a8, 0x2b0, 0x2b8, 0x2c0, 0x2c8, 0x2d0, 0x2d8, 0x2e0, 0x2e8, 0x2f0, 0x2f8, + 0x300, 0x308, 0x310, 0x318, 0x320, 0x328, 0x330, 0x338, 0x340, 0x348, 0x350, 0x358, + 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x390, 0x398, 0x3a0, 0x3a8, 0x3b0, 0x3b8, + 0x3c0, 0x3c8, 0x3d0, 0x3d8, 0x3e0, 0x3e8, 0x3f0, 0xc00, 0x4000, 0x4008, 0x4010, + 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, 0x4050, 0x4058, 0x4060, + 0x8000, 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, + 0xc000, 0xc008, 0xc010, 0x10000, 0x10008, 0x10010, 0x10018, 0x10020, 0x10028, + 0x10030, 0x10038, 0x10040, +}; + +static const u32 t8140_ps_aon_all[] = { + 0x0, 0x58, 0x60, 0x68, 0x70, 0x78, 0x80, 0x88, 0x90, 0x98, 0xa0, 0xa8, 0xb0, + 0x4000, 0x4008, 0x4010, 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, + 0x4050, 0x4058, 0x4060, 0x4068, 0x4070, 0x4078, 0x4080, 0x4088, 0x4090, 0x4098, + 0x40a0, 0x40a8, 0x40b0, 0x40b8, 0x40c0, 0x40c8, 0x40d0, 0x40d8, 0x40e0, 0x40e8, + 0x40f0, 0x40f8, 0x4100, 0x4108, 0x4110, 0x4118, 0x4120, 0x4128, 0x4130, 0x4138, + 0x4140, 0x4148, 0x4150, 0x4158, 0x4160, 0x4168, 0x4170, 0x4178, 0x4180, 0x8000, + 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, 0x8050, + 0x8058, 0x8060, 0x8068, 0x8070, 0x8078, +}; + +/* What macOS has on at the commit and Linux leaves off, in its order: nothing uses them here. */ +static const u32 t8140_ps_macos_on[] = { + 0x2f0, 0x2f8, 0x358, 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x118, +}; + +/* + * The power words macOS's wake writes back on before any driver runs, in its order + * (logs/wakerec-20261105-154137Z). The engine leaves domains like GPIO dark, and genpd still + * has them on, so nothing else would. + */ +static const u32 t8140_ps_wake_order[] = { + 0xc00, 0x110, 0x118, 0x120, 0x168, 0x178, 0x2a8, 0x2f0, 0x2f8, 0x300, 0x358, 0x360, + 0x368, 0x370, 0x378, 0x380, 0x388, 0x3d0, 0x3d8, +}; + +static const u32 t8140_ps_aon_wake_order[] = { 0x78, 0x88, 0x8030, 0x8038 }; + +/* Each word at the commit, and as the wake found it. */ +static u32 t8140_ps_commit[ARRAY_SIZE(t8140_ps_all)]; +static u32 t8140_ps_aon_commit[ARRAY_SIZE(t8140_ps_aon_all)]; +static u32 t8140_ps_woke[ARRAY_SIZE(t8140_ps_all)]; +static u32 t8140_ps_aon_woke[ARRAY_SIZE(t8140_ps_aon_all)]; + +#define PS_PARENT_MISSING BIT(11) +#define PS_AUTO_STATE GENMASK(27, 24) +#define PS_REWAKE_PASSES 2 + +static void t8140_ps_save(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + t8140_ps_commit[i] = readl(s->ps + t8140_ps_all[i]); + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + t8140_ps_aon_commit[i] = readl(s->ps_aon + t8140_ps_aon_all[i]); +} + +static int t8140_ps_find(const u32 *all, int n, u32 offset) +{ + int i; + + for (i = 0; i < n; i++) + if (all[i] == offset) + return i; + return -1; +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Never + * DEV_DISABLE, which keeps it off: the SEP's is set at the commit, and macOS's wake writes + * that word without it. The hardware owns the auto state field, so it stays as read. + */ +static int t8140_ps_restore(void __iomem *reg, u32 saved) +{ + u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_PARENT_MISSING | PS_DEV_DISABLE | + PS_FLAGS | PS_ACTUAL)) | (readl(reg) & PS_AUTO_STATE); + u32 now; + int ret; + + /* + * Off under AUTO_ENABLE at the commit, as the SEP's is: the hardware powers it when + * it's wanted, and forcing it on doesn't. macOS writes that word once, AUTO set. + */ + if ((saved & PS_AUTO_ENABLE) && !FIELD_GET(PS_ACTUAL, saved)) { + writel(keep | PS_AUTO_ENABLE, reg); + return 0; + } + + writel(keep, reg); + ret = readl_poll_timeout_atomic(reg, now, + FIELD_GET(PS_ACTUAL, now) == FIELD_GET(PS_TARGET, saved), 1, + PS_OFF_TIMEOUT_US); + if (saved & PS_AUTO_ENABLE) + writel(keep | PS_AUTO_ENABLE, reg); + + return ret; +} + +/* Linux had it on and the wake left it otherwise. With AUTO_ENABLE, off is allowed. */ +static bool t8140_ps_dropped(u32 saved, u32 woke) +{ + u32 control = PS_AUTO_ENABLE | PS_DEV_DISABLE | PS_TARGET; + + if (!FIELD_GET(PS_TARGET, saved)) + return false; + if ((saved ^ woke) & control) + return true; + return !(saved & PS_AUTO_ENABLE) && + FIELD_GET(PS_ACTUAL, woke) != FIELD_GET(PS_TARGET, saved); +} + +/* Words that didn't read on in time, for the report. */ +static bool t8140_ps_late[ARRAY_SIZE(t8140_ps_all)]; +static bool t8140_ps_aon_late[ARRAY_SIZE(t8140_ps_aon_all)]; + +/* + * Every domain back as it was at the commit, before any driver's resume reads its block: + * macOS's wake words first, in its order, whatever they read, if Linux had them on. Then + * any other domain the wake left otherwise, a second time for one whose parent came later. + */ +static void t8140_ps_rewake(struct t8140_sleep *s) +{ + bool *redo = t8140_ps_late, *redo_aon = t8140_ps_aon_late; + bool done[ARRAY_SIZE(t8140_ps_all)] = {}, done_aon[ARRAY_SIZE(t8140_ps_aon_all)] = {}; + int i, j, pass; + + memset(t8140_ps_late, 0, sizeof(t8140_ps_late)); + memset(t8140_ps_aon_late, 0, sizeof(t8140_ps_aon_late)); + s->ps_rewoken = 0; + s->ps_rewake_timeouts = 0; + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + t8140_ps_woke[i] = readl(s->ps + t8140_ps_all[i]); + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + t8140_ps_aon_woke[i] = readl(s->ps_aon + t8140_ps_aon_all[i]); + + for (j = 0; j < ARRAY_SIZE(t8140_ps_wake_order); j++) { + i = t8140_ps_find(t8140_ps_all, ARRAY_SIZE(t8140_ps_all), t8140_ps_wake_order[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, t8140_ps_commit[i])) + continue; + done[i] = true; + s->ps_rewoken++; + if (t8140_ps_restore(s->ps + t8140_ps_all[i], t8140_ps_commit[i])) + redo[i] = true; + } + for (j = 0; j < ARRAY_SIZE(t8140_ps_aon_wake_order); j++) { + i = t8140_ps_find(t8140_ps_aon_all, ARRAY_SIZE(t8140_ps_aon_all), + t8140_ps_aon_wake_order[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, t8140_ps_aon_commit[i])) + continue; + done_aon[i] = true; + s->ps_rewoken++; + if (t8140_ps_restore(s->ps_aon + t8140_ps_aon_all[i], t8140_ps_aon_commit[i])) + redo_aon[i] = true; + } + + for (pass = 0; pass < PS_REWAKE_PASSES; pass++) { + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) { + if (!redo[i] && + (done[i] || !t8140_ps_dropped(t8140_ps_commit[i], t8140_ps_woke[i]))) + continue; + if (!done[i]) { + done[i] = true; + s->ps_rewoken++; + } + redo[i] = t8140_ps_restore(s->ps + t8140_ps_all[i], t8140_ps_commit[i]); + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) { + if (!redo_aon[i] && + (done_aon[i] || + !t8140_ps_dropped(t8140_ps_aon_commit[i], t8140_ps_aon_woke[i]))) + continue; + if (!done_aon[i]) { + done_aon[i] = true; + s->ps_rewoken++; + } + redo_aon[i] = t8140_ps_restore(s->ps_aon + t8140_ps_aon_all[i], + t8140_ps_aon_commit[i]); + } + } + + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + s->ps_rewake_timeouts += redo[i]; + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + s->ps_rewake_timeouts += redo_aon[i]; +} + +static void t8140_ps_rewake_report(struct t8140_sleep *s) +{ + int i; + + dev_dbg(s->dev, "power words: %u written back, %u not on in time\n", s->ps_rewoken, + s->ps_rewake_timeouts); + for (i = 0; i < ARRAY_SIZE(t8140_ps_all); i++) + if (t8140_ps_late[i]) + dev_err(s->dev, "power word %#x not on in time: %#x at the commit, %#x now\n", + t8140_ps_all[i], t8140_ps_commit[i], + readl(s->ps + t8140_ps_all[i])); + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_all); i++) + if (t8140_ps_aon_late[i]) + dev_err(s->dev, "power word aon %#x not on in time: %#x at the commit, %#x now\n", + t8140_ps_aon_all[i], t8140_ps_aon_commit[i], + readl(s->ps_aon + t8140_ps_aon_all[i])); +} + +/* What it has off, highest first, as it lowers them. Domains with their own stop aren't here. */ +static const u32 t8140_ps_macos_off[] = { + 0x4060, 0x4058, 0x4050, 0x4048, 0x4040, 0x4038, 0x4030, 0x4028, 0x4020, 0x4018, + 0x4010, 0x4008, 0x4000, 0x398, 0x390, 0x350, 0x348, 0x340, 0x338, 0x330, + 0x308, 0x2e8, 0x2e0, 0x2d8, 0x2d0, 0x2c8, 0x2c0, 0x2b8, 0x2b0, 0x298, + 0x290, 0x270, 0x268, 0x250, 0x248, 0x240, 0x238, 0x230, 0x1a0, 0x190, + 0x188, 0x180, 0x170, 0x160, 0x158, 0x150, 0x148, 0x140, 0x108, +}; + +static const u32 t8140_ps_aon_macos_off[] = { + 0x8028, 0x4140, 0x40f8, 0x80, +}; + +/* The AOP's SPMI words: on, with AUTO_ENABLE off on macOS. */ +static const u32 t8140_ps_aon_macos_no_auto[] = { + 0x4078, 0x4080, +}; + +/* As macOS writes the PMU: the reply's vector masked, then acknowledged, so none is left raised. */ +static void t8140_pmu_write(struct t8140_sleep *s, u16 reg, u8 val) +{ + u32 status, enable; + + enable = readl(s->spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->spmi + SPMI_NOTIFY_ENABLE); + writel(((u32)reg << 16) | SPMI_CMD_WRITE, s->spmi + SPMI_CMD); + writel(val, s->spmi + SPMI_CMD); + if (!readl_poll_timeout_atomic(s->spmi + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) + readl(s->spmi + SPMI_REPLY); + writel(1, s->spmi + SPMI_NOTIFY_ACK); + writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); +} + +static void t8140_sleep_match(struct t8140_sleep *s) +{ + int i, raised = 0, lowered = 0, failed = 0; + + writel(0, s->wdt + WDT_WD0_COUNT); + writel(0, s->wdt + WDT_WD1_COUNT); + t8140_pmu_write(s, PMU_BOOT_STAGE, PMU_BOOT_STAGE_SLEEP); + + s->spmi_saved[0] = readl(s->spmi + SPMI_NOTIFY_ENABLE); + s->spmi_saved[1] = readl(s->spmi_a0); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi + SPMI_NOTIFY_ENABLE); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi_a0); + + for (i = 0; i < ARRAY_SIZE(t8140_ps_macos_on); i++) { + u32 offset = t8140_ps_macos_on[i]; + + s->ps_on_saved[i] = readl(s->ps + offset); + /* State 4 reads as on, and macOS has these at f. */ + if (FIELD_GET(PS_ACTUAL, s->ps_on_saved[i]) == 0xf) + continue; + /* Measured at the commit with AUTO_ENABLE clear, unlike after a wake. */ + if (t8140_ps_raise_start(s, offset)) + failed++; + raised++; + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_macos_off); i++) { + u32 offset = t8140_ps_macos_off[i]; + + s->ps_off_saved[i] = readl(s->ps + offset); + if (!t8140_ps_on(s, offset)) + continue; + t8140_ps_lower(s, offset); + lowered++; + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_macos_off); i++) { + void __iomem *reg = s->ps_aon + t8140_ps_aon_macos_off[i]; + u32 val = readl(reg); + + s->ps_aon_saved[i] = val; + if (!FIELD_GET(PS_ACTUAL, val)) + continue; + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); + lowered++; + } + for (i = 0; i < ARRAY_SIZE(t8140_ps_aon_macos_no_auto); i++) { + void __iomem *reg = s->ps_aon + t8140_ps_aon_macos_no_auto[i]; + u32 val = readl(reg); + + s->ps_aon_auto_saved[i] = val; + writel(val & ~(PS_AUTO_ENABLE | PS_FLAGS), reg); + } + s->matched = true; + + for (i = 0; i < ARRAY_SIZE(t8140_ps_macos_off); i++) + if (t8140_ps_wait_off(s, t8140_ps_macos_off[i])) + dev_dbg(s->dev, "still on, macOS has it off: %x = %x\n", + t8140_ps_macos_off[i], readl(s->ps + t8140_ps_macos_off[i])); + dev_dbg(s->dev, "macOS's power words: raised %d, %d of them failed, lowered %d\n", + raised, failed, lowered); +} + +/* Back as they were, after a refusal past the match or a real wake. */ +static void t8140_sleep_match_undo(struct t8140_sleep *s) +{ + int i; + + if (!s->matched) + return; + s->matched = false; + + for (i = ARRAY_SIZE(t8140_ps_aon_macos_no_auto) - 1; i >= 0; i--) + writel(s->ps_aon_auto_saved[i] & ~PS_FLAGS, + s->ps_aon + t8140_ps_aon_macos_no_auto[i]); + for (i = ARRAY_SIZE(t8140_ps_aon_macos_off) - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_aon_saved[i])) + writel(s->ps_aon_saved[i] & ~PS_FLAGS, + s->ps_aon + t8140_ps_aon_macos_off[i]); + for (i = ARRAY_SIZE(t8140_ps_macos_off) - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_off_saved[i])) + t8140_ps_raise(s, t8140_ps_macos_off[i]); + for (i = ARRAY_SIZE(t8140_ps_macos_on) - 1; i >= 0; i--) { + u32 val = s->ps_on_saved[i]; + + if (!FIELD_GET(PS_ACTUAL, val)) + t8140_ps_lower(s, t8140_ps_macos_on[i]); + else if (FIELD_GET(PS_ACTUAL, val) != 0xf) + writel(val & ~PS_FLAGS, s->ps + t8140_ps_macos_on[i]); + } + writel(s->spmi_saved[0], s->spmi + SPMI_NOTIFY_ENABLE); + writel(s->spmi_saved[1], s->spmi_a0); +} + +/* The safety net, as macOS arms it. */ +static void t8140_sleep_arm_wd2(struct t8140_sleep *s) +{ + writel(0, s->wdt + WDT_WD0_CTL); + t8140_sleep_set_wd2(s, WDT_SLEEP_SECS); + writel(0, s->aon_misc + AON_AP_DEADLINE); +} + +static int t8140_perf_wait(void __iomem *reg, unsigned int state) +{ + u32 val; + + return readl_poll_timeout_atomic(reg, val, + (val & (DCS_BUSY | DCS_STATE)) == + FIELD_PREP(DCS_STATE, state), + 1, DCS_TIMEOUT_US); +} + +static int t8140_dcs_wait(struct t8140_sleep *s, unsigned int state) +{ + return t8140_perf_wait(s->dcs, state); +} + +/* + * As macOS does at about -18 ms, on each channel in turn: the enables down a bit at a time, + * then each of four lanes strobed and cleared, two lanes on every channel before the + * next two. Each write is built from a fresh read, as the trace has it. + */ +static void t8140_sleep_mcc_stop(struct t8140_sleep *s) +{ + int ch, pass, n; + u32 val; + + for (ch = 0; ch < MCC_CHANNELS; ch++) { + val = readl(s->mcc[ch] + MCC_ENABLES); + for (n = 0; val && n < 32; n++) { + val &= val - 1; + writel(val, s->mcc[ch] + MCC_ENABLES); + val = readl(s->mcc[ch] + MCC_ENABLES); + } + } + + for (pass = 0; pass < 2; pass++) { + for (ch = 0; ch < MCC_CHANNELS; ch++) { + void __iomem *mcc = s->mcc[ch]; + + for (n = 2 * pass; n < 2 * pass + 2; n++) { + writel(readl(mcc + MCC_LANES_MASK), mcc + MCC_LANES_MASK); + writel(readl(mcc + MCC_LANES) | MCC_LANE_STROBE(n), + mcc + MCC_LANES); + writel(readl(mcc + MCC_LANES) & ~MCC_LANE(n), mcc + MCC_LANES); + } + } + } +} + +/* + * The engine's first step polls for DCS at this state and has no way to get it there. + * macOS has every voter at 0 by now, which is how it lands there, and steps it down itself + * on the path without the PMC. Both, here, in that order. Nothing can fail by now. + */ +static void t8140_sleep_dcs_down(struct t8140_sleep *s) +{ + unsigned int state; + int agent; + + for (agent = 0; agent < PMC_VOTER_AGENTS; agent++) + if (readl(s->pmc_voters + agent * PMC_VOTER_STRIDE)) + writel(0, s->pmc_voters + agent * PMC_VOTER_STRIDE); + if (!t8140_dcs_wait(s, DCS_SLEEP_STATE)) + return; + + state = FIELD_GET(DCS_STATE, readl(s->dcs)); + if (state > DCS_STEP_STATE) { + writel(DCS_STEP_STATE, s->dcs); + t8140_dcs_wait(s, DCS_STEP_STATE); + } + if (state > DCS_SLEEP_STATE) { + writel(DCS_SLEEP_STATE, s->dcs); + t8140_dcs_wait(s, DCS_SLEEP_STATE); + } +} + +/* The boot core's last WFI. An event that ends it is taken, and the WFI taken again. */ +static void __noreturn t8140_boot_wfi(struct t8140_sleep *s) +{ + t8140_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + aic_sleep_ack(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +/* XNU polls these off just before its complex words. The sleep goes on whatever they read. */ +static void t8140_sleep_wait_still_on(struct t8140_sleep *s) +{ + static const u32 words[] = { + PS_ATC0_COMMON, PS_DISP_SYS, PS_DISPEXT0_SYS, PS_ANS, 0x270 /* AVD_SYS */, + 0x240 /* VENC0_SYS */, 0x248 /* VENC1_SYS */, 0x1f0 /* GFX */, 0x230 /* JPG */, + PS_APCIE_SYS_GP, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(words); i++) + t8140_ps_wait_off(s, words[i]); +} + +/* + * The last mile, in macOS's order. It starts after every refusal: this can't be + * undone, and returning now would leave the clusters armed off. No printing, and + * under 33 ms from the complex words to the WFI (the engine's bound for the + * cores to go idle). A hang from here ends at the watchdog. + * + * What's left out on purpose: + * - The two writes of 0 to 0x302800000: reads 0, changes nothing. + * - The RTC alarm: macOS arms it after the commit. rtcwake has armed it long + * before this, with no SPMI traffic near the engine. + */ +static int t8140_sleep_commit(unsigned long unused) +{ + struct t8140_sleep *s = t8140_sleep; + int plane, agent; + u32 val; + + /* + * The other cores stop first, as on macOS. Each answered the hold in .enter, and they + * stay coherent until the commit. + */ + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + s->cpus_stop_ret = spin_table_stop_parked(); + + t8140_sleep_lower_io(s); + t8140_sleep_arm_wd2(s); + t8140_sleep_wait_still_on(s); + + /* The SoC comes down before memory does, as on macOS. */ + if (t8140_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE)) { + writel(SOC_SLEEP_STATE, s->clk_perf + CLK_PERF_SOC); + t8140_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE); + } + t8140_sleep_dcs_down(s); + t8140_sleep_mcc_stop(s); + + /* Gates, PTD, the first voter twice, gates: macOS's order, measured once. */ + for (agent = 0; agent < 2; agent++) { + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + } + writeq(0, s->ptd + PTD_A); + writeq(0, s->ptd + PTD_B); + for (agent = 0; agent < 2; agent++) { + readl(s->pmc_voters); + writel(0, s->pmc_voters); + readl(s->pmc_voters + 4); + } + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + + /* Already off since .prepare, as on macOS. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_commit_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, 5000); + writel(0, s->cluster + CLUSTER_E); + writel(0, s->cluster + CLUSTER_P); + + writel(readl(s->ps + PS_SIO_CPU) & ~PS_AUTO_ENABLE, s->ps + PS_SIO_CPU); + + /* The SoC refuses the first, on macOS too. */ + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_38); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_3C); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_34); + + writel(readl(s->cllt + CLLT_0) & ~CLLT_0_CLEAR, s->cllt + CLLT_0); + writel(readl(s->cllt + CLLT_1) & ~CLLT_1_CLEAR, s->cllt + CLLT_1); + writel(readl(s->cllt + CLLT_2) & ~CLLT_2_CLEAR, s->cllt + CLLT_2); + + /* + * Two runs put these writes after the CLLT's: 5 to 6 ms before the commit + * in one, and the CLLT at 5.75 ms in the other. No run has hooked both. + */ + t8140_dvfm_park(s); + + t8140_ps_save(s); + for (plane = 0; plane < ARRAY_SIZE(s->amcc); plane++) { + void __iomem *amcc = s->amcc[plane]; + + writel(readl(amcc + AMCC_WORD0_ENABLE) & ~AMCC_WORD0_DSID_AGE_OUT, + amcc + AMCC_WORD0_ENABLE); + writel(readl(amcc + AMCC_WORD1_ENABLE) & ~AMCC_WORD1_UNEXP_RT_HIT, + amcc + AMCC_WORD1_ENABLE); + } + + /* aura reads this with the MMU off. */ + s->resumes_before = READ_ONCE(s->mbox->resumes); + WRITE_ONCE(s->mbox->resume_pa, __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dsb(sy); + + writel(1, s->commit); + + /* As on macOS: a raised line would end the last WFI, and the engine waits for it. */ + aic_sleep_mask_all(); + + /* Each core's deep WFI comes after the commit, this one last. No SPMI from here on. */ + spin_table_release_stopped(); + t8140_boot_wfi(s); +} + +/* Back on the boot core through the reset vector, MMU on, in macOS's order. */ +static void t8140_sleep_return(struct t8140_sleep *s) +{ + u32 val; + int i; + + /* + * The reset leaves the timer control at its EL2 default, without the event stream + * or EL0's counter. The timer driver's CPU PM exit puts it back too, later. + */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + + /* This core takes the AIC's IRQs, and a reset may have changed that. */ + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + + spin_table_stop_finished(); + aic_sleep_restore(); + + writel(CLUSTER_ON, s->cluster + CLUSTER_E); + s->cluster_e_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_E, val, + val & CLUSTER_E_READY, 1, + CLUSTER_TIMEOUT_US); + writel(CLUSTER_ON, s->cluster + CLUSTER_P); + + writel(PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_1); + + t8140_ps_rewake(s); + + val = readl(s->ps + PS_SEP); + if (val & PS_DEV_DISABLE) + writel(val & ~PS_DEV_DISABLE, s->ps + PS_SEP); + + /* The complex comes back on, and macOS turns it off. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, CLUSTER_TIMEOUT_US); + + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + + t8140_dvfm_unpark(s); + + /* macOS stops it here. Keep a longer one over device resume, until .end. */ + t8140_sleep_set_wd2(s, WDT_SUSPEND_SECS); + + /* + * What the commit lowered for a driver comes back before anything uses it, in + * macOS's order, measured about 14 ms after this point. With no driver, genpd's + * books decide, as they do for every other domain. + */ + if (s->pcie_ans_stopped) + s->pcie_ans_on_ret = t8140_sleep_raise_apcie(s); + s->apcie_port_down = true; + s->pins_lost = s->pins_saved_valid; + + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + s->resumes_after = READ_ONCE(s->mbox->resumes); + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_status[i] = readl(s->wake + WAKE_STATUS(i)); + s->returned = true; +} + +static int t8140_sleep_enter(suspend_state_t state) +{ + struct t8140_sleep *s = t8140_sleep; + int ret; + + ret = t8140_sleep_refuse(s); + if (ret) + return ret; + ret = t8140_sleep_cpus_hold(s); + /* As on macOS, the display goes off before the GPU. */ + if (!ret) + ret = t8140_display_off(s); + if (!ret) + ret = t8140_sleep_coprocs_check(s); + if (!ret) + ret = t8140_sleep_enclave_tell(s); + if (ret) { + spin_table_unhold_parked(); + t8140_sleep_stray_restore(s); + return ret; + } + t8140_sleep_coprocs_stop(s, true); + + t8140_wake_arm(s); + t8140_sleep_match(s); + + if (readl(s->wake + WAKE_SLP_DDR)) + writel(0, s->wake + WAKE_SLP_DDR); + /* Acknowledge a leftover done flag, as the DCS word has none on macOS. */ + if (readl(s->cluster_dcs) & CLUSTER_UP_DONE) + writel(readl(s->cluster_dcs), s->cluster_dcs); + t8140_sleep_coprocs_stop(s, false); + /* Done by now unless no SMC driver is bound. */ + t8140_sleep_usb_down(s); + + s->cntkctl = read_sysreg(cntkctl_el1); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, t8140_sleep_commit); + if (!ret) + t8140_sleep_return(s); + + t8140_sleep_match_undo(s); + t8140_sleep_stray_restore(s); + t8140_wake_restore(s); + + return ret; +} + +/* From here to .end a hang resets the machine. Nobody has to be there to recover it. */ +static int t8140_sleep_begin(suspend_state_t state) +{ + /* The last sleep's cut is its port controller's to take, by its .complete. */ + WRITE_ONCE(t8140_debug_link_cut, false); + if (t8140_sleep) { + t8140_sleep->apcie_port_stopped = false; + t8140_sleep->usb_down_ran = false; + t8140_sleep->smc_woke_alone = false; + t8140_pins_save(t8140_sleep); + } + t8140_sleep_set_wd2(t8140_sleep, WDT_SUSPEND_SECS); + pm_set_suspend_via_firmware(); + + return 0; +} + +static void t8140_sleep_end(void) +{ + writel(0, t8140_sleep->wdt + WDT_WD2_CTL); +} + +/* + * macOS turns this complex off at boot and again at the end of each wake, so at the commit + * its word is already 0 and the engine has no sequence to run for it. Turning it off at the + * commit instead puts that sequence, with its untimed wait, right before the CPU complexes'. + */ +static void t8140_sleep_ave_off(struct t8140_sleep *s) +{ + u32 val; + int ret; + + /* Target 0, and the leftover done flag from iBoot's up, as XNU's first write does. */ + writel(CLUSTER_UP_DONE, s->cluster + CLUSTER_AVE); + ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, 1, CLUSTER_TIMEOUT_US); + if (ret) + dev_warn(s->dev, "AVE complex didn't go off: %#x\n", val); +} + +static int t8140_sleep_prepare(void) +{ + struct t8140_sleep *s = t8140_sleep; + int i; + + t8140_sleep_ave_off(s); + + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + struct platform_device *pdev; + + s->display[i].bound = false; + if (!s->display[i].np) + continue; + pdev = of_find_device_by_node(s->display[i].np); + if (pdev) { + s->display[i].bound = device_is_bound(&pdev->dev); + put_device(&pdev->dev); + } + } + t8140_sleep_gate_prepare(s); + + return 0; +} + +/* Only the SMC's sources woke the SoC. */ +static bool t8140_sleep_smc_woke_alone(struct t8140_sleep *s) +{ + u32 smc[WAKE_WORDS] = {}; + bool any = false; + int i; + + /* SMC.CPUWakeupAP and SMC.OutboxNotEmpty, by their names. */ + for (i = 0; i < ARRAY_SIZE(t8140_wake_names); i++) { + u32 id = t8140_wake_names[i].id; + + if (strstarts(t8140_wake_names[i].name, "SMC.") && id / 32 < WAKE_WORDS) + smc[id / 32] |= BIT(id % 32); + } + for (i = 0; i < WAKE_WORDS; i++) { + u32 woke = s->wake_status[i] & s->wake_armed[i]; + + if (woke & ~smc[i]) + return false; + any |= !!woke; + } + + return any; +} + +static void t8140_sleep_report(void) +{ + struct t8140_sleep *s = t8140_sleep; + int i; + + s->smc_woke_alone = false; + if (!s->returned) + return; + s->returned = false; + s->smc_woke_alone = t8140_sleep_smc_woke_alone(s); + + dev_info(s->dev, "back from sleep, boot object resumes %llu -> %llu\n", + s->resumes_before, s->resumes_after); + for (i = 0; i < ARRAY_SIZE(t8140_wake_names); i++) { + u32 id = t8140_wake_names[i].id; + + if (s->wake_status[id / 32] & BIT(id % 32)) + dev_info(s->dev, "wake source: %s\n", t8140_wake_names[i].name); + } + t8140_ps_rewake_report(s); + + if (s->cpus_stop_ret) + dev_err(s->dev, "other CPUs did not stop: %d\n", s->cpus_stop_ret); + if (s->cluster_e_ret || s->cluster_ave_ret || s->cluster_ave_commit_ret) + dev_err(s->dev, "cluster power timed out: E %d AVE %d, AVE at the commit %d\n", + s->cluster_e_ret, s->cluster_ave_ret, s->cluster_ave_commit_ret); + if (s->ans_off_ret) + dev_err(s->dev, "ANS did not read off at the commit: %d\n", s->ans_off_ret); + if (s->usb_down_ret) + dev_err(s->dev, "debug link: the stop failed: %d\n", s->usb_down_ret); + if (s->pcie_ans_stopped && s->pcie_ans_on_ret) + dev_err(s->dev, "ANS's PCIe block did not come up after the wake: %d\n", + s->pcie_ans_on_ret); +} + +static const struct platform_suspend_ops t8140_sleep_ops = { + .valid = suspend_valid_only_mem, + .begin = t8140_sleep_begin, + .prepare = t8140_sleep_prepare, + .enter = t8140_sleep_enter, + .wake = t8140_sleep_report, + .end = t8140_sleep_end, +}; + +static int t8140_sleep_read_wake_sources(struct t8140_sleep *s) +{ + struct device_node *np = s->dev->of_node; + int count; + + count = of_property_read_variable_u32_array(np, "apple,wake-enable", s->wake_enable, + 1, WAKE_SOURCES_MAX); + if (count < 0) + return dev_err_probe(s->dev, count, "no wake source enabled\n"); + s->wake_enable_count = count; + + return 0; +} + +static int t8140_sleep_map_displays(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < T8140_DISPLAY_COUNT; i++) { + struct device_node *np; + int index; + + np = of_parse_phandle(s->dev->of_node, "apple,display-coprocessors", i); + if (!np) + return dev_err_probe(s->dev, -ENODEV, "no display coprocessor %d\n", i); + s->display[i].np = np; + + index = of_property_match_string(np, "reg-names", "coproc"); + if (index < 0) + return dev_err_probe(s->dev, index, "%pOF has no coproc registers\n", np); + s->display[i].asc = of_iomap(np, index); + if (!s->display[i].asc) + return dev_err_probe(s->dev, -ENOMEM, "cannot map %pOF\n", np); + } + + return 0; +} + +static void __iomem *t8140_sleep_map(struct device *dev, const char *name) +{ + struct device_node *np = dev->of_node; + struct resource res; + void __iomem *base; + int index; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return IOMEM_ERR_PTR(index); + if (of_address_to_resource(np, index, &res)) + return IOMEM_ERR_PTR(-EINVAL); + + /* Shared with the drivers of these blocks, so not claimed. */ + base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!base) + return IOMEM_ERR_PTR(-ENOMEM); + + return base; +} + +/* The register blocks every sleep writes, by their reg-names. */ +static const struct { + const char *name; + size_t offset; +} t8140_sleep_regs[] = { + { "ps", offsetof(struct t8140_sleep, ps) }, + { "pwrgate", offsetof(struct t8140_sleep, pwrgate) }, + { "cpu-start", offsetof(struct t8140_sleep, cpu_start) }, + { "cllt", offsetof(struct t8140_sleep, cllt) }, + { "cluster", offsetof(struct t8140_sleep, cluster) }, + { "commit", offsetof(struct t8140_sleep, commit) }, + { "wdt", offsetof(struct t8140_sleep, wdt) }, + { "aon-misc", offsetof(struct t8140_sleep, aon_misc) }, + { "wake", offsetof(struct t8140_sleep, wake) }, + { "pmc-dcp", offsetof(struct t8140_sleep, pmc_dcp) }, + { "spmi", offsetof(struct t8140_sleep, spmi) }, + { "pmc-voters", offsetof(struct t8140_sleep, pmc_voters) }, + { "dcs", offsetof(struct t8140_sleep, dcs) }, + { "ps-aon", offsetof(struct t8140_sleep, ps_aon) }, + { "spmi-a0", offsetof(struct t8140_sleep, spmi_a0) }, + { "clk-mux", offsetof(struct t8140_sleep, clk_mux) }, + { "mcc0", offsetof(struct t8140_sleep, mcc[0]) }, + { "mcc1", offsetof(struct t8140_sleep, mcc[1]) }, + { "mcc2", offsetof(struct t8140_sleep, mcc[2]) }, + { "ptd", offsetof(struct t8140_sleep, ptd) }, + { "clk-perf", offsetof(struct t8140_sleep, clk_perf) }, + { "cluster-dcs", offsetof(struct t8140_sleep, cluster_dcs) }, + { "dvfm-e", offsetof(struct t8140_sleep, dvfm[0]) }, + { "dvfm-p", offsetof(struct t8140_sleep, dvfm[1]) }, + { "amcc0", offsetof(struct t8140_sleep, amcc[0]) }, + { "amcc1", offsetof(struct t8140_sleep, amcc[1]) }, + { "amcc2", offsetof(struct t8140_sleep, amcc[2]) }, + { "amcc3", offsetof(struct t8140_sleep, amcc[3]) }, +}; + +static int t8140_sleep_map_regs(struct t8140_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t8140_sleep_regs); i++) { + void __iomem **field = (void __iomem **)((u8 *)s + t8140_sleep_regs[i].offset); + + *field = t8140_sleep_map(s->dev, t8140_sleep_regs[i].name); + if (IS_ERR(*field)) + return dev_err_probe(s->dev, PTR_ERR(*field), "no %s registers\n", + t8140_sleep_regs[i].name); + } + + return 0; +} + +static int t8140_sleep_map_mailbox(struct t8140_sleep *s) +{ + struct device_node *chosen; + u64 pa; + int ret; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return -ENODEV; + ret = of_property_read_u64(chosen, "aurora,s2r-mailbox", &pa); + of_node_put(chosen); + if (ret) { + dev_info(s->dev, "no aurora,s2r-mailbox from the boot object: no deep sleep\n"); + return -ENODEV; + } + + s->mbox = devm_memremap(s->dev, pa, sizeof(*s->mbox), MEMREMAP_WB); + if (IS_ERR(s->mbox)) + return PTR_ERR(s->mbox); + + /* aura wrote it with the MMU off, so nothing of it can be in a cache. */ + if (s->mbox->magic != T8140_S2R_MAGIC) + return dev_err_probe(s->dev, -EINVAL, "bad mailbox magic %llx\n", + s->mbox->magic); + /* Probe can run on any CPU. Logical CPU 0 is always the boot CPU. */ + if (s->mbox->boot_mpidr != cpu_logical_map(0)) + return dev_err_probe(s->dev, -EINVAL, + "mailbox is for CPU %llx, not this boot CPU\n", + s->mbox->boot_mpidr); + + return 0; +} + +static struct apple_tunable *t8140_sleep_apcie_tunable(struct t8140_sleep *s, + struct device_node *np, + const char *name, const char *prop) +{ + int idx = of_property_match_string(np, "reg-names", name); + struct apple_tunable *t; + struct resource res; + + if (idx < 0 || of_address_to_resource(np, idx, &res)) + return NULL; + t = devm_apple_tunable_parse(s->dev, np, prop, &res); + + return IS_ERR(t) ? NULL : t; +} + +static void __iomem *t8140_sleep_apcie_block(struct t8140_sleep *s, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable) +{ + int idx = of_property_match_string(np, "reg-names", name); + + if (idx < 0) + return NULL; + if (prop) { + *tunable = t8140_sleep_apcie_tunable(s, np, name, prop); + if (!*tunable) + return NULL; + } + + return of_iomap(np, idx); +} + +/* + * Its registers go with PCIe's node, which a test tree may have disabled, and a loader + * publishes the tunables there. Missing ones only refuse a real sleep with ANS stopped. + */ +static void t8140_sleep_map_apcie(struct t8140_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,apcie", 0); + struct device_node *engine; + + if (!np) + return; + s->apcie_axi = t8140_sleep_apcie_block(s, np, "axi2af", "apple,tunable-axi2af", + &s->apcie_axi_tunable); + s->apcie_cio3pll = t8140_sleep_apcie_block(s, np, "cio3pll", "apple,tunable-cio3pll", + &s->apcie_cio3pll_tunable); + s->apcie_clkgen = t8140_sleep_apcie_block(s, np, "clkgen", "apple,tunable-clkgen", + &s->apcie_clkgen_tunable); + s->apcie_phy = t8140_sleep_apcie_block(s, np, "phy-block", "apple,tunable-phy", + &s->apcie_phy_tunable); + s->apcie_rc = t8140_sleep_apcie_block(s, np, "rc", NULL, NULL); + s->apcie_phy_ip = t8140_sleep_apcie_block(s, np, "phy-ip", "apple,tunable-phy-ip-pll", + &s->apcie_phy_ip_pll); + s->apcie_phy_ip_lanes = t8140_sleep_apcie_tunable(s, np, "phy-ip", + "apple,tunable-phy-ip-auspma"); + if (!s->apcie_axi || !s->apcie_cio3pll || !s->apcie_clkgen || !s->apcie_phy || + !s->apcie_rc || !s->apcie_phy_ip || !s->apcie_phy_ip_lanes) { + dev_info(s->dev, + "no PCIe tunables for ANS: a real sleep with it stopped refuses, s2idle stays the default\n"); + s->apcie_phy_tunable = NULL; + s->apcie_phy = NULL; + s->apcie_clkgen = NULL; + s->apcie_axi = NULL; + } + + s->apcie_port = t8140_sleep_apcie_block(s, np, "port0", NULL, NULL); + s->apcie_port_phy = t8140_sleep_apcie_block(s, np, "phy0", NULL, NULL); + engine = of_parse_phandle(np, "apple,piodma", 0); + if (engine) { + s->apcie_engine = t8140_sleep_apcie_block(s, engine, "engine", NULL, NULL); + s->apcie_fabric = t8140_sleep_apcie_block(s, engine, "fabric", NULL, NULL); + of_node_put(engine); + } + of_node_put(np); +} + +/* The radio port's PERST# on the AP pins @np, from the enabled T8140 PCIe host's ports. */ +static void t8140_sleep_find_radio_perst(struct t8140_sleep *s, struct device_node *np) +{ + struct device_node *host, *port; + struct of_phandle_args args; + + host = of_find_compatible_node(NULL, NULL, "apple,t8140-pcie"); + if (!host) + return; + if (of_device_is_available(host)) { + for_each_available_child_of_node(host, port) { + if (of_parse_phandle_with_args(port, "reset-gpios", "#gpio-cells", 0, + &args)) + continue; + if (args.np == np && args.args_count >= 2 && args.args[0] < s->npins && + !test_bit(args.args[0], s->pins_skip)) { + s->radio_perst = args.args[0]; + s->radio_perst_low = args.args[1] & GPIO_ACTIVE_LOW; + } + of_node_put(args.np); + } + } + of_node_put(host); +} + +/* macOS leaves the skipped pins alone, and reading one may fault. */ +static void t8140_sleep_map_pins(struct t8140_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,pinctrl", 0); + u32 npins, pin; + int i, n; + + s->radio_perst = -1; + if (!np) + return; + if (!of_device_is_available(np) || of_property_read_u32(np, "apple,npins", &npins) || + !npins || npins > 512) + goto out; + s->pins_saved = devm_kcalloc(s->dev, npins, sizeof(*s->pins_saved), GFP_KERNEL); + s->pins_skip = devm_bitmap_zalloc(s->dev, npins, GFP_KERNEL); + if (!s->pins_saved || !s->pins_skip) + goto out; + n = of_property_count_u32_elems(s->dev->of_node, "apple,pinctrl-skip-pins"); + for (i = 0; i < n; i++) + if (!of_property_read_u32_index(s->dev->of_node, "apple,pinctrl-skip-pins", i, + &pin) && pin < npins) + set_bit(pin, s->pins_skip); + s->pin_irq_groups = of_irq_count(np); + s->npins = npins; + s->pins = of_iomap(np, 0); + if (s->pins) + t8140_sleep_find_radio_perst(s, np); +out: + of_node_put(np); +} + +static void t8140_sleep_map_debug_port(struct t8140_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,debug-port", 0); + struct device_node *bus; + + if (!np) + return; + bus = of_get_parent(np); + if (bus && !of_property_read_u32_index(np, "reg", 0, &s->hpm_sid)) + s->hpm_spmi = of_iomap(bus, 0); + of_node_put(bus); + /* Kept, for as long as the driver: no unbind. */ + s->hpm_np = np; +} + +static int t8140_sleep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct t8140_sleep *s; + int ret; + + /* Below EL2 a monitor owns all of this. */ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + s = devm_kzalloc(dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = dev; + + /* Without the boot object's half, a wake would rerun its boot flow. */ + ret = t8140_sleep_map_mailbox(s); + if (ret) + return ret; + + ret = t8140_sleep_map_regs(s); + if (ret) + return ret; + + ret = t8140_sleep_read_wake_sources(s); + if (ret) + return ret; + ret = t8140_sleep_map_displays(s); + if (ret) + return ret; + t8140_sleep_map_apcie(s); + t8140_sleep_map_pins(s); + t8140_sleep_map_debug_port(s); + + platform_set_drvdata(pdev, s); + + ret = spin_table_set_power_ops(&t8140_cpu_power_ops); + if (ret) + return dev_err_probe(dev, ret, "cannot control CPU power\n"); + + /* Last: the exported helpers take it as a probed driver, and a failure frees s. */ + t8140_sleep = s; + /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ + suspend_set_ops(&t8140_sleep_ops); + /* Not when every real sleep would refuse because ANS couldn't come back. */ + if (!s->apcie_phy_tunable && mem_sleep_default == PM_SUSPEND_MAX) + mem_sleep_current = PM_SUSPEND_TO_IDLE; + + dev_info(dev, "deep sleep %s: boot object resumes %llu\n", + mem_sleep_current == PM_SUSPEND_MEM ? "is the default" : + "available, not the default", READ_ONCE(s->mbox->resumes)); + + return 0; +} + +static const struct of_device_id t8140_sleep_of_match[] = { + { .compatible = "apple,t8140-sleep" }, + { } +}; + +static struct platform_driver t8140_sleep_driver = { + .driver = { + .name = "t8140-sleep", + .of_match_table = t8140_sleep_of_match, + .suppress_bind_attrs = true, + }, + .probe = t8140_sleep_probe, +}; +builtin_platform_driver(t8140_sleep_driver); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 58ada41ed96ec..e655fad881fac 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -12,7 +12,9 @@ */ #include +#include #include +#include #include #include #include @@ -23,8 +25,10 @@ #include #include #include +#include #include #include +#include /* SPMI Controller Registers */ #define SPMI_STATUS_REG 0 @@ -98,6 +102,7 @@ struct apple_spmi { DECLARE_BITMAP(irq_mask_cache, SPMI_MAX_PERIPHERAL_IRQS); int irq; bool notify_irq; + bool notify_parked; bool prev_fail; }; @@ -485,6 +490,63 @@ static int apple_spmi_init_irq(struct platform_device *pdev, return 0; } +/* + * Every command asks for an interrupt on its response, on the vector this + * driver waits on. The system's wake logic watches this queue's interrupt line + * for the PMU's RTC alarm, so that vector has to be quiet and clear before a + * firmware-driven suspend, or the commit may read our own traffic as a wake. + * macOS clears the same enable bit at about the same point (measured under aura). + * Any command sent from here on polls for its response. + */ +static int apple_spmi_suspend_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + unsigned long flags; + + if (!spmi->notify_irq || !pm_suspend_via_firmware()) + return 0; + + guard(mutex)(&spmi->fifo_lock); + spmi->notify_irq = false; + spmi->notify_parked = true; + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + apple_spmi_irq_mask_raw(spmi, spmi->hw->num_irqs); + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + + return 0; +} + +static int apple_spmi_resume_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + u32 words[SPMI_MAX_PERIPHERAL_IRQS / 32]; + unsigned long flags; + size_t i; + + if (!spmi->notify_parked) + return 0; + + guard(mutex)(&spmi->fifo_lock); + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + /* Whether the enables survive a real sleep has not been measured. */ + if (pm_resume_via_firmware()) { + bitmap_to_arr32(words, spmi->irq_mask_cache, spmi->hw->num_irqs); + for (i = 0; i < spmi->hw->num_irqs / 32; i++) + writel(words[i], spmi->regs + SPMI_IRQ_MASK_BASE + i * 4); + } + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); + apple_spmi_irq_unmask_raw(spmi, spmi->hw->num_irqs); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + spmi->notify_parked = false; + spmi->notify_irq = true; + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(apple_spmi_pm_ops, apple_spmi_suspend_noirq, + apple_spmi_resume_noirq); + static int apple_spmi_probe(struct platform_device *pdev) { struct apple_spmi *spmi; @@ -547,6 +609,7 @@ static struct platform_driver apple_spmi_driver = { .driver = { .name = "apple-spmi", .of_match_table = apple_spmi_match_table, + .pm = pm_sleep_ptr(&apple_spmi_pm_ops), }, }; module_platform_driver(apple_spmi_driver); diff --git a/drivers/tty/serial/apple_dockchannel.h b/drivers/tty/serial/apple_dockchannel.h index 97fcccc6a5a57..4fcecee3b81c5 100644 --- a/drivers/tty/serial/apple_dockchannel.h +++ b/drivers/tty/serial/apple_dockchannel.h @@ -24,13 +24,20 @@ */ #define DOCKCHANNEL_NEVER_WAIT "apple,tx-never-wait" -/* Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. */ -static inline void dockchannel_wait_putc(void __iomem *data, unsigned char c) +/* + * Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. With the + * debug host gone the FIFO stays full: after one character timed out, the next + * are tried once each (*@no_reader) until there's room again, so printing on the + * way into and out of a sleep doesn't hold anything up. + */ +static inline void dockchannel_wait_putc(void __iomem *data, bool *no_reader, + unsigned char c) { - unsigned int retries = DOCKCHANNEL_TX_RETRIES; + unsigned int retries = *no_reader ? 1 : DOCKCHANNEL_TX_RETRIES; while (readl(data + DOCKCHANNEL_TX_FREE) == 0 && --retries) cpu_relax(); + *no_reader = !retries; if (!retries) return; diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 6c427189d34be..5004f17501e47 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -16,13 +16,15 @@ static struct dockchannel_tx dockchannel_early_tx; static bool dockchannel_early_never_wait; +/* The last character timed out: see dockchannel_wait_putc(). */ +static bool dockchannel_early_no_reader; static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) { if (dockchannel_early_never_wait) dockchannel_console_putc(port->membase, &dockchannel_early_tx, c); else - dockchannel_wait_putc(port->membase, c); + dockchannel_wait_putc(port->membase, &dockchannel_early_no_reader, c); } /* The DT's DockChannel UART node asks for DOCKCHANNEL_NEVER_WAIT (flat DT). */ diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index d4255cb949848..7150cf1cca350 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -109,6 +109,9 @@ static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) dcuart_irq_update(dcuart, bits, false); } +/* The last console character timed out: see dockchannel_wait_putc(). */ +static bool dcuart_no_reader; + static void dcuart_putc(struct uart_port *port, unsigned char c) { struct dcuart_port *dcuart = container_of(port, struct dcuart_port, port); @@ -116,7 +119,7 @@ static void dcuart_putc(struct uart_port *port, unsigned char c) if (dcuart->never_wait) dockchannel_console_putc(port->membase, &dcuart->console_tx, c); else - dockchannel_wait_putc(port->membase, c); + dockchannel_wait_putc(port->membase, &dcuart_no_reader, c); } static void dcuart_write(struct console *co, const char *s, diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 8e8a5f59b319b..0ea6a9a729bc5 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -2280,7 +2280,9 @@ int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg) usb_set_device_state(rhdev, USB_STATE_SUSPENDED); hcd->state = HC_STATE_SUSPENDED; - if (!PMSG_IS_AUTO(msg)) + /* A suspend after a raced autoresume finds them released already. */ + if (!PMSG_IS_AUTO(msg) && + !test_and_set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); @@ -2321,11 +2323,12 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) return 0; } - if (!PMSG_IS_AUTO(msg)) { + if (!PMSG_IS_AUTO(msg) && test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { status = usb_phy_roothub_resume(hcd->self.sysdev, hcd->phy_roothub); if (status) return status; + clear_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); } if (!hcd->driver->bus_resume) @@ -2368,7 +2371,11 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) } } else { hcd->state = old_state; - usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + /* Only release what this resume took: an autoresume took nothing. */ + if (!PMSG_IS_AUTO(msg)) { + usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); + } dev_dbg(&rhdev->dev, "bus %s fail, err %d\n", "resume", status); if (status != -ESHUTDOWN) @@ -3084,8 +3091,16 @@ void usb_remove_hcd(struct usb_hcd *hcd) usb_deregister_bus(&hcd->self); hcd_buffer_destroy(hcd); - usb_phy_roothub_power_off(hcd->phy_roothub); - usb_phy_roothub_exit(hcd->phy_roothub); + /* + * A root hub whose resume failed still has its PHYs released by the suspend, + * powered off and, without wakeup, exited too. Don't release them twice. + */ + if (!test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { + usb_phy_roothub_power_off(hcd->phy_roothub); + usb_phy_roothub_exit(hcd->phy_roothub); + } else if (device_may_wakeup(hcd->self.sysdev)) { + usb_phy_roothub_exit(hcd->phy_roothub); + } usb_put_invalidate_rhdev(hcd); hcd->flags = 0; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 30b5c5fc4d071..7821ccb4686e5 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1002,10 +1002,13 @@ struct dwc3_scratchpad_array { * need to be passed on to glue layer * @pre_set_role: Notify glue of role switch notifications * @pre_run_stop: Notify run stop enable/disable information to glue + * @pre_host_resume: Before xHCI touches the core at a system resume; an error keeps it away, + * and a positive value says the core lost its state and was set up again */ struct dwc3_glue_ops { void (*pre_set_role)(struct dwc3 *dwc, enum usb_role role); void (*pre_run_stop)(struct dwc3 *dwc, bool is_on); + int (*pre_host_resume)(struct dwc3 *dwc); }; /** diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index 7a04604ddd4a1..69abb0f32653f 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -38,10 +38,19 @@ static void dwc3_apple_non_role_test(struct kunit *test) } } +/* A role that arrives during a sleep decides what comes back after it. */ +static void dwc3_apple_role_during_sleep_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST), DWC3_APPLE_HOST); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE), DWC3_APPLE_DEVICE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE), DWC3_APPLE_NO_CABLE); +} + static struct kunit_case dwc3_apple_state_test_cases[] = { KUNIT_CASE(dwc3_apple_host_left_to_xhci_test), KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), KUNIT_CASE(dwc3_apple_non_role_test), + KUNIT_CASE(dwc3_apple_role_during_sleep_test), {} }; diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index 42c13d900428e..bb8d827fdec4c 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -9,6 +9,7 @@ #include #include +#include /* See the comment at the top of dwc3-apple.c for how these states are used. */ enum dwc3_apple_state { @@ -47,4 +48,22 @@ static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target } } +/** + * dwc3_apple_state_for_role() - state a role asks for while the core is stopped for sleep + * @role: the role asked for + * + * Return: the state to bring the controller up in after the sleep. + */ +static inline enum dwc3_apple_state dwc3_apple_state_for_role(enum usb_role role) +{ + switch (role) { + case USB_ROLE_HOST: + return DWC3_APPLE_HOST; + case USB_ROLE_DEVICE: + return DWC3_APPLE_DEVICE; + default: + return DWC3_APPLE_NO_CABLE; + } +} + #endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f66370e6b5f88..f2d575269c66f 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -9,16 +9,23 @@ */ #include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include +#include +#include +#include #include #include "dwc3-apple-state.h" @@ -91,6 +98,14 @@ * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device * may wake the system, as a halted xHC could not signal the wakeup. * + * A deep sleep takes the controller's power domain away. Behind a fixed hub on T8140 the host + * sleeps in place, as macOS's does: xHCI suspends and resumes as usual, and the PHY takes the + * core through hibernation in between, so the hub and its devices stay. A core that comes + * back without its registers is set up again in place and xHCI starts over, so the devices + * reset-resume. Anywhere else, or when the PHY can't, the glue stops everything before the + * freezer runs, the way a cable disconnect does, and brings it back once the system has + * resumed. See dwc3_apple_pm_notify(). + * * Runtime PM (T6040 and T8132, see dwc3_apple_core_probe()): the glue holds a reference from * the reset deassert to the end of the shutdown, so with no cable the controller can * runtime-suspend and its power domain drop, its DARTs (which save and restore their state) and, @@ -144,6 +159,8 @@ struct dwc3_apple { struct phy *usb3_phy; struct usb_role_switch *role_sw; bool fixed_hub; + /* The root port the fixed hub sits on, its node's reg. */ + u32 hub_port; bool host_only; struct mutex lock; @@ -159,6 +176,22 @@ struct dwc3_apple { bool rpm_held; bool pre_core_rpm; bool dt_dis_u3_susphy; + struct notifier_block pm_nb; + /* Kept through a deep sleep, with the PHY's hibernation, and the GUID it had. */ + bool sleep_in_place; + u32 sleep_guid; + /* The core lost its registers in that sleep and was set up again in place. */ + bool sleep_reinit; + /* Looks for the fixed hub once a host restarted after a sleep has had time to find it. */ + struct delayed_work hub_check; + struct work_struct sleep_finish; + bool sleep_finish_kept; + /* The core didn't come back from a sleep, and nothing may touch it until a reboot. */ + bool sleep_dead; + /* Stopped for a deep sleep: the state to come back to, and the wakeup setting. */ + bool sleep_stopped; + bool sleep_wakeup; + enum dwc3_apple_state sleep_state; void __iomem *drd_regs[DWC3_APPLE_NR_DRD_REGS]; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; @@ -210,6 +243,27 @@ static inline void dwc3_apple_mask(struct dwc3_apple *appledwc, u32 offset, u32 dwc3_apple_writel(appledwc, offset, reg); } +/* + * macOS's T8140 controller reset sets GCTL's hibernation enable and a power-down scale + * of 0xd. Without the enable the core's power management accepts a + * D0/D3 request but saves and restores nothing, so the PHY's hibernation for a sleep + * needs it. + */ +#define DWC3_APPLE_T8140_PWRDNSCALE 0xd + +static void dwc3_apple_setup_hibernation(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 reg; + + if (!of_device_is_compatible(appledwc->dev->of_node, "apple,t8140-dwc3")) + return; + reg = dwc3_readl(dwc, DWC3_GCTL); + reg &= ~DWC3_GCTL_PWRDNSCALE_MASK; + reg |= DWC3_GCTL_PWRDNSCALE(DWC3_APPLE_T8140_PWRDNSCALE) | DWC3_GCTL_GBLHIBERNATIONEN; + dwc3_writel(dwc, DWC3_GCTL, reg); +} + static void dwc3_apple_setup_cio(struct dwc3_apple *appledwc) { dwc3_apple_writel(appledwc, APPLE_DWC3_CIO_LFPS_OFFSET, APPLE_DWC3_CIO_LFPS_OFFSET_VALUE); @@ -361,6 +415,96 @@ static void dwc3_apple_set_ptrcap(struct dwc3_apple *appledwc, u32 mode) dwc3_set_prtcap(&appledwc->dwc, mode, false); } +/* Written to the core's GUID scratch register before a sleep in place, read back after it. */ +#define DWC3_APPLE_SLEEP_GUID 0x534c5045 + +/* macOS waits up to 100 ms for this after the core's D0, and only logs a timeout. */ +#define DWC3_APPLE_CFG_STATUS 0x2c +#define DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY BIT(30) +#define DWC3_APPLE_PIPE_PHY_READY_US 100000 + +/* + * The core is powered but its registers are at their reset values. Set it up again the way + * dwc3_apple_core_start() and dwc3_apple_set_role() do, in place: the PHYs and the xHCI + * device stay, so xHCI only has to start over and its devices reset-resume. Runs in the + * resume, while nothing else touches the core. + */ +static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + int i, ret; + + ret = dwc3_core_init(dwc); + if (ret) + return ret; + /* dwc3_core_init() took PHY references the core holds already. */ + for (i = 0; i < dwc->num_usb3_ports; i++) { + phy_power_off(dwc->usb3_generic_phy[i]); + phy_exit(dwc->usb3_generic_phy[i]); + } + for (i = 0; i < dwc->num_usb2_ports; i++) { + phy_power_off(dwc->usb2_generic_phy[i]); + phy_exit(dwc->usb2_generic_phy[i]); + } + + dwc3_apple_setup_cio(appledwc); + dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); + dwc3_apple_set_ptrcap(appledwc, DWC3_GCTL_PRTCAP_HOST); + dwc->dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; + dwc3_enable_susphy(dwc, true); + /* A PHY that came back in a USB3 mode routes the PIPE to it now, before xHCI starts. */ + ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_HOST); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed after the sleep, USB2 only, err=%d\n", + ret); + + return 0; +} + +/* + * Runs before xHCI touches the core after a sleep in place. A core that may be unpowered must + * be left alone, as an access could be an SError: dwc3_apple_sleep_kept() deals with it once + * the system is back. A core that came back without its registers is set up again, and xHCI + * is told it lost its state (a positive return). A sleep refused before the hibernation left + * the core as it was. + */ +static int dwc3_apple_pre_host_resume(struct dwc3 *dwc) +{ + struct dwc3_apple *appledwc = to_dwc3_apple(dwc); + u32 val; + int ret; + + if (!READ_ONCE(appledwc->sleep_in_place)) + return 0; + + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + if (ret == -EBUSY) + ret = 0; + if (!ret && appledwc->drd_regs[0] && /* drd-cfg */ + readl_poll_timeout(appledwc->drd_regs[0] + DWC3_APPLE_CFG_STATUS, val, + val & DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY, 100, + DWC3_APPLE_PIPE_PHY_READY_US)) + dev_warn(appledwc->dev, "PIPE PHY not ready after the sleep: %#x\n", val); + if (!ret && dwc3_readl(dwc, DWC3_GUID) != DWC3_APPLE_SLEEP_GUID) { + ret = dwc3_apple_core_reinit(appledwc); + if (!ret) { + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core lost its registers in the sleep: set up again, xHCI starts over\n"); + return 1; + } + } + if (ret) + dev_err(appledwc->dev, "not resuming xHCI, the core didn't come back: %d\n", ret); + + return ret; +} + +static const struct dwc3_glue_ops dwc3_apple_glue_ops = { + .pre_host_resume = dwc3_apple_pre_host_resume, +}; + static bool dwc3_apple_host_before_core_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { @@ -378,6 +522,7 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc, enum dwc3_apple_st WARN_ON_ONCE(appledwc->state != DWC3_APPLE_PROBE_PENDING); appledwc->dwc.dev = appledwc->dev; + appledwc->dwc.glue_ops = &dwc3_apple_glue_ops; probe_data.dwc = &appledwc->dwc; probe_data.res = appledwc->mmio_resource; probe_data.ignore_clocks_and_resets = true; @@ -622,6 +767,7 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st dwc3_apple_host_before_core_init(appledwc, state)) dwc3_apple_setup_cio(appledwc); dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); appledwc->core_active = true; appledwc->usb2_power_off_pending = BIT(appledwc->dwc.num_usb2_ports) - 1; appledwc->usb3_power_off_pending = BIT(appledwc->dwc.num_usb3_ports) - 1; @@ -784,6 +930,152 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st return ret; } +static int dwc3_apple_exit(struct dwc3_apple *appledwc); + +/* A hub that hasn't enumerated by then didn't come back. */ +#define DWC3_APPLE_HUB_CHECK_MS 5000 + +/* Any way a fixed-hub host came out of a sleep, its hub must be there a little later. */ +static void dwc3_apple_check_hub_later(struct dwc3_apple *appledwc) +{ + if (appledwc->fixed_hub) + mod_delayed_work(system_freezable_wq, &appledwc->hub_check, + msecs_to_jiffies(DWC3_APPLE_HUB_CHECK_MS)); +} + +/* + * Bring a controller up again after a sleep that stopped it or lost it. A fixed-hub host + * starts as at probe, where the core has no role yet, so dwc3_core_soft_reset() runs too: + * rebinding the driver brought the J700's hub back where a restart without it didn't. + */ +static int dwc3_apple_start_after_sleep(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +{ + int ret; + + lockdep_assert_held(&appledwc->lock); + if (appledwc->fixed_hub) + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, state); + if (ret) { + dev_err(appledwc->dev, "Failed to start after sleep, err=%d\n", ret); + return ret; + } + dwc3_apple_check_hub_later(appledwc); + return 0; +} + +/* Whether the fixed hub is on its USB2 root port. Takes the xHCI device's lock. */ +static bool dwc3_apple_hub_present(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root; + struct usb_hcd *hcd; + bool present = true; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return true; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + present = usb_hub_find_child(root, appledwc->hub_port); + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return present; +} + +/* Whether @udev or a device below it can wake the system. Takes the hubs' locks. */ +static bool dwc3_apple_can_wake(struct usb_device *udev) +{ + struct usb_device *child; + bool wake = device_may_wakeup(&udev->dev); + int port1; + + if (wake || !udev->maxchild) + return wake; + + usb_lock_device(udev); + usb_hub_for_each_child(udev, port1, child) { + wake = dwc3_apple_can_wake(child); + if (wake) + break; + } + usb_unlock_device(udev); + + return wake; +} + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The hub's own setting doesn't count. Takes the xHCI device's lock and the hubs'. + */ +static unsigned long dwc3_apple_hub_wake_ports(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root, *hub, *child; + unsigned long ports = 0; + struct usb_hcd *hcd; + int port1; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return 0; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + hub = usb_hub_find_child(root, appledwc->hub_port); + if (hub) { + usb_lock_device(hub); + usb_hub_for_each_child(hub, port1, child) { + if (port1 < BITS_PER_LONG && dwc3_apple_can_wake(child)) + ports |= BIT(port1); + } + usb_unlock_device(hub); + } + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return ports; +} + +/* + * Once after a sleep: if the hub isn't there, stop the host and start it again, with the + * hub's link down for a moment first, as unbinding and binding the driver does. This also + * catches an xHCI that didn't come back after the core was set up again. + */ +static void dwc3_apple_hub_check(struct work_struct *work) +{ + struct dwc3_apple *appledwc = container_of(to_delayed_work(work), struct dwc3_apple, + hub_check); + int ret; + + guard(mutex)(&appledwc->lock); + if (appledwc->shutting_down || appledwc->state != DWC3_APPLE_HOST || + dwc3_apple_hub_present(appledwc)) + return; + + dev_warn(appledwc->dev, "hub not back after the sleep: starting the host again\n"); + ret = dwc3_apple_exit(appledwc); + if (ret) { + dev_err(appledwc->dev, "Failed to stop the host again, err=%d\n", ret); + return; + } + msleep(500); + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); + if (ret) + dev_err(appledwc->dev, "Failed to start the host again, err=%d\n", ret); +} + static int dwc3_apple_exit(struct dwc3_apple *appledwc) { lockdep_assert_held(&appledwc->lock); @@ -821,6 +1113,12 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role ro if (appledwc->fixed_hub) return 0; + /* Stopped for a sleep: this is what comes back after it. */ + if (appledwc->sleep_stopped) { + appledwc->sleep_state = dwc3_apple_state_for_role(role); + return 0; + } + /* * Stay off instead of starting device mode (see dwc3_apple_host_only()). Reporting success * keeps the port controller from retrying: get() then reports USB_ROLE_NONE. @@ -875,7 +1173,7 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) guard(mutex)(&appledwc->lock); - switch (appledwc->state) { + switch (appledwc->sleep_stopped ? appledwc->sleep_state : appledwc->state) { case DWC3_APPLE_HOST: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: @@ -905,11 +1203,184 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } -static bool dwc3_apple_has_fixed_hub(struct device *dev) +/* + * After a sleep in place: the PHY says whether the core came back to D0, and the GUID register + * whether it kept its registers. A core set up again at the resume kept its host and devices, + * which reset-resumed. Otherwise restart the host as after a stop, and the devices enumerate + * again. With a core that may be unpowered even that would touch it, so USB stays off, and + * later sleeps are refused, as xHCI's suspend would touch it too. Returns whether the host + * was kept. + */ +static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 guid = 0; + int ret; + + appledwc->sleep_in_place = false; + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + if (ret == -ENODEV) { + appledwc->sleep_dead = true; + dev_err(appledwc->dev, "the core didn't come back from the sleep: USB off until a reboot\n"); + return false; + } + if (appledwc->sleep_reinit) { + appledwc->sleep_reinit = false; + dev_dbg(appledwc->dev, "host kept through the sleep, set up again\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + if (!ret || ret == -EBUSY) { + guid = dwc3_readl(dwc, DWC3_GUID); + if (guid == DWC3_APPLE_SLEEP_GUID) { + dwc3_writel(dwc, DWC3_GUID, appledwc->sleep_guid); + dev_dbg(appledwc->dev, "host kept through the sleep\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + ret = -EIO; + } + + dev_warn(appledwc->dev, "host lost its state in the sleep (%d, GUID %#x): restarting it\n", + ret, guid); + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after sleep, err=%d\n", ret); + else + dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + + return false; +} + +/* + * Runs before the freezer, when the sleep's real target isn't known yet, so this goes by the + * state "mem" means now. Writing "freeze" with deep selected costs a USB restart and nothing else. + * A stop that fails refuses the sleep. PM_POST_SUSPEND follows an abandoned sleep as well. + */ +static int dwc3_apple_pm_notify_locked(struct dwc3_apple *appledwc, unsigned long action, + bool in_place, bool *finish, bool *kept) +{ + struct device *dev = appledwc->dev; + int ret; + + lockdep_assert_held(&appledwc->lock); + + switch (action) { + case PM_SUSPEND_PREPARE: + if (pm_suspend_default_s2idle() || appledwc->shutting_down) { + *finish = in_place; + *kept = true; + break; + } + if (appledwc->sleep_dead) { + dev_err(dev, "refusing the sleep: the core is gone until a reboot\n"); + return notifier_from_errno(-EIO); + } + /* xHCI suspends and resumes as usual, and the wakeup setting stays. */ + if (in_place) { + appledwc->sleep_guid = dwc3_readl(&appledwc->dwc, DWC3_GUID); + dwc3_writel(&appledwc->dwc, DWC3_GUID, DWC3_APPLE_SLEEP_GUID); + appledwc->sleep_in_place = true; + /* For the port controllers' USB wake. Changes after this go unseen. */ + apple_sleep_usb_set_wake_ports(dwc3_apple_hub_wake_ports(appledwc)); + break; + } + appledwc->sleep_state = appledwc->state; + ret = dwc3_apple_exit(appledwc); + if (ret) + return notifier_from_errno(ret); + /* A wakeup path would keep the domain on, and it's going away. */ + appledwc->sleep_wakeup = device_may_wakeup(dev); + device_set_wakeup_enable(dev, false); + appledwc->sleep_stopped = true; + break; + case PM_POST_SUSPEND: + if (appledwc->sleep_in_place) { + apple_sleep_usb_set_wake_ports(0); + *kept = dwc3_apple_sleep_kept(appledwc); + *finish = true; + break; + } + if (!appledwc->sleep_stopped) + break; + appledwc->sleep_stopped = false; + device_set_wakeup_enable(dev, appledwc->sleep_wakeup); + if (appledwc->shutting_down || + (appledwc->sleep_state != DWC3_APPLE_HOST && + appledwc->sleep_state != DWC3_APPLE_DEVICE)) + break; + dwc3_apple_start_after_sleep(appledwc, appledwc->sleep_state); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +/* + * The PHY's sleep prepare and finish may take the SuperSpeed port down and up, under the + * controller's device lock, so they run outside the glue's lock. + */ +static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct dwc3_apple *appledwc = container_of(nb, struct dwc3_apple, pm_nb); + bool in_place = false, finish = false, kept = false; + int ret; + + /* + * The last wake's SuperSpeed restore ends before this sleep looks at the PHY. The hub + * check takes the glue's lock, and has nothing to look at until the sleep is over. + */ + if (action == PM_SUSPEND_PREPARE) { + flush_work(&appledwc->sleep_finish); + cancel_delayed_work_sync(&appledwc->hub_check); + } + if (action == PM_SUSPEND_PREPARE && !pm_suspend_default_s2idle()) { + scoped_guard(mutex, &appledwc->lock) + in_place = appledwc->fixed_hub && appledwc->state == DWC3_APPLE_HOST && + !appledwc->shutting_down && !appledwc->sleep_dead; + in_place = in_place && !apple_atcphy_sleep_prepare(appledwc->usb2_phy); + } + + scoped_guard(mutex, &appledwc->lock) + ret = dwc3_apple_pm_notify_locked(appledwc, action, in_place, &finish, &kept); + + /* After a wake, user space doesn't wait for SuperSpeed: it comes back in the background. */ + if (finish && action == PM_POST_SUSPEND) { + WRITE_ONCE(appledwc->sleep_finish_kept, kept); + queue_work(system_unbound_wq, &appledwc->sleep_finish); + } else if (finish) { + apple_atcphy_sleep_finish(appledwc->usb2_phy, kept); + } + + return ret; +} + +static void dwc3_apple_sleep_finish(struct work_struct *work) +{ + struct dwc3_apple *appledwc = container_of(work, struct dwc3_apple, sleep_finish); + + apple_atcphy_sleep_finish(appledwc->usb2_phy, READ_ONCE(appledwc->sleep_finish_kept)); +} + +static void dwc3_apple_pm_notifier_remove(void *data) +{ + struct dwc3_apple *appledwc = data; + + unregister_pm_notifier(&appledwc->pm_nb); +} + +/* A fixed hub's node gives the root port it sits on in its reg, 1 if it doesn't. */ +static bool dwc3_apple_has_fixed_hub(struct device *dev, u32 *port) { struct device_node *hub __free(device_node) = of_get_available_child_by_name(dev->of_node, "hub"); + *port = 1; + if (hub) + of_property_read_u32(hub, "reg", port); + return hub; } @@ -924,7 +1395,7 @@ static int dwc3_apple_probe(struct platform_device *pdev) return -ENOMEM; appledwc->dev = &pdev->dev; - appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev); + appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev, &appledwc->hub_port); appledwc->host_only = dwc3_apple_host_only(dev); appledwc->dt_dis_u3_susphy = device_property_read_bool(dev, "snps,dis_u3_susphy_quirk"); /* The PHY driver and the DARTs follow runtime PM on these SoCs. */ @@ -995,6 +1466,19 @@ static int dwc3_apple_probe(struct platform_device *pdev) * details. */ appledwc->state = DWC3_APPLE_PROBE_PENDING; + ret = devm_delayed_work_autocancel(dev, &appledwc->hub_check, dwc3_apple_hub_check); + if (ret) + return ret; + ret = devm_work_autocancel(dev, &appledwc->sleep_finish, dwc3_apple_sleep_finish); + if (ret) + return ret; + appledwc->pm_nb.notifier_call = dwc3_apple_pm_notify; + ret = register_pm_notifier(&appledwc->pm_nb); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dwc3_apple_pm_notifier_remove, appledwc); + if (ret) + return ret; /* * Until the first cable there is no core to keep powered: let the controller * runtime-suspend, and its DARTs with it, so that its power domain can drop. @@ -1044,6 +1528,7 @@ static void dwc3_apple_remove(struct platform_device *pdev) guard(mutex)(&appledwc->lock); + appledwc->shutting_down = true; usb_role_switch_unregister(appledwc->role_sw); /* diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 89e1c0c492dbe..652942d3f6f4f 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -79,8 +79,20 @@ static void dwc3_xhci_plat_start(struct usb_hcd *hcd) dwc3_enable_susphy(dwc, true); } +static int dwc3_xhci_plat_resume_quirk(struct usb_hcd *hcd) +{ + struct platform_device *pdev = to_platform_device(hcd->self.controller); + struct dwc3 *dwc = dev_get_drvdata(pdev->dev.parent); + + if (!usb_hcd_is_primary_hcd(hcd) || !dwc->glue_ops || !dwc->glue_ops->pre_host_resume) + return 0; + + return dwc->glue_ops->pre_host_resume(dwc); +} + static const struct xhci_plat_priv dwc3_xhci_plat_quirk = { .plat_start = dwc3_xhci_plat_start, + .resume_quirk = dwc3_xhci_plat_resume_quirk, }; static void dwc3_host_fill_xhci_irq_res(struct dwc3 *dwc, diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index dd1e3ef800e31..9fb820269d768 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -650,11 +650,12 @@ static int xhci_plat_resume_common(struct device *dev, bool power_lost) } } + /* A positive result: the controller lost its state, which the quirk set up again. */ ret = xhci_priv_resume_quirk(hcd); - if (ret) + if (ret < 0) goto disable_clks; - ret = xhci_resume(xhci, power_lost || priv->power_lost, false); + ret = xhci_resume(xhci, power_lost || priv->power_lost || ret > 0, false); if (ret) goto disable_clks; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 25325dde72a8c..3f748c24d458e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -1577,6 +1579,51 @@ static void cd321x_prepare(struct tps6598x *tps) /* Gate scheduling first; neither worker may be drained under its mutex. */ cancel_delayed_work_sync(&cd321x->resume_work); cancel_delayed_work_sync(&cd321x->update_work); + + /* As macOS does a second or so before the commit, measured on the same controller. */ + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) { + int ret; + + guard(mutex)(&tps->lock); + ret = tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S3); + if (ret) + dev_warn(tps->dev, "didn't take system power state S3: %d\n", ret); + else + dev_dbg(tps->dev, "system power state S3\n"); + } +} + +/* + * A sleep cut this port's debug link. Rearm it the way a boot loader does: unlock with + * the board's key, debug mode on, enable the debug VDM, debug mode off, lock. + */ +static void cd321x_rearm_debug_link(struct tps6598x *tps) +{ + static const u8 vdm[] = { 0x06, 0x46, 0x82, 0x01 }; + static const u8 no_key[4]; + const char *key; + u8 on = 1, off = 0, out = 0; + int ret; + + if (device_property_read_string(tps->dev, "apple,debug-lock-key", &key) || + strlen(key) != sizeof(no_key) || !apple_sleep_debug_link_was_cut(tps->dev)) + return; + + ret = tps6598x_exec_cmd(tps, "LOCK", sizeof(no_key), (const u8 *)key, 1, &out); + if (!ret && !(out & 0xf)) + ret = tps6598x_exec_cmd(tps, "DBMa", 1, &on, 1, &out); + if (!ret && !(out & 0xf)) + ret = tps6598x_exec_cmd(tps, "DVEn", sizeof(vdm), vdm, 1, &out); + if (!ret && (out & 0xf)) + ret = -EIO; + if (!ret) + ret = tps6598x_exec_cmd(tps, "DBMa", 1, &off, 0, NULL); + if (!ret) + ret = tps6598x_exec_cmd(tps, "LOCK", sizeof(no_key), no_key, 0, NULL); + if (ret) + dev_warn(tps->dev, "debug link not rearmed: %d\n", ret); + else + dev_dbg(tps->dev, "debug link rearmed\n"); } static void cd321x_complete(struct tps6598x *tps) @@ -1584,6 +1631,17 @@ static void cd321x_complete(struct tps6598x *tps) struct cd321x *cd321x = container_of(tps, struct cd321x, tps); guard(mutex)(&tps->lock); + /* + * The cut was a cold reset of this controller's firmware: its interrupt mask and power + * state are back at their defaults, so set them up again whether or not an event comes. + */ + if (apple_sleep_debug_link_cut_peek(tps->dev)) { + cd321x->pm.setup_pending = true; + cd321x->reset_by_cut = true; + } + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + cd321x_rearm_debug_link(tps); if (cd321x_pm_resume(&cd321x->pm)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } @@ -1653,8 +1711,11 @@ static int cd321x_check_setup(struct cd321x *cd321x, u64 event) dev_err(tps->dev, "controller setup recovery exhausted: %d\n", ret); return ret; } - if (ret) + if (ret && cd321x->reset_by_cut) + dev_dbg(tps->dev, "set up again after the debug link's cut\n"); + else if (ret) dev_warn(tps->dev, "restored controller power state and interrupt mask\n"); + cd321x->reset_by_cut = false; /* Only a fresh snapshot may be applied after restoring the controller. */ cd321x->state_valid = false; cd321x->pm.phase = CD321X_PM_REVALIDATE; @@ -3197,6 +3258,87 @@ void tipd_remove(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_remove); +/* + * macOS tells the port controller of a USB root port with dock-remote-wake when that port + * sleeps, and the controller then watches the USB2 lines for a remote wakeup (static; what its + * firmware does with it isn't known). The J700's root port has none, but measured on a J700, + * a USB keyboard on a connector wakes it this way. The mode goes by the speed on the port: 0x09 + * for full and high speed. At the wake the controller hands the lines back before the port + * resumes. Only a firmware-driven sleep needs this, and a failure only logs. + */ +#define TIPD_USBW_HIGH_SPEED 0x09 +#define TIPD_USBW_HAND_BACK 0x80 +#define TIPD_USBW_OFF 0x00 + +static void tipd_usbw(struct tps6598x *tps, u8 mode) +{ + int ret; + + guard(mutex)(&tps->lock); + ret = tps6598x_exec_cmd(tps, "USBw", sizeof(mode), &mode, 0, NULL); + if (ret) + dev_warn(tps->dev, "USBw %#x failed: %d\n", mode, ret); + else + dev_dbg(tps->dev, "USBw %#x\n", mode); +} + +/* The fixed hub's port that the connector's USB2 pair goes to, from the graph, or 0. */ +static u32 tipd_hub_port(struct tps6598x *tps) +{ + struct fwnode_handle *con __free(fwnode_handle) = + device_get_named_child_node(tps->dev, "connector"); + struct fwnode_handle *ep __free(fwnode_handle) = + fwnode_graph_get_endpoint_by_id(con, 0, 0, 0); + struct fwnode_handle *port __free(fwnode_handle) = fwnode_graph_get_remote_port(ep); + u32 reg = 0; + + fwnode_property_read_u32(port, "reg", ®); + return reg; +} + +void tipd_usb_wake(struct tps6598x *tps, bool asleep) +{ + bool armed; + + if (!pm_suspend_via_firmware() || + !device_property_read_bool(tps->dev, "apple,usb-remote-wake")) + return; + if (asleep) { + u32 port = tipd_hub_port(tps); + bool wake = apple_sleep_usb_port_can_wake(port); + + /* + * Only for a device on this connector that can wake. Armed without one, the + * controller reported a USB2 plug at the sleep's entry and the sleep came straight + * back, on an empty port and on one left with a passive adapter (measured on a + * J700). A device plugged in during the sleep still wakes through the SMC's plug + * event. + */ + if (wake) + dev_dbg(tps->dev, "a device on hub port %u can wake: arming USB wake\n", + port); + else + dev_dbg(tps->dev, "nothing on hub port %u can wake: USB wake not armed\n", + port); + scoped_guard(mutex, &tps->lock) + tps->usbw_armed = wake; + if (wake) + tipd_usbw(tps, TIPD_USBW_HIGH_SPEED); + return; + } + /* Out of S3 first: .complete would only come once the port has resumed. */ + scoped_guard(mutex, &tps->lock) { + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + armed = tps->usbw_armed; + tps->usbw_armed = false; + } + if (!armed) + return; + tipd_usbw(tps, TIPD_USBW_HAND_BACK); + tipd_usbw(tps, TIPD_USBW_OFF); +} +EXPORT_SYMBOL_GPL(tipd_usb_wake); + int tipd_suspend(struct tps6598x *tps) { if (tps->wakeup) { diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 30d472c3d83a4..bbe2ef971b35f 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -855,8 +855,11 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) } else { err = sn201202x_wake(sn); } - if (!err) + if (!err) { + /* Registered before xHCI, so this is before the root port resumes. */ + tipd_usb_wake(tps, false); err = tipd_resume(tps); + } /* The resume revalidation acknowledges events latched while asleep. */ if (tps->irq) enable_irq(tps->irq); @@ -906,6 +909,8 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) else dev_warn(dev, "port state unreadable, no wakeup for an attach\n"); } + /* Registered before xHCI, so its root port has suspended by now. */ + tipd_usb_wake(tps, true); reinit_completion(&sn->sleep_completion); err = spmi_command_sleep(sn->sdev); if (err) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 20b6b73436a29..5aae06c95b060 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -395,6 +395,8 @@ struct tps6598x { struct tipd_debugfs *debugfs; struct typec_altmode *partner_altmode_dp; + /* USB remote wake armed on this port for the sleep (USBw). */ + bool usbw_armed; }; /* TPS_REG_USB4_STATUS */ @@ -466,6 +468,8 @@ struct cd321x { atomic_t ready_event; atomic_t restore_event; bool display_route_active; + /* The sleep's debug link cut reset this controller: its setup is expected. */ + bool reset_by_cut; bool dp_sink_reported; bool dp_irq_hpd; @@ -522,6 +526,7 @@ struct cd321x_sleep_state { }; int cd321x_sleep_events(struct tps6598x *tps, struct cd321x_sleep_state *st); +void tipd_usb_wake(struct tps6598x *tps, bool asleep); int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, size_t in_len, const u8 *in_data, size_t out_len, u8 *out_data, diff --git a/include/linux/irqchip/irq-apple-aic.h b/include/linux/irqchip/irq-apple-aic.h new file mode 100644 index 0000000000000..3c3781322a52b --- /dev/null +++ b/include/linux/irqchip/irq-apple-aic.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_IRQCHIP_IRQ_APPLE_AIC_H +#define _LINUX_IRQCHIP_IRQ_APPLE_AIC_H + +#include + +#ifdef CONFIG_APPLE_AIC +void aic_sleep_mask_all(void); +void aic_sleep_restore(void); +u32 aic_sleep_ack(void); +#else +static inline void aic_sleep_mask_all(void) { } +static inline void aic_sleep_restore(void) { } +static inline u32 aic_sleep_ack(void) { return 0; } +#endif + +#endif diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index fbc047b00afd8..bf8781ba4c52e 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -112,6 +112,15 @@ struct apple_smc { struct apple_rtkit *rtk; + /* Between the AP power notices of a firmware-driven suspend. */ + bool ap_notified; + /* Told of a coming sleep, and not yet of its end. */ + bool sleep_warned; + /* In a noirq phase: replies are polled for. */ + bool polled; + /* MBSW at the last wake: what the SMC had woken the AP for. */ + u64 wake_record; + struct completion init_done; enum apple_smc_boot_stage boot_stage; diff --git a/include/linux/phy/apple-atc.h b/include/linux/phy/apple-atc.h new file mode 100644 index 0000000000000..180e97e6e8eaf --- /dev/null +++ b/include/linux/phy/apple-atc.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What the Apple Type-C PHY offers the USB controller it serves, beyond the + * generic PHY operations. + */ +#ifndef _LINUX_PHY_APPLE_ATC_H_ +#define _LINUX_PHY_APPLE_ATC_H_ + +#include +#include + +struct phy; + +#if IS_REACHABLE(CONFIG_PHY_APPLE_ATC) +/* + * For a firmware-driven system sleep, before any device suspends and without the + * controller's locks held. 0 means the PHY takes the controller through hibernation after + * xHCI has saved its state and brings it back before xHCI resumes, so the host can stay + * up; with USB3 its PIPE is parked for it. Otherwise the controller has to be stopped for + * the sleep. + */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy); + +/* + * After the sleep: 0 if the controller is in D0. -ENODEV if it may be unpowered, so that its + * registers must not be touched. Any other error: it's in D0, but the PHY isn't set up for it. + */ +int apple_atcphy_sleep_result(struct phy *usb2_phy); + +/* + * Ends a sleep that prepare took on, with the SuperSpeed side back as the mux wants it. + * Without @kept the host was restarted; only a failure of the PHY's own then makes the + * controller stop for later sleeps. + */ +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept); +#else +static inline int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ +} +#endif + +#endif diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 10d05a7cc042b..95af6ada083d8 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -153,12 +153,69 @@ int apple_dart_save_tunnel_state(struct device *dev); int apple_dart_restore_tunnel_state(struct device *dev); /* Registered J713 managed radio provider, with no owned valid root or fault. */ int apple_dart_radio_check_idle(struct device *dev); +/* + * A DART clocked by its host's port: after a real sleep it stays gated until the host has + * set the port up and calls apple_dart_host_resume(). + */ +int apple_dart_set_host_resumes(struct device *dev); +int apple_dart_host_resume(struct device *dev); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_radio_check_idle(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_set_host_resumes(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_host_resume(struct device *dev) { return -EOPNOTSUPP; } +#endif + +/* For a consumer whose firmware gates itself around a firmware-driven suspend. */ +#if IS_REACHABLE(CONFIG_APPLE_DART) +int apple_dart_sleep_prepare(struct device *dev); +void apple_dart_sleep_restore(struct device *dev); +/* A display DART's streams around a display power cycle, see the DART driver. */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush); +/* A GAPF slice released before its coprocessor's CPU domain comes up after a wake. */ +int apple_dart_gapf_ready(struct device *dev); +/* A display DART whose domain goes off at runtime while others still map into it. */ +void apple_dart_power_dark(struct device *dev, bool dark); +/* A DART's state kept across its consumer's own power cycle, outside a system sleep. */ +int apple_dart_state_save(struct device *dev); +int apple_dart_state_restore(struct device *dev); +#else +static inline int apple_dart_sleep_prepare(struct device *dev) +{ + return -ENODEV; +} + +static inline void apple_dart_sleep_restore(struct device *dev) +{ +} + +static inline int apple_dart_display_streams(struct device *dev, u32 streams, bool on, + bool flush) +{ + return -ENODEV; +} + +static inline int apple_dart_gapf_ready(struct device *dev) +{ + return 0; +} + +static inline void apple_dart_power_dark(struct device *dev, bool dark) +{ +} + +static inline int apple_dart_state_save(struct device *dev) +{ + return -ENODEV; +} + +static inline int apple_dart_state_restore(struct device *dev) +{ + return -ENODEV; +} #endif #endif diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index d7fc9dd4134f1..b689316b2083d 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -100,8 +100,12 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); +/* For a caller with interrupts off, where apple_mbox_send()'s own deadline can't pass. */ +bool apple_mbox_send_ready(struct apple_mbox *mbox); int apple_mbox_restore(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* Bit 0 of the receive control word, for firmware that powers itself down. */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled); /* * Leave received messages in the hardware FIFO, without an interrupt, until * apple_mbox_release_rx(). For consumers that must not wake the system. @@ -123,6 +127,14 @@ int apple_mbox_ascwrap_v6_get_lifecycle( int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); +/* + * For a CPU that a system sleep's last step stops with interrupts off: its control word, for + * polling its status, and the stop's register writes alone. Once the sleep is over, + * apple_mbox_ascwrap_v6_cpu_stopped() settles the books without touching the registers. + */ +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox); +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, u32 *control, u32 *status); diff --git a/include/linux/soc/apple/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h index 3b2971b8b1db4..3a496a2040989 100644 --- a/include/linux/soc/apple/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -5,16 +5,30 @@ #include #include +struct module; struct pci_dev; #if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) /* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); +/* A function that a sleep left unusable asks for both to be restarted from cold. */ +void apple_piodma_radio_recover(void); +/* What the host calls before it removes Wi-Fi for that. A failure keeps it bound. */ +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner); +#define apple_piodma_radio_set_stop(stop) __apple_piodma_radio_set_stop(stop, THIS_MODULE) #else static inline int apple_piodma_radio_check(struct pci_dev *pdev) { return -ENODEV; } + +static inline void apple_piodma_radio_recover(void) +{ +} + +static inline void apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev)) +{ +} #endif #endif /* _LINUX_SOC_APPLE_PCI_PIODMA_H */ diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 66e97565edc1e..77fed7a9e7edd 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -24,6 +24,11 @@ int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); +void apple_pmp_report_sleep(struct device *pmp, bool at_commit); +void apple_pmp_report_wake(struct device *pmp, bool lost_power); +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable); +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned); +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry); /* * Set or clear one bit of the PMP power request word of the report bound to @report_np, under the * report's lock, with the acknowledge wait (see pmp-report.c). @@ -40,6 +45,31 @@ void apple_pmp_agx_activity_end(void); void apple_pmp_agx_activity_release(void); void apple_pmp_agx_activity_resume(void); #else +static inline int apple_pmp_report_entry_request(const struct device_node *entry, + bool enable) +{ + return 0; +} + +static inline int apple_pmp_report_entry_commit_owned(const struct device_node *entry, + bool owned) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + return 0; +} + +static inline void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ +} + +static inline void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ +} + static inline int apple_pmp_report_ane_available(const struct device_node *np) { return -EOPNOTSUPP; diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 8b329bf5d4dd5..8663ca195febe 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -5,6 +5,7 @@ #include #include struct device; +struct dev_pm_ops; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); @@ -25,8 +26,27 @@ static inline int apple_pmp_t6030_gpu_ready(void) } #endif /* Registration is private to the PMP Rust driver and its C bridge. */ +bool apple_pmp_keep_bound(void); int apple_pmp_register(struct device *dev, const void *data); void apple_pmp_unregister(const void *data); int apple_pmp_send_power_command(const void *data, u8 command, u16 device_id, u32 enabled); +extern const struct dev_pm_ops apple_pmp_pm_ops; +int apple_pmp_sleep(const void *data); +int apple_pmp_wake(const void *data); + +/* + * What the sleep's last step needs to idle and stop the PMP with interrupts off: its RTKit and + * CPU control, and where the SRAM it keeps goes (no range: sram NULL). Filled in suspend_late. + */ +struct apple_rtkit; +struct apple_pmp_commit_stop { + struct apple_rtkit *rtk; + void __iomem *control; + void __iomem *sram; + u32 *words; + size_t count; +}; + +int apple_pmp_sleep_prepare(const void *data, struct apple_pmp_commit_stop *stop); #endif diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index ac646240f8587..dcd2474b4369f 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -215,6 +215,9 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* apple_rtkit_boot(), calling @before_ap between the IOP's report and the AP going on. */ +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data); + /* * Mark an instance created with apple_rtkit_init() as running without a * handshake, see apple_rtkit_init_adopted(). @@ -259,6 +262,92 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk); */ int apple_rtkit_idle(struct apple_rtkit *rtk); +/* + * Turn the mailbox off for a co-processor that has acknowledged idle, before its + * run bit and power state go. Only apple_rtkit_warm_restart() follows this. + */ +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk); + +/* + * The same with interrupts off and one CPU up, for a co-processor that doesn't come + * back from idle without losing power, so that its idle has to follow every refusal. + * @us is about how long the two acknowledgements took. Only + * apple_rtkit_warm_restart() follows this, once the sleep has reached the hardware. + */ +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us); + +/* + * Only the mailbox's receive bit, for the moment before that co-processor's run bit + * goes. Safe with interrupts off, and apple_rtkit_warm_restart() sets it again. + */ +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk); + +/* + * Prepare for a firmware being restarted with its buffers kept. + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk); + +/* + * For a firmware whose last endpoint map is answered, on a warm restart, with + * only the endpoints in @last_mask rather than an echo of all of them. + */ +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask); + +/* + * Tell a co-processor that keeps running across a system sleep about the AP's + * power state, without changing its own: idle before the sleep and on after. + */ +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on); + +/* The same, for a noirq phase: device interrupts are off, so it polls. */ +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on); + +/* + * Stop a co-processor for a system sleep that may remove its power: idle, wait + * for the status that shows it has finished, then clear the run bit. On an error + * the run bit is still set and the firmware still runs. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Bring back a co-processor taken down for a system sleep with + * apple_rtkit_sleep_stop() or apple_rtkit_idle(), from a process context with its + * mailbox's supplier resumed. + * + * Where the sleep reached the hardware (pm_resume_via_firmware()) the firmware + * restarts and keeps its buffers and endpoints: this restarts the receive path, sets + * the run bit, echoes the endpoint map and takes the IOP state it reports unasked, + * then sets the AP state. Where it didn't, one that idled gets -EAGAIN: an idled + * firmware doesn't wake with its power kept, so it stays down, and comes back the + * same way after the next sleep that does reach the hardware. One whose idle + * failed still runs, and gets 0. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * A start as after a real sleep, for a co-processor its driver power-cycled without one: + * the warm restart, the run bit set, then RTKit's handshake. + */ +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * The first half of apple_rtkit_sleep_return() after a real sleep: the restart and the run + * bit, with no wait. apple_rtkit_boot() then waits for the firmware to report on, so other + * firmware can be started in between. -EOPNOTSUPP after a sleep that kept the power. + */ +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Echo each OSLOG channel-0 counter the firmware sends, as macOS does. On by default on + * T8140, where a firmware with one outstanding takes no AP power notice. + */ +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo); + /* * Checks if RTKit is running and ready to handle messages. */ @@ -301,6 +390,21 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * A port that only answers the co-processor, polled, with no interrupts. Map answers + * its HELLO and endpoint map, and on brings the AP side of the port up. Handshake is + * both. Idle takes the AP side down. Restart prepares for a co-processor that was + * stopped. + */ +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_restart(struct apple_rtkit *rtk); + +/* apple_rtkit_idle(), calling @after_ap between the AP's idle and the IOP's. */ +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data); + /* * Stop taking interrupts for messages from the co-processor, which then * stay queued in the mailbox until apple_rtkit_resume_rx(). diff --git a/include/linux/soc/apple/sart.h b/include/linux/soc/apple/sart.h index f5d0db5022831..53186e759e55d 100644 --- a/include/linux/soc/apple/sart.h +++ b/include/linux/soc/apple/sart.h @@ -64,4 +64,10 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, size_t size); +/* + * After a firmware-driven sleep, writes the entries in use before it back again and reads + * each back. Returns how many didn't hold, or -EOPNOTSUPP. + */ +int apple_sart_replay(struct apple_sart *sart); + #endif /* _LINUX_SOC_APPLE_SART_H_ */ diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h new file mode 100644 index 0000000000000..c167ae877030b --- /dev/null +++ b/include/linux/soc/apple/sleep.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What drivers tell the code that takes an Apple SoC into a firmware-driven + * suspend. + */ +#ifndef _LINUX_SOC_APPLE_SLEEP_H_ +#define _LINUX_SOC_APPLE_SLEEP_H_ + +#include +#include +#include +#include + +struct device; + +#if IS_ENABLED(CONFIG_APPLE_T8140_SLEEP) +/* + * A coprocessor that can't be stopped until a sleep is certain, and can't be + * restarted without one. While one is registered and not ready, a suspend is + * refused. Once ready, the last step before the commit runs @last_stop, with + * interrupts off and every refusal behind it, then clears its run bit. With + * @pcie_ans, that also lowers the PCIe and ANS domains, which nothing else may do + * under it. @last_stop can't refuse: a nonzero return means it failed, and the sleep + * goes on. + */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data); + +/* + * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power + * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. + */ +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data); +void apple_sleep_coproc_unregister(struct device *dev); + +/* + * For a registered coprocessor: the last of its stop that may still refuse the sleep, run + * with interrupts off before any coprocessor's @last_stop. A negative return refuses, and the + * coprocessor must still be able to carry on. + */ +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)); + +/* + * The enclave's sleep request, which it comes back from only through a real sleep: @tell + * runs with interrupts off as the very last refusal, after every other one and before any + * coprocessor stops. Nonzero refuses the sleep. + */ +int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), void *data); +void apple_sleep_enclave_unregister(struct device *dev); + +/* Its driver has quiesced it for that last step, or has taken it back. */ +void apple_sleep_coproc_set_ready(struct device *dev, bool ready); + +/* Whether the last step stopped it: after a refused sleep, false means it still runs. */ +bool apple_sleep_coproc_stopped(struct device *dev); + +/* + * The USB host sleeps in place for this sleep, as macOS's does: its always-on domain stays + * up through the commit. Its PHY sets it once the controller is in its hibernation state, + * and clears it at the wake. + */ +void apple_sleep_usb_set_asleep(bool asleep); + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The host that sleeps in place sets them before the freeze and clears them after. + */ +void apple_sleep_usb_set_wake_ports(unsigned long ports); + +/* For the port controller wired to the fixed hub's @port, as it arms USB wake. */ +bool apple_sleep_usb_port_can_wake(unsigned int port); + +/* The last sleep ended the debug link on @dev, whose driver may rearm it. */ +bool apple_sleep_debug_link_was_cut(struct device *dev); + +/* The same, without taking it: the port controller's own setup after its cold reset. */ +bool apple_sleep_debug_link_cut_peek(struct device *dev); + +/* + * End the debug link now, if this sleep hasn't: true if this call did. For the SMC's driver, + * ahead of its AP idle notice. It takes over a second, and doesn't sleep. + */ +bool apple_sleep_debug_link_cut(void); + +/* + * From the SMC's driver, in its noirq resume: the SMC woke the AP for the lid alone, and the + * lid reads closed. If nothing else woke the SoC, the PM core goes back to sleep once every + * device has resumed, as macOS does from a dark wake. + */ +void apple_sleep_lid_closed_wake(void); + +/* The radio's host has its port back after a real sleep, so the next commit may stop it. */ +void apple_sleep_radio_port_up(void); + +/* After a deep sleep, put back the pins it reset. ANS's driver calls it before the run bit. */ +void apple_sleep_restore_pins(void); + +/* + * After a refused sleep, for a firmware that idled on a domain that can't gate alone: the + * sleep engine's PMGR reset of its own domains, its DARTs kept. The caller starts it after. + */ +int apple_sleep_engine_reset(struct device *dev); +#else +static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, + bool pcie_ans, int (*last_stop)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_coproc_register_first(struct device *dev, + int (*last_stop)(void *data), void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_coproc_unregister(struct device *dev) +{ +} + +static inline int apple_sleep_coproc_set_last_check(struct device *dev, + int (*last_check)(void *data)) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_enclave_unregister(struct device *dev) +{ +} + +static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ +} + +static inline bool apple_sleep_coproc_stopped(struct device *dev) +{ + return false; +} + +static inline void apple_sleep_usb_set_asleep(bool asleep) +{ +} + +static inline void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ +} + +static inline bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return false; +} + +static inline bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut(void) +{ + return false; +} + +static inline void apple_sleep_lid_closed_wake(void) +{ +} + +static inline void apple_sleep_radio_port_up(void) +{ +} + +static inline void apple_sleep_restore_pins(void) +{ +} + +static inline int apple_sleep_engine_reset(struct device *dev) +{ + return -EOPNOTSUPP; +} +#endif + +#endif diff --git a/include/linux/suspend.h b/include/linux/suspend.h index b02876f1ae38a..365437ce2cc02 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -159,6 +159,7 @@ extern unsigned int pm_suspend_global_flags; #define PM_SUSPEND_FLAG_FW_SUSPEND BIT(0) #define PM_SUSPEND_FLAG_FW_RESUME BIT(1) #define PM_SUSPEND_FLAG_NO_PLATFORM BIT(2) +#define PM_SUSPEND_FLAG_DARK_WAKE BIT(3) static inline void pm_suspend_clear_flags(void) { @@ -180,6 +181,11 @@ static inline void pm_set_suspend_no_platform(void) pm_suspend_global_flags |= PM_SUSPEND_FLAG_NO_PLATFORM; } +static inline void pm_set_dark_wake(void) +{ + pm_suspend_global_flags |= PM_SUSPEND_FLAG_DARK_WAKE; +} + /** * pm_suspend_via_firmware - Check if platform firmware will suspend the system. * @@ -237,6 +243,27 @@ static inline bool pm_suspend_no_platform(void) return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_NO_PLATFORM); } +/** + * pm_dark_wake - Check if the system woke up for nothing user space has to see. + * + * To be called during system-wide power management transitions from sleep + * states. + * + * Return 'true' if the platform found that the event that woke up the system + * needs nothing from user space (for example, a laptop lid closing while the + * system slept). Every device then resumes as usual, and the system goes back + * to the same sleep state with user space still frozen. A device should not do + * what the user would notice, such as lighting a display, during such a wake. + * + * Only a wake from the sleep state can be dark. A device that had to leave out + * part of its resume finishes it in a later resume that is not dark, or in + * ->complete() if the next suspend fails before suspending it. + */ +static inline bool pm_dark_wake(void) +{ + return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_DARK_WAKE); +} + /* Suspend-to-idle state machnine. */ enum s2idle_states { S2IDLE_STATE_NONE, /* Not suspended/suspending. */ @@ -287,6 +314,8 @@ static inline void pm_set_resume_via_firmware(void) {} static inline bool pm_suspend_via_firmware(void) { return false; } static inline bool pm_resume_via_firmware(void) { return false; } static inline bool pm_suspend_no_platform(void) { return false; } +static inline void pm_set_dark_wake(void) {} +static inline bool pm_dark_wake(void) { return false; } static inline bool pm_suspend_default_s2idle(void) { return false; } static inline void suspend_set_ops(const struct platform_suspend_ops *ops) {} diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 604b9dda94317..b8bf424370bc7 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -112,6 +112,7 @@ struct usb_hcd { #define HCD_FLAG_DEAD 6 /* controller has died? */ #define HCD_FLAG_INTF_AUTHORIZED 7 /* authorize interfaces? */ #define HCD_FLAG_DEFER_RH_REGISTER 8 /* Defer roothub registration */ +#define HCD_FLAG_PHY_SUSPENDED 9 /* root hub PHYs released by a suspend */ /* The flags can be tested using these macros; they are likely to * be slightly faster than test_bit(). diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 57c44268698f7..34253036bbbd9 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -532,6 +532,9 @@ int suspend_devices_and_enter(suspend_state_t state) do { error = suspend_enter(state, &wakeup); } while (!error && !wakeup && platform_suspend_again(state)); + /* Only a wake from the sleep state can be dark. */ + if (error) + pm_suspend_global_flags &= ~PM_SUSPEND_FLAG_DARK_WAKE; Resume_devices: suspend_test_start(); @@ -616,6 +619,15 @@ static int enter_state(suspend_state_t state) trace_suspend_resume(TPS("suspend_enter"), state, false); pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); error = suspend_devices_and_enter(state); + /* + * Every device has resumed, but nothing woke the system that user space + * needs to see: go back to sleep with user space still frozen. + */ + while (!error && pm_dark_wake()) { + pr_info("dark wake: suspending again (%s)\n", mem_sleep_labels[state]); + pm_suspend_clear_flags(); + error = suspend_devices_and_enter(state); + } Finish: events_check_enabled = false; diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f763d5541472..389c67d5740ed 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -102,6 +102,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "suspend.c" #include "srcu.c" #include "sync.c" #include "task.c" diff --git a/rust/helpers/suspend.c b/rust/helpers/suspend.c new file mode 100644 index 0000000000000..a616907ecc0b5 --- /dev/null +++ b/rust/helpers/suspend.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_pm_suspend_via_firmware(void) +{ + return pm_suspend_via_firmware(); +} + +__rust_helper bool rust_helper_pm_resume_via_firmware(void) +{ + return pm_resume_via_firmware(); +} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 1f845937899db..7bce2fe29e841 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -80,6 +80,7 @@ unsafe impl driver::RegistrationOps for Adapter { (*pdrv.get()).driver.of_match_table = of_table; (*pdrv.get()).driver.acpi_match_table = acpi_table; (*pdrv.get()).driver.suppress_bind_attrs = T::suppress_bind_attrs(); + (*pdrv.get()).driver.pm = T::pm_ops().map_or(core::ptr::null(), core::ptr::from_ref); } // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. @@ -230,6 +231,11 @@ pub trait Driver { false } + /// The driver's power management callbacks, if it has any. + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + None + } + /// Platform driver probe. /// /// Called when a new platform device is added or discovered. diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index c4f86930e3c65..e0c4be586801d 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -91,11 +91,34 @@ impl Mailbox { unsafe { device::Device::from_raw((*self.mbox).dev) } } + /// The C mailbox, for code that drives it with interrupts off. + pub fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + /// Sends the specified message pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { // SAFETY: Calling the c function, `mbox` is a valid pointer to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) } + + /// Sets bit 0 of the receive control word, for firmware that powers itself down. + pub fn set_recv_enabled(&self, enabled: bool) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_set_recv_enabled(self.mbox, enabled) }; + } + + /// Stops taking receive interrupts, so that no handler reads the mailbox. + pub fn hold_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_hold_rx(self.mbox) }; + } + + /// Undoes [`Self::hold_rx`]. + pub fn release_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_release_rx(self.mbox) }; + } } impl Drop for Mailbox { diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index ac9d7ab5032b6..236948ab75b13 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -373,6 +373,26 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) } + /// Asks the coprocessor to go idle, AP then IOP state, and waits for both acknowledgments. + pub fn idle(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_idle(self.rtk) }) + } + + /// Turns the mailbox off for a coprocessor that has acknowledged idle, before its run bit and + /// power state go. Only [`Self::warm_restart`] follows this. + pub fn mailbox_off(self: Pin<&mut Self>) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_mailbox_off(self.rtk) }; + } + + /// Prepares for HELLO from firmware restarting warm, which keeps its buffers and rings. The + /// caller then sets the run bit and calls [`Self::boot`]. + pub fn warm_restart(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_warm_restart(self.rtk) }) + } + /// Starts a non-system endpoint. pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { // SAFETY: `rtk` is valid per the type invariant. @@ -387,6 +407,12 @@ impl RtKit { }) } + /// Echoes the OSLOG counters the firmware sends, or drops them. + pub fn set_oslog_echo(&self, echo: bool) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_set_oslog_echo(self.rtk, echo) }; + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs index 4d3442bbfeafe..34f9c75b17d49 100644 --- a/sound/soc/apple/aop_audio.rs +++ b/sound/soc/apple/aop_audio.rs @@ -1092,6 +1092,36 @@ kernel::of_device_table!( ] ); +/// A system sleep with a firmware behind it. The firmware keeps running, so what it runs for +/// the streams has to stop first. Other variants' firmware isn't traced around a sleep. +/// +/// # Safety +/// +/// Called by the PM core with a live device of this driver. +unsafe extern "C" fn aop_audio_suspend(dev: *mut bindings::device) -> i32 { + // SAFETY: the helper only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: the PM core passes a live device, and keeps unbind away for this call. + let dev = unsafe { device::Device::::from_raw(dev) }; + let driver = dev.get_drvdata::(); + if driver.is_null() { + return 0; + } + // SAFETY: this driver stored it at probe, and it stays until unbind. + if let AopAudio::T8140(driver) = unsafe { &*driver } { + driver.sleep_stop(); + } + 0 +} + +static AOP_AUDIO_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend: Some(aop_audio_suspend), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + /// The bound driver, as the variant set it up. enum AopAudio { /// Held for its `Drop`, which frees the card. @@ -1105,6 +1135,10 @@ impl platform::Driver for AopAudio { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_AUDIO_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/sound/soc/apple/aop_t8140.rs b/sound/soc/apple/aop_t8140.rs index aa97deda3b5c3..929aefbf54f64 100644 --- a/sound/soc/apple/aop_t8140.rs +++ b/sound/soc/apple/aop_t8140.rs @@ -62,6 +62,10 @@ use kernel::{ new_spinlock, platform, prelude::*, + regulator::{ + self, + Regulator, // + }, soc::apple::aop::{ from_fourcc, EPICService, @@ -492,6 +496,10 @@ struct SndSocT8140AopData { spkr_want_run: AtomicFlag, /// A microphone-array stream has been triggered and not yet stopped. hpai_want_run: AtomicFlag, + /// The array's rail (ADT function-input_power_enable), held while its + /// service runs. + #[pin] + hpai_supply: Mutex>>, /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. spkr_volume: Atomic, /// REG_SPKR_FULL_SCALE: the volume is below SPKR_CEILING_Q31, not the @@ -958,7 +966,11 @@ impl SndSocT8140AopData { if !self.hpai_want_run.load(Relaxed) || self.hpai.running.load(Relaxed) { return Ok(()); } - self.service_run(&self.hpai) + // As macOS does: the rail, then the stream. A failed start drops it again. + let supply = Regulator::::get(&self.dev, c"mic")?; + self.service_run(&self.hpai)?; + *self.hpai_supply.lock() = Some(supply); + Ok(()) } /// Called from sleeping close paths: a queued start must finish before @@ -972,7 +984,9 @@ impl SndSocT8140AopData { fn hpai_shutdown(&self) { self.flush_hpai_start(); - self.service_shutdown(&self.hpai, &self.hpai_domains()) + self.service_shutdown(&self.hpai, &self.hpai_domains()); + // As macOS does: the stream's stop, then the rail. + *self.hpai_supply.lock() = None; } // ---- internal speakers: the spkr service on LEAP TX0 (leap-ns ADMAC @@ -1037,6 +1051,35 @@ impl SndSocT8140AopData { Ok(()) } + /// `pw0 ` if running; the leaves stay with the stream. + fn service_stop(&self, svc: &Service) { + if svc.running.load(Relaxed) { + if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { + dev_err!(self.dev, "{}: pw0 failed: {:?}\n", svc.name, e); + } + svc.running.store(false, Relaxed); + } + } + + /// A system sleep: nothing the firmware runs may outlast the streams, whose DMA stops with + /// the sleep. Each one's next prepare or start asks for it again. + fn sleep_stop(&self) { + if self.powered.xchg(false, Relaxed) { + if let Err(e) = self.lpai_run_put() { + dev_err!(self.dev, "unable to return lpai to idle: {:?}\n", e); + } + } + self.flush_hpai_start(); + self.service_stop(&self.hpai); + *self.hpai_supply.lock() = None; + if let Err(e) = self.spkr_prepare() { + dev_err!(self.dev, "spkr: stop failed: {:?}\n", e); + } + self.service_stop(&self.tap); + self.service_stop(&self.cout); + self.service_stop(&self.cin); + } + /// Called from sleeping prepare/shutdown paths while ALSA serializes /// this stream: a queued start must finish before the mute and pw0. fn flush_spkr_start(&self) { @@ -2609,6 +2652,8 @@ impl AsocContext { pub(crate) struct SndSocT8140AopDriver { card: *mut bindings::snd_card, + /// The card's two microphone PCMs, null until created. + pcms: [*mut bindings::snd_pcm; 2], data: Arc, asoc: KBox, asoc_registered: bool, @@ -2664,6 +2709,7 @@ impl SndSocT8140AopDriver { // whatever was set up. let mut this = SndSocT8140AopDriver { card: ptr::null_mut(), + pcms: [ptr::null_mut(); 2], asoc: AsocContext::new(data.clone())?, data: data.clone(), asoc_registered: false, @@ -2777,6 +2823,7 @@ impl SndSocT8140AopDriver { (*pcm).private_free = Some(pcm_free_private); (*pcm).info_flags = 0; } + self.pcms[device as usize] = pcm; // SAFETY: as above. copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; if device == 0 { @@ -2797,6 +2844,21 @@ impl SndSocT8140AopDriver { Ok(()) } + /// A system sleep: stop the card's own streams, then whatever the firmware runs for them + /// and for the sound card's. Userspace prepares its streams again afterwards, since none + /// of them can resume. + pub(crate) fn sleep_stop(&self) { + for pcm in self.pcms.iter().copied().filter(|pcm| !pcm.is_null()) { + // SAFETY: the PCMs live as long as the card, and the driver core keeps + // unbind away from a system sleep. + let ret = unsafe { bindings::snd_pcm_suspend_all(pcm) }; + if ret < 0 { + dev_err!(self.data.dev, "unable to suspend a PCM: {}\n", ret); + } + } + self.data.sleep_stop(); + } + /// Undo probe: stop the reports, unregister the ASoC component and free /// the card (which closes every stream, releasing its channel and power /// references), then return lpai to idle. Runs from unbind, and again @@ -2889,6 +2951,9 @@ impl SndSocT8140AopDriver { // SAFETY: the pointer was checked above and stays valid with the child. let svc = unsafe { *service }; + // A capture takes the array's rail, which needs the SMC's GPIOs bound. + Regulator::::get(&dev, c"mic")?; + // The source ring: 512 KiB through our own IOMMU stream (dart-aop 9), // owned by the AOP driver because the firmware binds it once per boot. // SAFETY: no DMA is in flight on this device yet. @@ -2940,6 +3005,7 @@ impl SndSocT8140AopDriver { pd_rx0, spkr_want_run: AtomicFlag::new(false), hpai_want_run: AtomicFlag::new(false), + hpai_supply <- new_mutex!(None), spkr_volume: Atomic::new(SPKR_VOLUME_MAX), spkr_full_scale: AtomicFlag::new(false), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 18d73b50af861..9dfb329f46201 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -1467,10 +1468,41 @@ static const struct of_device_id apple_mca_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_mca_of_match); +/* + * hw_params is the only place a stream's cluster and serdes setup is written. A + * sleep that takes the registers would leave an open stream silent, and nothing + * here restores them yet. + */ +static int __maybe_unused apple_mca_suspend_late(struct device *dev) +{ + struct mca_data *mca = dev_get_drvdata(dev); + int i; + + if (!pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < mca->nclusters; i++) { + struct mca_cluster *cl = &mca->clusters[i]; + + if (cl->pd_link || cl->clocks_in_use[SNDRV_PCM_STREAM_PLAYBACK] || + cl->clocks_in_use[SNDRV_PCM_STREAM_CAPTURE]) { + dev_err(dev, "cluster %d is in use, refusing the sleep\n", i); + return -EBUSY; + } + } + + return 0; +} + +static const struct dev_pm_ops apple_mca_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_mca_suspend_late, NULL) +}; + static struct platform_driver apple_mca_driver = { .driver = { .name = "apple-mca", .of_match_table = apple_mca_of_match, + .pm = pm_sleep_ptr(&apple_mca_pm_ops), }, .probe = apple_mca_probe, .remove = apple_mca_remove, diff --git a/tools/asahi/aop-bootargs-tests.py b/tools/asahi/aop-bootargs-tests.py index d2e24ecb07eb2..fd2920a7a3353 100644 --- a/tools/asahi/aop-bootargs-tests.py +++ b/tools/asahi/aop-bootargs-tests.py @@ -90,9 +90,29 @@ def method(source): def configs(source): + """struct AopHwConfig and the HW_CFG_* constants, each with its doc comment. + + Other items between them (the sleep's PM callbacks) use kernel bindings the harness + doesn't have. + """ start = source.index('struct AopHwConfig {') end = source.index('kernel::of_device_table!', start) - return source[start:end] + out, pending, item, keep, depth = [], [], [], None, 0 + for line in source[start:end].splitlines(keepends=True): + if keep is None: + if not line.strip() or line.startswith(('//', '#[')): + pending.append(line) + continue + keep = line.startswith(('struct AopHwConfig', 'const HW_CFG_')) + item, pending, depth = pending + [line], [], 0 + else: + item.append(line) + depth += line.count('{') - line.count('}') + if depth == 0 and line.rstrip().endswith((';', '}')): + if keep: + out += item + keep = None + return ''.join(out + pending) def compile_run(rustc, p, binary, code): diff --git a/tools/asahi/apple-dart-fw/fixture.c b/tools/asahi/apple-dart-fw/fixture.c index 13e39fed942d5..c684d3c210435 100644 --- a/tools/asahi/apple-dart-fw/fixture.c +++ b/tools/asahi/apple-dart-fw/fixture.c @@ -50,7 +50,7 @@ static unsigned lock_depth; static size_t find_next_bit(unsigned long*p,size_t n,size_t a){for(;a\n' + 'static inline bool pm_suspend_via_firmware(void) { return false; }\n' + 'static inline bool pm_resume_via_firmware(void) { return false; }\n') + # The bridge's sleep-commit stop: MMIO, RTKit and the sleep driver, reached only in a suspend. + write(tmp, 'linux/io.h', '#include "mock.h"\n' + 'static inline u32 readl_relaxed(const volatile void __iomem *a) { assert(!"no system sleep here"); return 0; }\n' + 'static inline void iounmap(volatile void __iomem *a) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/of_address.h', '#include "mock.h"\n' + 'static inline void __iomem *of_iomap(struct device_node *n, int i) { assert(!"no system sleep here"); return NULL; }\n') + write(tmp, 'linux/soc/apple/rtkit.h', '#include "mock.h"\n#include \n' + 'struct apple_rtkit;\n' + 'static inline int apple_rtkit_sleep_idle_polled(struct apple_rtkit *r, void __iomem *c,\n' + '\t\t\t\t\t\tunsigned int *us) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_rtkit_mailbox_recv_off(struct apple_rtkit *r) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/soc/apple/sleep.h', '#include "mock.h"\n#include \n' + 'static inline int apple_sleep_coproc_register(struct device *d, void __iomem *a, bool p,\n' + '\t\t\t\t\t int (*f)(void *), void *x) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_sleep_coproc_set_ready(struct device *d, bool r) { assert(!"no system sleep here"); }\n' + 'static inline bool apple_sleep_coproc_stopped(struct device *d) { assert(!"no system sleep here"); return false; }\n') + write(tmp, 'linux/soc/apple/pmp-display.h', '#include \nbool apple_pmp_display_retained_board(void);\n') write(tmp, 'mock.h', '''#pragma once #include #include #include #include +#define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_err_probe(d, e, ...) (e) +#define dev_info(...) ((void)0) +#include +#define __iomem #include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; @@ -121,6 +152,7 @@ def main(): /* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} +static inline int of_property_match_string(const struct device_node*n,const char*p,const char*s){assert(!"no system sleep here");return -EINVAL;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" @@ -129,6 +161,12 @@ def main(): int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} +bool apple_pmp_display_retained_board(void){return false;} +void apple_pmp_report_sleep(struct device*d,bool at_commit){assert(!"no system sleep here");} +void apple_pmp_report_wake(struct device*d,bool real){assert(!"no system sleep here");} +int apple_pmp_sleep(const void*data){assert(!"no system sleep here");return -EIO;} +int apple_pmp_sleep_prepare(const void*data,struct apple_pmp_commit_stop*stop){assert(!"no system sleep here");return -EIO;} +int apple_pmp_wake(const void*data){assert(!"no system sleep here");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index d5905d4d71e34..d3b4d3a2ca787 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -124,6 +124,8 @@ def function(source, name): static void mt_bss_presence(struct mt7932*m,const struct mt7932_event*e){} static void mt_data_complete(struct mt7932*m,const u8*p,size_t n){} static void mt_packet_receive(struct mt7932*m,const u8*p,size_t n){} +static void mt_hif_event(struct mt7932*m,const struct mt7932_event*e){} +static void mt_wow_reason_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_rf_fail_locked(struct mt7932*m,int e){m->rf_ready=false;m->link_failed=true;m->cal_state.error=e;m->cal_state.active=false;m->connect_error=e;complete_all(&m->assoc_done);complete_all(&m->cal_response);} CHANNEL LINK @@ -252,6 +254,9 @@ def main(): headers.append(header) cal=(root/DRIVER/"calibration.h").read_text() headers.append(cal[cal.index("struct mt7932_cal_completion {"):cal.index("struct mt7932_cal_segment {")]) + # The receive path's other events, by their production IDs: the firmware's HIF and WoW reports. + headers.append(re.search(r"^#define MT7932_CID_HIF_CTRL\b.*$", rx, re.M).group(0)) + headers.append(re.search(r"^#define MT7932_EID_WOW_REASON\b.*$", (root/DRIVER/"wow.h").read_text(), re.M).group(0)) parts["HEADERS"]="\n".join(headers) candidates={"production":parts} if args.old_source: diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py index 4f4d183021905..fe4a55aa16d79 100644 --- a/tools/asahi/mtp-pm-tests.py +++ b/tools/asahi/mtp-pm-tests.py @@ -46,12 +46,18 @@ def function(source, name): static bool schedule_delayed_work(struct delayed_work *w, unsigned long d) { w->queued++; w->pending = true; return true; } struct hid_device { struct magicmouse_sc *data; int id; }; struct dchid_iface { struct hid_device *hid; }; -struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; +struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended, host_asleep; }; struct device { struct dockchannel_hid *data; }; static struct dockchannel_hid *transport; static unsigned suspends[16], resumes[16], key_releases; static int suspend_error_at = -1, resume_error_at = -1; static void *dev_get_drvdata(struct device *d) { return d->data; } +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 +#define dev_warn(...) ((void)0) +/* Not a firmware-driven suspend: the host state is never set here. */ +static bool pm_suspend_via_firmware(void) { return false; } +static int dchid_host_state(struct dockchannel_hid *d, int state) { (void)d; (void)state; assert(0); return 0; } static void *hid_get_drvdata(struct hid_device *d) { return d->data; } static void magicmouse_force_key(struct magicmouse_sc *m, bool down) { assert(!down); key_releases++; diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 469cacc4b3082..541990a615941 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -70,6 +70,8 @@ int ops, fail; }; static int disabled, pm_suspend_target_state = PM_SUSPEND_TO_IDLE; +static bool firmware_suspend; +static bool pm_suspend_via_firmware(void) { return firmware_suspend; } static inline __attribute__((unused)) s64 sign_extend64(u64 v, int bit) { return (s64)(v << (63 - bit)) >> (63 - bit); } static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } static void mutex_lock(struct mutex *m) { (void)m; } @@ -275,6 +277,10 @@ reset(v); assert(!apple_pmu_rtc_prepare(&dev)); pm_suspend_target_state = 2; assert(apple_pmu_rtc_prepare(&dev) == -EBUSY); + /* A firmware-driven suspend programs the wake itself. */ + firmware_suspend = true; + assert(!apple_pmu_rtc_prepare(&dev)); + firmware_suspend = false; pm_suspend_target_state = PM_SUSPEND_TO_IDLE; } int main(void) { diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index e8e7166399a0e..9df97f65c1591 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -37,6 +37,7 @@ def function(source, name): #define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) #define APPLE_RTKIT_MAX_ENDPOINTS 256 #define WRITE_ONCE(x,v) ((x)=(v)) +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) #define spin_lock_irqsave(l,f) ((void)(l), (f)=0) #define spin_unlock_irqrestore(l,f) ((void)(l), (void)(f)) struct apple_rtkit_shmem { size_t size; }; @@ -49,6 +50,9 @@ def function(source, name): struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer, pmp_aux_buffer; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; + bool oslog_echo; + u64 oslog_owed; + int boot_result; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4], started[4]; char *syslog_msg_buffer; @@ -69,6 +73,7 @@ def function(source, name): static void set_bit(unsigned n, unsigned long *b) { b[n/64] |= 1UL << (n%64); } static void bitmap_zero(unsigned long *b, unsigned n) { memset(b,0,n/8); } static void reinit_completion(int *c) { *c=0; } +static void complete_all(int *c) { *c=1; } static void apple_rtkit_rx(void) {} static void apple_rtkit_detach_rx(struct apple_rtkit *r) { ++detachments; r->shutting_down=true; } static void flush_workqueue(void *w) { (void)w; ++flushes; } diff --git a/tools/asahi/rtkit-retained-tests.py b/tools/asahi/rtkit-retained-tests.py index b79a9cd542bd2..f4e81d99f681d 100755 --- a/tools/asahi/rtkit-retained-tests.py +++ b/tools/asahi/rtkit-retained-tests.py @@ -66,18 +66,20 @@ def constants(source): { (void)rtk; sent++; sent_type = type; sent_data = data; return 0; } -/* A signal is pending for every interruptible wait. */ +/* A fatal signal is pending for every killable wait. */ #define ERESTARTSYS 512 #define READ_ONCE(x) (x) +enum { PM_SUSPEND_ON, PM_SUSPEND_TO_IDLE, PM_SUSPEND_STANDBY, PM_SUSPEND_MEM }; +static int pm_suspend_target_state; struct completion { int unused; }; -static unsigned waited_uninterruptible, waited_interruptible; +static unsigned waited_uninterruptible, waited_killable; static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long t) { (void)c; waited_uninterruptible++; return t; } -static long wait_for_completion_interruptible_timeout(struct completion *c, unsigned long t) +static long wait_for_completion_killable_timeout(struct completion *c, unsigned long t) { - (void)c; (void)t; waited_interruptible++; return -ERESTARTSYS; + (void)c; (void)t; waited_killable++; return -ERESTARTSYS; } ''' @@ -156,14 +158,23 @@ def constants(source): assert(sent == 0 && r.iop_pwr_ack_completion == 1); assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_QUIESCED); - /* A retained wake waits through a signal; other waits stay interruptible. */ + /* + * A retained wake waits through a signal, and so does a wait in a system + * sleep; any other wait ends only for a fatal one, not the freezer's. + */ struct completion c; r = fresh(); assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); - assert(waited_uninterruptible == 1 && waited_interruptible == 0); + assert(waited_uninterruptible == 1 && waited_killable == 0); r.retained_wake = false; assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); - assert(waited_uninterruptible == 1 && waited_interruptible == 1); + assert(waited_uninterruptible == 1 && waited_killable == 1); + pm_suspend_target_state = PM_SUSPEND_MEM; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); + assert(waited_uninterruptible == 2 && waited_killable == 1); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); + assert(waited_uninterruptible == 2 && waited_killable == 2); puts("PASS retained endpoint map replies, IOP acknowledgments and waits"); return 0; diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 840d961ff8cbb..4c6b2952484f8 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -159,6 +159,9 @@ struct apple_pmp_report { /* Requests held until the PMP runs; T6030 does not defer them. */ u64 shadow, known; bool defer; + /* Firmware-driven sleep: the request word at its start, the DVFS words kept. */ + u64 sleep_request; + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; struct apple_pmp_ane ane; @@ -628,6 +631,11 @@ static void *devm_kzalloc(struct device *d, size_t n, int g) allocations[allocation_count++] = v; return v; } + +static void *devm_kcalloc(struct device *d, size_t n, size_t size, int g) +{ + return devm_kzalloc(d, n * size, g); +} #define kzalloc_obj(v) calloc(1, sizeof(v)) #define kfree free static char *kstrdup(const char *s, int g) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index 69f832bff9cb3..468edbf9446a1 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -126,6 +126,8 @@ static bool passive_present, passive_lockable, passive_data; static const char *machine; static const void *pmp_data; static struct device *pmp_device; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py index 14c837ceeb5f3..95c02e87ddc02 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -19,7 +19,8 @@ def oslog(root): #define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctzll(mask)) #define APPLE_RTKIT_EP_OSLOG 8 struct buffer { unsigned sentinel; unsigned size; }; -struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack; }; +struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack, oslog_echo; u64 oslog_owed; }; +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) static unsigned warnings, notifications, requests, acks, errors; static u64 observed; static bool neo, m5; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 6c140662b6967..f7eafd88b68ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -146,6 +146,15 @@ def atc_restore_helpers(source): return atc_function(source, 'atcphy_setup_pipehandler') +def atc_common_state_helpers(source): + """What atcphy_restore_after_pd_off() calls to drop the common block's DisplayPort and + tunnel state after a power-down. Older sources do that inline: ''. + """ + if 'atcphy_forget_common_state(' not in atc_function(source, 'atcphy_restore_after_pd_off'): + return '' + return atc_function(source, 'atcphy_forget_common_state') + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 4c9f0ac4faad6..d29d8771807d9 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -86,6 +86,9 @@ struct dwc3_apple { void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending, shutting_down; + /* Stopped for a deep sleep, and the state to bring back. */ + bool sleep_stopped; + enum dwc3_apple_state sleep_state; bool rpm_held, host_only; /* T8132 only: runtime PM before the first core probe; never set here. */ bool pre_core_rpm; @@ -329,6 +332,11 @@ static void dwc3_apple_setup_cio(struct dwc3_apple *a) (void)a; setup_cio++; } +/* The hibernation block's register setup; teardown order doesn't depend on it. */ +static void __attribute__((unused)) dwc3_apple_setup_hibernation(struct dwc3_apple *a) +{ + (void)a; +} static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, enum phy_mode *m) { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 0bf6edeac8c19..6cea3b059279e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -792,6 +792,8 @@ def poweroff_replug(root): { (void)d; (void)generation; return 0; } static void dcp_external_crtc_released(struct apple_dcp *d) { (void)d; } static bool dcp_uses_soft_dpms(struct apple_dcp *d) { return d->soft_dpms; } +/* The T8140 display power cycle has its own test on hardware; here it only has to be reached. */ +static void dcp_dpms_power_off(struct apple_dcp *d) { (void)d; } static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return !d->typec_output; } static void __attribute__((unused)) iomfb_blank_h17p(struct apple_dcp *d) { (void)d; assert(!"unexpected internal-panel blank in external-link test"); } @@ -1068,6 +1070,9 @@ def fn(n, name): for name in ('atcphy_mode_configuration', 'atcphy_hw', 'atcphy_dp_link_rate_configuration'): decls.append(block(files['atc'], files['atc'].index('struct ' + name + ' {')) + ';') decls.append(block(files['state'], files['state'].index('enum dwc3_apple_state {')) + ';') + role = 'static inline enum dwc3_apple_state dwc3_apple_state_for_role(' + if role in files['state']: + decls.append(block(files['state'], files['state'].index(role))) start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') end = files['atc'].index('\n};', start) + 3 decls.append(files['atc'][start:end]) @@ -1087,7 +1092,7 @@ def fn(n, name): 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', - 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_restore_after_pd_off', + 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_forget_common_state', 'atcphy_restore_after_pd_off', 'atcphy_usb2_init', 'atcphy_usb3_power_off', 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index a56a3a3ccfbff..40d3d5d17fc8a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -4,7 +4,7 @@ from pathlib import Path import re from common import (block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE, - atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, + atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, atc_common_state_helpers, atc_cio_helpers) def parking(root): @@ -181,6 +181,7 @@ def function(source, name): parts.append(function(atc, name)) parts.append(dp) parts.append(atc_restore_helpers(atc)) + parts.append(atc_common_state_helpers(atc)) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_set_orientation', diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 8cae88d722d88..9c59dda578c79 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -428,6 +428,8 @@ def ownership(root, usb_unreachable=False): helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n' helpers+=''.join(atc_function(s,name) for name in ('atcphy_park_retry_delay','atcphy_park_retry','atcphy_park_due','atcphy_mux_change')) + # The mux callback's body, split out so the wake can set the mode it had. + helpers+=''.join(atc_function(s,name) for name in ('atcphy_mux_done','atcphy_mux_update_to')) helpers+=ns['function'](s,'atcphy_mux_update')+'\n' # Lane fixtures provide the target mapping and locked body; use the real callback here. # The mux callback powers an idle-power-off PHY first; this PHY is not one. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index b293087d4917c..c208cd9587224 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -95,7 +95,7 @@ def unit_for(source): struct dwc3 { bool sys_wakeup, dis_u3_susphy_quirk; struct platform_device *xhci; }; enum { DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_HOST, DWC3_APPLE_DEVICE }; /* pre_core_rpm is T8132 only: runtime PM before the first core probe; never set here. */ -struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy; }; +struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy, shutting_down; }; static struct dwc3_apple apple_context; #define to_dwc3_apple(d) container_of(d,struct dwc3_apple,dwc) static bool inside_host_remove; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 631237b2ac7b3..e63e5b4799a07 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -56,7 +56,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): actual=actual.replace('static struct apple_dcp_typec_port *sole_port(void) { return &port; }', 'static struct apple_dcp_typec_port *sole_port(void) { return &port; }\nstatic struct apple_dcp_typec_route * __attribute__((unused)) sole_route(void) { return &route; }',1) actual=actual.replace('bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;', 'struct typec_altmode *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;',1) - actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel;',1) + actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel,sleep_parked;',1) extra=r''' /* actual DCP leaves */ #define WRITE_ONCE(x,v) ((x)=(v)) diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c index 806148f64d1bf..8154098ee4157 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -19,6 +19,8 @@ struct device { void *of_node; }; static struct device owner; static struct device *pmp_device = &owner; static const void *pmp_data = &owner; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static bool t6030, requested; static int ready, check, power_result[2], report_result[3], powers, reports; static char calls[16];