diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index 501d9b1017f9c..870507ca4457b 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -9,6 +9,12 @@ on: - 'drivers/soc/apple/**' - 'include/uapi/linux/apple-sep.h' - 'tools/aurora-sep/**' + - 'Kconfig' + - '**/Kconfig*' + - 'arch/arm64/configs/defconfig' + - 'drivers/usb/dwc3/**' + - 'include/linux/phy/apple-atc.h' + - 'scripts/ci/apple-dwc3-atc-compile.sh' - '.github/workflows/sep-source.yml' jobs: @@ -116,3 +122,38 @@ jobs: --make_options LLVM=1 \ --kunitconfig=drivers/soc/apple/sep.kunitconfig \ --jobs="$(nproc)" --timeout=900 'apple_sep*' + + dwc3-atc-compile: + name: DWC3/ATC ${{ matrix.profile }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + max-parallel: 2 + matrix: + profile: [builtin-no-atc, modular-no-atc, builtin-modular-atc] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Install arm64 C build dependencies + run: | + sudo apt-get update -q + sudo apt-get install -y -q clang-18 lld-18 llvm-18 libelf-dev libssl-dev \ + flex bison bc pkg-config make perl + echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" + - name: Compile the Apple DWC3 object with the resolved PHY profile + env: + PROFILE: ${{ matrix.profile }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: scripts/ci/apple-dwc3-atc-compile.sh "$PROFILE" + - name: Keep configuration and compiler receipts + if: always() + uses: actions/upload-artifact@v4 + with: + name: dwc3-atc-${{ matrix.profile }} + path: ${{ runner.temp }}/dwc3-atc-${{ matrix.profile }}/artifacts/ + retention-days: 14 diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 53948d60375dc..4908e46f58712 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -64,6 +64,19 @@ recovery remain refused. Supporting these operations requires a proven engine teardown/reset and endpoint recovery contract first. Renaming the provider does not qualify those operations. +Deep sleep keeps both functions bound and the radio powered, with its firmware +running. Their drivers stop for it, the link goes through L2 and PERST#, and the SoC +powers the engine off. On the way back the engine's whole register file must read as it +did before the first command, the state the bootstrap checks for; if anything differs, +the radios stay down. Only then does the retained state reopen for one more validated +read, on the same memory, which the retrained link needs before native ECAM. The +functions' configuration is then restored and their drivers resume, with no firmware +load. If a driver finds that its side didn't survive, the host removes both functions, +which resets the chip, and scans again, so that they probe and load their firmware as at +boot. If the link didn't come back, the radio's rail is cycled first. An abandoned sleep +never ran that power-off, so it replays nothing. This needs tunables +from a loader that publishes them, and is refused otherwise. + Suspend-to-idle is allowed only after radio admission, with no IRQ fault, idle command/FIFO checks and intact arena contents. Resume restores the configuration banks after genpd power restoration without submitting another diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 41090bda6e80d..54245ac4f98de 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -78,11 +78,14 @@ properties: description: Optional host display-engine interrupt sources. mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 mbox-names: + minItems: 1 items: - const: mbox + - const: secure iommus: maxItems: 1 @@ -145,6 +148,13 @@ properties: items: pattern: '^(dp-phy|typec[0-9]+)$' + apple,display-power: + $ref: /schemas/types.yaml#/definitions/phandle + description: + A node whose ps, pwrgate and pmc-dcp registers hold the display power + words this coprocessor's driver writes around a firmware restart, such + as an apple,t8140-display-power node. + apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array items: diff --git a/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml new file mode 100644 index 0000000000000..41bd5586820f2 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,t8140-display-power.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 display power words + +maintainers: + - Diljit Singh + +description: + The words a display coprocessor's driver writes to power its domains off + and on around a restart of its firmware, as macOS does. A display + coprocessor names this node in its apple,display-power property. The words + belong to the power manager and the display block, so no driver binds it. + +properties: + compatible: + const: apple,t8140-display-power + + reg: + items: + - description: the display PMC word that counts the powered coprocessors + - description: the power manager's power state words + - description: the power manager's power gate words + + reg-names: + items: + - const: pmc-dcp + - const: ps + - const: pwrgate + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + }; diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index ad512ac8491df..b40cca489279f 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -82,6 +82,11 @@ allOf: reg: minItems: 3 maxItems: 3 + reg-names: + items: + - const: asc + - const: auxiliary + - const: asc-tail else: if: properties: @@ -137,6 +142,10 @@ properties: status: true + reg-names: + minItems: 1 + maxItems: 3 + '#dma-cells': const: 1 description: diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 992783b2dc68f..2d7b803e77963 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -98,6 +98,13 @@ properties: GFX supplier power cannot represent GPU work on firmware that keeps its coprocessor domain powered while the cores are idle. + apple,pmp-report-entry: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + PMP report entry that tells the PMP the GPU is in use. The driver withdraws + this request before it lowers the GPU's power state for a system sleep, and + makes it again after raising it. + apple,idle-off-delay-ms: minimum: 2 maximum: 1000 diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 0bee5d822fb21..3f0a7adf403d8 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -79,13 +79,15 @@ properties: - description: DART registers - description: Address filter (DAPF) registers, which restrict the physical - addresses the DART lets its masters reach + addresses the DART lets its masters reach, or the slice of the + fabric filter (GAPF) that protects the power state of the + coprocessor CPU behind this DART, cleared to power that CPU down reg-names: minItems: 1 items: - const: dart - - enum: [dapf, auxiliary] + - enum: [dapf, gapf, auxiliary] interrupts: maxItems: 1 @@ -117,6 +119,64 @@ properties: may select a firmware-visible alias of the translation aperture. A consumer may further specify its own aperture in its binding. + apple,dapf-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-matrix + items: + items: + - description: First address of a range + - description: Last address of a range + description: + The address filter the loader programs into the dapf registers, one + entry per range. A sleep that reaches the hardware clears it, and the + DART writes it back before its consumer's firmware starts again. + + apple,dapf-r20: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The last word of each filter entry, one per range in apple,dapf-ranges. + 1 where absent. + + apple,restore-on-power-up: + type: boolean + description: + Reset and restore the DART, and write its address filter back, every + time it powers up, even where its registers still read as they were + left. For a DART that keeps its values through a sleep in which the + SoC's sleep engine runs its consumer, but whose address filter only + takes effect once written again. + + apple,sleep-prepare: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes before a consumer that gates itself is told about a + firmware-driven system suspend, after the DART's state is saved. Streams + they stop are started again at the restore. Needs apple,sleep-restore. + + apple,sleep-restore: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after such a suspend, before the translations are put + back. With this, the consumer saves and restores the DART around the + suspend, as the order relative to its firmware's start matters. + + apple,sleep-restore-late: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after the translations are back and before the streams + are enabled. + apple,dma-offset: $ref: /schemas/types.yaml#/definitions/uint64 description: > @@ -168,6 +228,18 @@ properties: Tree by the bootloader and required to initialize dynamically powered DART instances. + apple,resume-tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + The same tuples for a DART whose bootloader has already applied them, + written back where a power-off lost them, before its consumer's + firmware starts again. + apple,bypass-sids: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index eb6d3f5580761..8afd38261fe30 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -76,6 +76,14 @@ properties: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox + - description: + A coprocessor's secure RTBuddy port, the ADT's + iop,secure-rtbuddy-proxy. It is polled, with no interrupts described. + items: + - enum: + - apple,t8140-secure-rtbuddy-proxy + - const: apple,secure-rtbuddy-proxy + - description: Later AOP setup-port mailboxes with the T8140 register layout. items: @@ -142,6 +150,16 @@ properties: maxItems: 2 allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,secure-rtbuddy-proxy + then: + required: + - interrupts + - interrupt-names - if: properties: compatible: @@ -220,8 +238,6 @@ allOf: required: - compatible - reg - - interrupts - - interrupt-names - "#mbox-cells" additionalProperties: false diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index b8e745edb4aa5..b0bdb730032e0 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -4,13 +4,14 @@ $id: http://devicetree.org/schemas/media/apple,t8140-isp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 Image Signal Processor +title: Apple T8140 and T8142 Image Signal Processors maintainers: - Ryan Murray description: - The T8140 ISP controls the attached camera sensor and processes its frames + The T8140 and T8142 ISPs control the attached camera sensor and process + its frames through bootloader-loaded firmware. The host exchanges commands and image buffers through three DART instances. A separate watchdog register window must be serviced while capturing to receive sensor pixels. @@ -57,24 +58,8 @@ properties: - description: Third ISP DART stream power-domains: - items: - - description: ISP system domain - - description: ISP CPU domain - - description: ISP CPU core 0 domain - - description: ISP CPU core 1 domain - - description: ISP FE domain - - description: ISP DPRX domain - - description: ISP secure domain - - description: ISP STS0 domain - - description: ISP STS1 domain - - description: ISP Pearl domain - - description: ISP BE domain - - description: ISP CLR domain - - description: ISP CVD domain - - description: ISP CVM domain - - description: Camera sensor 0 domain - - description: Camera sensor 1 domain - minItems: 14 + minItems: 8 + maxItems: 16 memory-region: minItems: 1 @@ -144,6 +129,50 @@ properties: additionalProperties: false additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-isp + then: + properties: + power-domains: + items: + - description: ISP system domain + - description: ISP CPU domain + - description: ISP CPU core 0 domain + - description: ISP CPU core 1 domain + - description: ISP FE domain + - description: ISP sensor 0 domain + - description: ISP sensor 1 domain + - description: ISP DPRX domain + - description: ISP secure domain + - description: ISP STS0 domain + - description: ISP STS1 domain + - description: ISP Pearl domain + - description: ISP BE domain + - description: ISP CLR domain + - description: ISP CVD domain + - description: ISP CVM domain + - if: + properties: + compatible: + contains: + const: apple,t8142-isp + then: + properties: + power-domains: + items: + - description: ISP system domain + - description: ISP CPU domain + - description: ISP CPU core 0 domain + - description: ISP FE domain + - description: ISP DPRX domain + - description: ISP STS0 domain + - description: ISP BE domain + - description: ISP CLR domain + required: - compatible - reg @@ -180,10 +209,11 @@ examples: iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, - <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_fe>, <&ps_ispsens0>, <&ps_ispsens1>, + <&ps_isp_dprx>, <&ps_isp_secure>, <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, - <&ps_isp_cvm>, <&ps_ispsens0>, <&ps_ispsens1>; + <&ps_isp_cvm>; memory-region = <&isp_heap>; memory-region-names = "heap"; apple,dart-vm-size = <0x0 0xa0000000>; @@ -200,3 +230,41 @@ examples: }; }; }; + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@4a0000000 { + compatible = "apple,t8142-isp", "apple,isp"; + reg = <0x4 0xa0000000 0x0 0x1801000>, + <0x4 0xa2920000 0x0 0x1000>, + <0x4 0xa2921000 0x0 0x100>, + <0x4 0xe3b10000 0x0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_fe>, + <&ps_isp_dprx>, <&ps_isp_sts0>, + <&ps_isp_be>, <&ps_isp_clr>; + memory-region = <&isp_heap>; + memory-region-names = "heap"; + apple,dart-vm-size = <0x0 0xe0000000>; + apple,platform-id = <0x71>; + apple,temporal-filter = <1>; + + sensor-presets { + preset0 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1920 1080>; + apple,crop = <0 584 3040 1710>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 78f154ce2edcd..084d75bdac4fc 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -196,8 +196,7 @@ properties: description: Initialization tuples for the oe-fabric register region. apple,tunable-axi2af: - $ref: '#/$defs/t6050-tunable' - description: T6050 tuples for the axi2af window, applied before a train. + description: Register tuples for the controller's AXI bridge. apple,tunable-cio3pllcore: $ref: '#/$defs/t6050-tunable' @@ -207,17 +206,22 @@ properties: $ref: '#/$defs/t6050-tunable' description: T6050 tuples for the pcieclkgen window. + apple,tunable-cio3pll: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's PLL. + + apple,tunable-clkgen: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's clock generator. + apple,tunable-phy: - $ref: '#/$defs/t6050-tunable' - description: T6050 tuples for the phy-window window. + description: Register tuples for the controller's PHY window. apple,tunable-phy-ip-pll: - $ref: '#/$defs/t6050-tunable' - description: T6050 PHY PLL tuples for the phy-ip window. + description: PLL register tuples relative to the controller's phy-ip window. apple,tunable-phy-ip-auspma: - $ref: '#/$defs/t6050-tunable' - description: T6050 PHY lane tuples for the phy-ip window. + description: Lane register tuples relative to the controller's phy-ip window. apple,piodma: $ref: /schemas/types.yaml#/definitions/phandle @@ -315,6 +319,16 @@ allOf: apple,msi-vector-base: const: 64 power-domains: false + apple,tunable-axi2af: + $ref: '#/$defs/t6050-tunable' + apple,tunable-phy: + $ref: '#/$defs/t6050-tunable' + apple,tunable-phy-ip-pll: + $ref: '#/$defs/t6050-tunable' + apple,tunable-phy-ip-auspma: + $ref: '#/$defs/t6050-tunable' + apple,tunable-cio3pll: false + apple,tunable-clkgen: false required: - interrupt-names - apple,msi-vector-base @@ -338,12 +352,16 @@ allOf: - apple,link-trained-by-bootloader else: properties: - apple,tunable-axi2af: false + apple,tunable-axi2af: + $ref: '#/$defs/tunable' apple,tunable-cio3pllcore: false apple,tunable-pcieclkgen: false - apple,tunable-phy: false - apple,tunable-phy-ip-pll: false - apple,tunable-phy-ip-auspma: false + apple,tunable-phy: + $ref: '#/$defs/tunable' + apple,tunable-phy-ip-pll: + $ref: '#/$defs/tunable' + apple,tunable-phy-ip-auspma: + $ref: '#/$defs/tunable' patternProperties: "^pci@": properties: @@ -482,8 +500,9 @@ allOf: properties: reg: minItems: 8 - maxItems: 8 + maxItems: 13 reg-names: + minItems: 8 items: - const: config - const: rc @@ -493,6 +512,11 @@ allOf: - const: phy0 - const: phy1 - const: phy2 + - const: axi2af + - const: cio3pll + - const: clkgen + - const: phy-block + - const: phy-ip interrupts: maxItems: 3 else: diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 5364f6dd8871b..d8273b43145c7 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -125,6 +125,10 @@ allOf: - description: USB2 PHY specific controls - description: USB3 PIPE interface controls - description: Secondary eUSB2 PHY register bank (T8140) + - description: + Power and hibernation control of the USB controller the PHY + serves (T8140). The PHY drives that controller's reset and + takes it through hibernation for a system sleep. reg-names: minItems: 5 items: @@ -134,6 +138,7 @@ allOf: - const: usb2phy - const: pipehandler - const: usb2phy-reg + - const: pwrctl required: - power-domains - if: @@ -443,6 +448,18 @@ properties: T8140, copied from Apple's Device Tree by the bootloader. Unlike the other tunables these are applied to the usb2phy-reg region. + apple,axi2af-gate: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset in the axi2af region + - description: Bits + description: > + The fabric bridge bits macOS clears, in this order, before the PHY's + common power domain goes off for a system sleep and sets again once it + is back. With pwrctl, the controller then keeps its power through the + sleep. + patternProperties: "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": $ref: /schemas/types.yaml#/definitions/uint32-array diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 8281b883bd2ac..420f47e4522bc 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -23,6 +23,7 @@ allOf: properties: apple,firmware-owned: false apple,sleep-retained: false + apple,sleep-driver-owned: false apple,always-on: false apple,min-state: false apple,force-disable: false @@ -135,6 +136,13 @@ properties: system sleep. Runtime power management is unaffected. type: boolean + apple,sleep-driver-owned: + description: + The driver behind this domain lowers it before a system sleep and brings + it back afterwards, in an order its hardware needs. The power controller + leaves it alone after a wake. + type: boolean + apple,firmware-owned: description: The boot loader hands over active hardware whose power domain remains diff --git a/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml new file mode 100644 index 0000000000000..8d4e6ab0029db --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,cluster-pwrgate.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC CPU cluster power-gate enables + +maintainers: + - Diljit Singh + +description: | + One PMGR word per CPU cluster, whose top bit lets the cluster's cores power + down in WFI. A core that PMGR powers on comes up allowed to, but the boot CPU + was started by the boot ROM and only follows the word, which the boot loader + may leave clear. The words are set early in boot, in the order listed. + +properties: + compatible: + items: + - enum: + - apple,t8140-cluster-pwrgate + - const: apple,cluster-pwrgate + + reg: + minItems: 1 + maxItems: 8 + description: One word per cluster. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml new file mode 100644 index 0000000000000..3498a9e913cdc --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-sleep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC system sleep controls + +maintainers: + - Diljit Singh + +description: | + The registers a kernel running directly at EL2 on an Apple SoC writes to enter + and return from suspend-to-RAM and to power its secondary CPUs off. They belong + to several hardware blocks, some of which other drivers also use, so this node + overlaps those drivers' register ranges and the driver does not claim them. + Each SoC has its own compatible, as its power domains differ. + +properties: + compatible: + enum: + - apple,t8140-sleep + + reg: + maxItems: 28 + + reg-names: + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: amcc2 + - const: amcc3 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: mcc2 + - const: ptd + - const: clk-perf + - const: cluster-dcs + + apple,wake-enable: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The wake source IDs left enabled across suspend-to-RAM. Every other + source is masked. + + apple,apcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PCIe host whose shared registers and tunables the storage controller's + link depends on. They are needed even with that host disabled. + + apple,pinctrl: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The pin controller whose pins a deep sleep resets. Its pins are saved + before a sleep and put back after it, before the storage controller + starts. + + apple,pinctrl-skip-pins: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Pins of that controller to leave alone across a sleep, neither saved nor + put back. + + apple,debug-port: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The USB-C port controller that carries the debug link, stopped for a sleep. + + apple,display-coprocessors: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The DCP and the external DCP, which must be off before a commit, in that + order. + +required: + - compatible + - reg + - reg-names + - apple,wake-enable + - apple,display-coprocessors + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index b023080e48c7f..7329ac57a538f 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -82,6 +82,10 @@ properties: Chassis identifier used to name the sound card, matching the apple,aop-audio convention. + mic-supply: + description: + The microphone array's power rail, switched on around each capture. + apple,machine-kind: $ref: /schemas/types.yaml#/definitions/string description: diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 7745fef5b686f..6612a5370ee36 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -150,6 +150,13 @@ properties: $ref: "#/$defs/apple,tunable" description: Tunables applied to the drd-ausbc region. + apple,sleep-ddr: + type: boolean + description: + While the controller runs as a host, a system sleep keeps DRAM for its + always-on side, as macOS's USB_SLP_DDR vote does from the host's start + to its stop. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index 282d26ae0c6d2..9540e20101706 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -41,6 +41,24 @@ properties: wakeup-source: true + apple,sleep-ssps: + type: boolean + description: + Tell the controller the system's power state (SSPS) around a + firmware-driven suspend, S3 before it and S0 after. + + apple,debug-lock-key: + $ref: /schemas/types.yaml#/definitions/string + description: + The four-character key that unlocks the controller's debug mode, for + rearming the debug link after a sleep ended it. + + apple,usb-remote-wake: + type: boolean + description: + This controller serves the USB controller's root port, and watches its + USB2 lines for a remote wakeup during a firmware-driven system sleep. + connector: $ref: /schemas/connector/usb-connector.yaml# diff --git a/MAINTAINERS b/MAINTAINERS index 30f3d7801ea57..c2e1de0244904 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,6 +2093,7 @@ F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml @@ -2109,6 +2110,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml F: Documentation/devicetree/bindings/sound/apple,t6041-aop-audio.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -2130,8 +2132,10 @@ F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c +F: drivers/soc/apple/sleep*.c F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h +F: include/linux/soc/apple/sleep.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/aop_t8140.rs @@ -2750,6 +2754,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -2816,7 +2821,9 @@ F: drivers/video/backlight/apple_dwi_bl.c F: drivers/watchdog/apple_wdt.c F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h +F: include/linux/irqchip/irq-apple-aic.h F: include/linux/mfd/macsmc.h +F: include/linux/phy/apple-atc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 69d6eff9aa756..dd7580f5f13df 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -23,6 +23,8 @@ /* First mapper in the measured ADT iommu-parent list. */ iommus = <&dcpext0_dart 23>; power-domains = <&ps_dispext0_cpu>; + /* DISPEXT0's words and pmc-dcp, for the restart after a sleep */ + apple,display-power = <&dcp_power>; apple,connector-type = "USB-C"; status = "disabled"; @@ -65,7 +67,11 @@ }; }; -/* Retain the inherited firmware supply once the loader publishes it. */ +/* + * Retain the inherited firmware supply once the loader publishes it. After a sleep the DCP + * driver raises it, the way macOS does, not the PMGR resync. + */ &ps_dispext0_cpu { apple,always-on; + apple,sleep-driver-owned; }; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 6596e56a233d8..c814ac7107760 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -30,13 +30,16 @@ serial2 = &uart5; serial3 = &uart0; pmp = &pmp; - rtc0 = &smc_rtc; - rtc1 = &pmu_rtc; + /* First the RTC that can wake the system */ + rtc0 = &pmu_rtc; + rtc1 = &smc_rtc; wifi0 = &wifi0; /* used by the loader to hand over the display */ dcp = &dcp; disp0 = &display; disp0-piodma = &disp0_piodma; + /* and the SIO, with its firmware */ + sio = &sio; }; chosen { @@ -414,20 +417,53 @@ regulator-name = "jack-codec"; regulator-always-on; }; + + /* + * The mic array's rail: SMC pcIO output 25, the ADT's + * function-input_power_enable. macOS switches it around each capture. + */ + mic_array_supply: regulator-mic-array { + compatible = "regulator-fixed"; + regulator-name = "mic-array"; + gpio = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; }; #include "isp-imx558.dtsi" -&ave_dart0 { - status = "okay"; +/* + * The J700 has one video encoder, the ADT's ave1 at 0x289100000 under + * VENC1_SYS. Encoder 0, which t8140.dtsi describes, isn't there: the ADT has + * no node for it and VENC0_SYS refuses its target. Its DARTs and domains + * would wait forever on a domain that never rises. + */ +&pmp_report_ave0 { + status = "disabled"; }; -&ave_dart1 { - status = "okay"; +&ps_venc0_sys { + status = "disabled"; }; -&ave { - status = "okay"; +&ps_venc0_dma { + status = "disabled"; +}; + +&ps_venc0_pipe4 { + status = "disabled"; +}; + +&ps_venc0_pipe5 { + status = "disabled"; +}; + +&ps_venc0_me0 { + status = "disabled"; +}; + +&ps_venc0_me1 { + status = "disabled"; }; &serial0 { @@ -567,6 +603,19 @@ status = "okay"; }; +&dcp_secure_mbox { + status = "okay"; +}; + +/* The SIO itself is enabled by the loader that hands it over. */ +&sio_dart { + status = "okay"; +}; + +&sio_mbox { + status = "okay"; +}; + &dcp { /* 2408x1506 at 219 ppi; 525 nits peak from the backlight node */ panel: panel { @@ -661,6 +710,7 @@ &aop_audio { apple,chassis-name = "J700"; apple,machine-kind = "MacBook Neo"; + mic-supply = <&mic_array_supply>; }; &admac_leap_ns { @@ -753,7 +803,8 @@ }; &smc_gpio { - apple,pcio-templates = <10 0x800000 0xa0000>; + apple,pcio-templates = <10 0x800000 0xa0000>, + <25 0x0 0x190000>; }; &aop_spmi0 { @@ -1859,13 +1910,22 @@ }; }; -/* USB PD controllers, on the nub SPMI A0 bus */ +&sleep { + apple,debug-port = <&hpm0>; +}; + +/* + * USB PD controllers, on the nub SPMI A0 bus. macOS sends them SSPS 3 and 0 around a sleep + * and nothing else: no USBw, as the J700 ADT has no dock-remote-wake. A USB key press + * wakes the J700 through the host's own wake source. + */ &nub_spmi_a0 { status = "okay"; hpm1: usb-pd@a { compatible = "apple,sn201202x"; reg = <0xa SPMI_USID>; + apple,sleep-ssps; interrupts = <37 IRQ_TYPE_EDGE_RISING>, <39 IRQ_TYPE_EDGE_RISING>, <43 IRQ_TYPE_EDGE_RISING>, @@ -1890,6 +1950,8 @@ hpm0: usb-pd@c { compatible = "apple,sn201202x"; reg = <0xc SPMI_USID>; + apple,debug-lock-key = "007J"; + apple,sleep-ssps; interrupts = <11 IRQ_TYPE_EDGE_RISING>, <13 IRQ_TYPE_EDGE_RISING>, <17 IRQ_TYPE_EDGE_RISING>, @@ -2127,6 +2189,10 @@ #include "t8140-j700-dcpext.dtsi" #include "t8140-j700-dpaudio.dtsi" +&sleep { + apple,display-coprocessors = <&dcp>, <&dcpext>; +}; + &display { iommus = <&disp0_dart 0>, <&dispext0_dart 0>; }; diff --git a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi index 68305db8af938..8aa61ea0f53e4 100644 --- a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi @@ -227,12 +227,11 @@ /* ADT /arm-io/dcp-exclave-mailbox. */ dcp_secure_mbox: mailbox@452e7c000 { - compatible = "apple,t8140-secure-mailbox", "apple,secure-mailbox"; + compatible = "apple,t8140-secure-rtbuddy-proxy", + "apple,secure-rtbuddy-proxy"; reg = <0x4 0x52e7c000 0x0 0x4000>; - interrupt-parent = <&aic>; - interrupts = , - ; - iommus = <&dcp_dart 24>; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; status = "disabled"; }; @@ -474,12 +473,14 @@ status = "disabled"; }; + /* Enabled by the boot loader with its firmware, through its alias. */ sio: dma-controller@385e00000 { compatible = "apple,t8140-sio", "apple,sio"; /* Split the ASC bank around the separately owned mailbox window. */ reg = <0x3 0x85e00000 0x0 0x8000>, <0x3 0x85850000 0x0 0x4000>, <0x3 0x85e0c000 0x0 0x7c000>; + reg-names = "asc", "auxiliary", "asc-tail"; #dma-cells = <1>; dma-channels = <128>; mboxes = <&sio_mbox>; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 4a0ef005ca7aa..22ca0b832f2e3 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -61,6 +61,7 @@ * does not survive this state going off for long. */ apple,always-on; + apple,sleep-driver-owned; }; ps_venc0_sys: power-controller@240 { @@ -208,16 +209,18 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_sys_st"; - power-domains = <&ps_ans>, <&ps_apcie_st>; + power-domains = <&ps_ans>, <&ps_apcie_st>, <&ps_apcie_phy_sw>; }; + /* SYS_ST's parent, kept on: only a real sleep lowers it, through the sleep controller. */ ps_apcie_phy_sw: power-controller@3c0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_phy_sw"; - power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + power-domains = <&ps_apcie_sys_gp>; + apple,always-on; }; ps_pmp: power-controller@3d0 { @@ -273,7 +276,8 @@ * enables between ISP_FE and ISP_SECURE. Plain power * gating only: a local domain parked with DEV_DISABLE (the t8112 * apple,force-disable pattern) never reached ACTIVE again on J700 - * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). + * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). No + * automatic power management either: macOS writes them exactly 0xf. */ ps_sep: power-controller@c00 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; @@ -289,6 +293,7 @@ reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cpu"; power-domains = <&ps_isp_sys>; }; @@ -298,6 +303,7 @@ reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cpu_core0"; power-domains = <&ps_isp_cpu>; }; @@ -307,6 +313,7 @@ reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cpu_core1"; power-domains = <&ps_isp_cpu>; }; @@ -316,6 +323,7 @@ reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_fe"; power-domains = <&ps_isp_sys>; }; @@ -325,6 +333,7 @@ reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_dprx"; power-domains = <&ps_isp_fe>; }; @@ -334,6 +343,7 @@ reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_secure"; power-domains = <&ps_isp_fe>; }; @@ -343,6 +353,7 @@ reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_sts0"; power-domains = <&ps_isp_fe>; }; @@ -352,6 +363,7 @@ reg = <0x4038 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_sts1"; power-domains = <&ps_isp_fe>; }; @@ -361,6 +373,7 @@ reg = <0x4040 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_pearl"; power-domains = <&ps_isp_fe>; }; @@ -370,6 +383,7 @@ reg = <0x4048 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_be"; power-domains = <&ps_isp_fe>; }; @@ -379,6 +393,7 @@ reg = <0x4050 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_clr"; power-domains = <&ps_isp_be>; }; @@ -388,6 +403,7 @@ reg = <0x4058 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cvd"; power-domains = <&ps_isp_be>; }; @@ -397,6 +413,7 @@ reg = <0x4060 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cvm"; power-domains = <&ps_isp_cvd>; }; @@ -891,6 +908,8 @@ */ power-domains = <&ps_sio>, <&ps_spi_p>, <&ps_uart_p>; status = "okay"; + /* The sleep reads the SIO's run bit at the commit, so it stays on until then. */ + apple,always-on; }; ps_fpwm0: power-controller@2b0 { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2eff89793e00d..724236ac38542 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -551,7 +551,42 @@ pmp_dart: iommu@300300000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x00300000 0x0 0xc000>; + reg = <0x3 0x00300000 0x0 0xc000>, + <0x3 0x00310000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* + * The loader's address filter, from the ADT's dapf-instance-0, + * written back after a sleep. + */ + apple,dapf-ranges = /bits/ 64 <0x220020000 0x220030000 + 0x220060000 0x220070000 + 0x2200a0000 0x2200b0000 + 0x2200e0000 0x2200f0000 + 0x220404000 0x220404070 + 0x220604000 0x220604070 + 0x220804000 0x220804070 + 0x30819d000 0x30819d00c + 0x210e40000 0x210e7ffff + 0x211e40000 0x211e7ffff + 0x40196c000 0x40197bfff + 0x480d14000 0x480e23fff + 0x210e78000 0x210ef0000 + 0x211e78000 0x211ef0000 + 0x401868000 0x40186df08 + 0x24f004000 0x24f004084 + 0x24f008000 0x24f00c140 + 0x250004000 0x250004084 + 0x250008000 0x25000c140 + 0x251004000 0x251004084 + 0x251008000 0x25100c140 + 0x22001c000 0x22001e000 + 0x22005c000 0x22005e000 + 0x22009c000 0x22009e000 + 0x2200dc000 0x2200de000 + 0x3082c8000 0x3082c8050 + 0x220104000 0x22011c000>; + apple,dapf-r20 = <0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x4 0x1 0x1 0x1 0x1 0x1 0x1 + 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1>; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -672,7 +707,9 @@ dcpext0_dart: iommu@416340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16340000 0x0 0xc000>; + reg = <0x4 0x16340000 0x0 0xc000>, + <0x3 0x0079c700 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -770,6 +807,13 @@ */ apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; apple,dma-offset = /bits/ 64 <0x0>; + /* What macOS writes around a sleep, as offset and value. */ + apple,sleep-prepare = <0x80 0xa102>, <0x704 0xffffffff>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x318 0x1>; + apple,sleep-restore-late = <0x80 0x8000>, <0x228 0x3f3f3f3f>, + <0x22c 0x1010101>; }; sep: sep@282600000 { @@ -825,6 +869,73 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + /* + * The CPU clusters' power-gate enables, in /arm-io/pmgr reg[22]: + * cluster-id 0 (E) at +0x0, 1 (P) at +0x40. iBoot sets E's only. + */ + cluster_pwrgate: power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + + /* + * Addresses from the ADT: /arm-io/pmgr reg[21], reg[22], + * reg[24] + 0x4000, reg[27], reg[28], reg[31] + 0x4000, /arm-io/wdt + * reg[0], /arm-io/pmgr reg[29] and reg[4]. The AMCC words were + * measured under aura. Sizes cover only the words used: others in + * these blocks are locked by the fabric. + */ + sleep: sleep-controller@300700000 { + compatible = "apple,t8140-sleep"; + reg = <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>, + <0x3 0x00734000 0x0 0x10>, + <0x3 0x00768000 0x0 0x40>, + <0x3 0x00780000 0x0 0xc>, + <0x3 0x082d4000 0x0 0x4>, + <0x3 0x082b0000 0x0 0x30>, + <0x3 0x082b8000 0x0 0x44>, + <0x3 0x08004000 0x0 0x48>, + <0x2 0x20025000 0x0 0x10>, + <0x2 0x20065000 0x0 0x10>, + <0x2 0x200a5000 0x0 0x10>, + <0x2 0x200e5000 0x0 0x10>, + <0x3 0x028a9c14 0x0 0x4>, + <0x2 0x10e70000 0x0 0x1000>, + <0x2 0x11e70000 0x0 0x1000>, + <0x3 0x08714000 0x0 0x84>, + <0x3 0x02800000 0x0 0x10000>, + <0x3 0x0005c000 0x0 0x4>, + <0x3 0x08280000 0x0 0x10000>, + <0x3 0x08908040 0x0 0x4>, + <0x3 0x00040034 0x0 0xc>, + <0x2 0x20400580 0x0 0x10>, + <0x2 0x20600580 0x0 0x10>, + <0x2 0x20800580 0x0 0x10>, + <0x3 0x003d0bc8 0x0 0x10>, + <0x3 0x00058000 0x0 0x10004>, + <0x3 0x00780010 0x0 0x4>; + reg-names = "ps", "pwrgate", "cpu-start", "cllt", "cluster", + "commit", "wdt", "aon-misc", "wake", + "amcc0", "amcc1", "amcc2", "amcc3", "pmc-dcp", + "dvfm-e", "dvfm-p", "spmi", "pmc-voters", "dcs", + "ps-aon", "spmi-a0", "clk-mux", "mcc0", "mcc1", "mcc2", + "ptd", "clk-perf", "cluster-dcs"; + apple,apcie = <&pcie0>; + /* All but the pins macOS leaves alone after a wake */ + apple,pinctrl = <&pinctrl_ap>; + apple,pinctrl-skip-pins = <49 50 51 134>; + /* + * NUB.SPMI0.SW3: the PMU's RTC alarm. ATC0_PMGR_USB_WAKEUP: the + * USB host's, which macOS enables at boot from the ADT and never + * masks. SMC.CPUWakeupAP and SMC.OutboxNotEmpty: the power button, + * the lid, AC, Wi-Fi and Bluetooth. MTP.DOCK.CHANNELS.AP0.IRQ and + * MTP.OUTBOX.N.EMPTY_IRQ: the internal keyboard and trackpad. + */ + apple,wake-enable = <131 70 145 165 80 99>; + }; + wdt: watchdog@3082b0000 { compatible = "apple,t8140-wdt", "apple,t8103-wdt"; reg = <0x3 0x082b0000 0x0 0x4000>; @@ -1195,9 +1306,15 @@ <0x3 0x92028000 0x0 0x8000>, <0x3 0x97020000 0x0 0x4000>, <0x3 0x97024000 0x0 0x4000>, - <0x3 0x97028000 0x0 0x4000>; + <0x3 0x97028000 0x0 0x4000>, + <0x3 0x96000000 0x0 0x1000000>, + <0x3 0x95046200 0x0 0x4000>, + <0x3 0x95044000 0x0 0x4000>, + <0x3 0x97000000 0x0 0x40000>, + <0x3 0x97040000 0x0 0x28000>; reg-names = "config", "rc", "port0", "port1", "port2", - "phy0", "phy1", "phy2"; + "phy0", "phy1", "phy2", + "axi2af", "cio3pll", "clkgen", "phy-block", "phy-ip"; interrupt-parent = <&aic>; interrupts = , , @@ -1244,8 +1361,9 @@ dwc3_0: usb@40a280000 { compatible = "apple,t8140-dwc3", "apple,t8103-dwc3"; reg = <0x4 0x0a280000 0x0 0xcd00>, - <0x4 0x0a28cd00 0x0 0x3200>; - reg-names = "dwc3-core", "dwc3-apple"; + <0x4 0x0a28cd00 0x0 0x3200>, + <0x4 0x0a200000 0x0 0x4000>; + reg-names = "dwc3-core", "dwc3-apple", "drd-cfg"; interrupt-parent = <&aic>; interrupts = ; dr_mode = "otg"; @@ -1256,6 +1374,8 @@ resets = <&atcphy0>; phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; phy-names = "usb2-phy", "usb3-phy"; + /* usb-drd's USB_SLP_DDR-V clock gate in the ADT */ + apple,sleep-ddr; status = "disabled"; }; @@ -1267,6 +1387,22 @@ power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + /* + * macOS's writes around a sleep that keeps the host: exceptions + * and block errors off, streams stopped, a flush; at the wake, its + * config words before the translations, then the rest. 0x104 is + * Linux's own unmasked error interrupt. + */ + apple,sleep-prepare = <0x4060 0xffffffff>, <0x4000 0x0>, + <0x704 0xffffffff>, <0xc20 0xffff>, + <0x80 0xa000>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x308 0x10000000>, + <0x310 0x3fffffff>, <0x318 0x1>, + <0x320 0xff00000>, <0x328 0xff017ff>; + apple,sleep-restore-late = <0x228 0x3f3f3f3f>, <0x22c 0x1010101>, + <0x104 0x0>, <0x4060 0x0>; status = "disabled"; }; @@ -1278,6 +1414,17 @@ power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + /* As dwc3_0_dart_0's. */ + apple,sleep-prepare = <0x4060 0xffffffff>, <0x4000 0x0>, + <0x704 0xffffffff>, <0xc20 0xffff>, + <0x80 0xa000>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x308 0x10000000>, + <0x310 0x3fffffff>, <0x318 0x1>, + <0x320 0xff00000>, <0x328 0xff017ff>; + apple,sleep-restore-late = <0x228 0x3f3f3f3f>, <0x22c 0x1010101>, + <0x104 0x0>, <0x4060 0x0>; status = "disabled"; }; @@ -1288,9 +1435,12 @@ <0x4 0x08000000 0x0 0x8000>, <0x4 0x0aa90000 0x0 0x4000>, <0x4 0x0aa84000 0x0 0x4000>, - <0x4 0x0a300000 0x0 0x4000>; + <0x4 0x0a300000 0x0 0x4000>, + <0x4 0x0aa80000 0x0 0x4000>; reg-names = "core", "lpdptx", "axi2af", "usb2phy", - "pipehandler", "usb2phy-reg"; + "pipehandler", "usb2phy-reg", "pwrctl"; + /* Bit 0 of both, around every ATC0_COMMON change on macOS. */ + apple,axi2af-gate = <0x104 0x1>, <0x108 0x1>; #phy-cells = <1>; #reset-cells = <0>; @@ -1313,6 +1463,7 @@ memory-region-names = "ttbs", "pagetables", "handoff"; operating-points-v2 = <&gpu_opp>; apple,clpc = <&clpc>; + apple,pmp-report-entry = <&pmp_report_gfx>; status = "disabled"; }; @@ -1368,7 +1519,9 @@ dcp_dart: iommu@412340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x12340000 0x0 0xc000>; + reg = <0x4 0x12340000 0x0 0xc000>, + <0x3 0x0079c880 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -1449,8 +1602,22 @@ #apple,bw-doorbell-cells = <2>; }; + /* + * Display power words the DCP driver writes around a warm restart. + * /arm-io/disp0 reg[5] + 0xa9c14, and /arm-io/pmgr reg[21] and reg[22]. + * The firmware owns DISP_CPU only while it runs. No driver binds this. + */ + dcp_power: display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + dcp: dcp@412e00000 { compatible = "apple,t8140-dcp", "apple,dcp"; + apple,display-power = <&dcp_power>; reg = <0x4 0x12e00000 0x0 0x4000>, <0x4 0x10000000 0x0 0xafc000>, <0x4 0x12240000 0x0 0x10000>, @@ -1460,8 +1627,8 @@ <0x4 0x12850000 0x0 0x4000>; reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3", "disp-4", "iop-vbar"; - mboxes = <&dcp_mbox>; - mbox-names = "mbox"; + mboxes = <&dcp_mbox>, <&dcp_secure_mbox>; + mbox-names = "mbox", "secure"; iommus = <&dcp_dart 23>; power-domains = <&ps_disp_cpu>; clocks = <&clk_disp0_154>, <&clk_disp0_194>; @@ -1845,6 +2012,17 @@ status = "disabled"; }; +#define T8140_ISP_DART_TUNABLES \ + apple,resume-tunable = <0x20c 0x80000521 0x80000521>, \ + <0x220 0x000f0f0f 0x000f0f0f>, \ + <0x224 0x00ffffff 0x00080808>, \ + <0x300 0x00001f31 0x00000001>, \ + <0x308 0x3ffffffc 0x10000000>, \ + <0x310 0x3ffffffc 0x3ffffffc>, \ + <0x318 0x00001f31 0x00000001>, \ + <0x320 0x3ffffffc 0x0ff00000>, \ + <0x328 0x3ffffffc 0x0ff017fc> + /* * ISP (live J700 ADT /arm-io/isp, sensor-type 0x70, H17 "ISP17a" * firmware, iBoot-loaded). "coproc" is the ISP block, the IRQ @@ -1859,10 +2037,35 @@ reg = <0x3 0x228c0000 0x0 0xc000>, <0x3 0x228d0000 0x0 0x4000>; reg-names = "dart", "dapf"; + /* The ADT's dart-tunables-instance-0 */ + T8140_ISP_DART_TUNABLES; + /* + * The loader's address filter, from the ADT's dart-isp + * dapf-instance-0, written back after a sleep: the + * firmware's PMGR, PMP and DSID windows. + */ + apple,dapf-ranges = /bits/ 64 <0x300704000 0x300704063 + 0x300730000 0x300733fff + 0x300e3c000 0x300e3c003 + 0x31062c000 0x310648003 + 0x401660000 0x401660003 + 0x220004000 0x2200162fc + 0x220044000 0x2200562fc + 0x220084000 0x2200962fc + 0x2200c4000 0x2200d62fc + 0x220104000 0x2201162fc + 0x302824000 0x302824004 + 0x3003c0000 0x3003e4000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_isp_sys>; + /* + * It keeps Linux's values through a wake that kept DRAM up, + * where the sleep engine runs the ISP through it, but its + * address filter works again only once written. + */ + apple,restore-on-power-up; /* * 3-level tables (ias <= 36) as iBoot and the native host * ran them: the ISP fetches at vm-base 0x10000000000 and the @@ -1879,6 +2082,8 @@ isp_dart1: iommu@3228e0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x228e0000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-1 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1899,6 +2104,8 @@ isp_dart2: iommu@322900000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x22900000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-2 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1935,15 +2142,21 @@ interrupt-parent = <&aic>; /* ADT [323 324 325 326 336]; 323 carries the IPC mailbox */ interrupts = ; + /* + * In macOS's order: the camera sensor domains come up + * after ISP_FE and before the rest of the pipeline. + */ power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, - <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_fe>, <&ps_ispsens0>, <&ps_ispsens1>, + <&ps_isp_dprx>, <&ps_isp_secure>, <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, - <&ps_isp_cvm>, <&ps_ispsens0>, <&ps_ispsens1>; + <&ps_isp_cvm>; apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; }; +#undef T8140_ISP_DART_TUNABLES ane_dart0: iommu@401800000 { compatible = "apple,t8140-ane-dart", "apple,t8140-dart", diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 90bdb7e044049..941043525695e 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -100,7 +100,7 @@ CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_MACSMC=y -CONFIG_RTC_DRV_APPLE_PMU=m +CONFIG_RTC_DRV_APPLE_PMU=y CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_MFD_MACSMC=y @@ -163,3 +163,4 @@ CONFIG_EDAC_APPLE_SOC=m # unless the bootloader supplies a window (/chosen/aurora,ctxr-exec-window) or # the ctxr_exec_window= command-line option is given. CONFIG_ARM64_CTXR_EXECMEM=y +CONFIG_APPLE_SOC_SLEEP=y diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index a444c8915e886..4ed50e0dc0b5c 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -7,6 +7,7 @@ #include #include +#include /** * struct cpu_operations - Callback operations for hotplugging CPUs. @@ -46,6 +47,58 @@ struct cpu_operations { #endif }; +/** + * struct spin_table_power_ops - Platform control of parking spin-table CPUs + * (smp_spin_table.c). + * + * @cpu_stop: Tell the hardware the calling CPU is stopping. The CPU keeps + * running, and stays coherent, until @cpu_off. + * @cpu_off: Power off the calling CPU, which must not return. + * @cpu_on: Start a CPU that @cpu_off powered off, at its reset vector. + * + * Each takes the CPU's MPIDR hardware ID. + */ +struct spin_table_power_ops { + void (*cpu_stop)(u64 mpidr); + void (*cpu_off)(u64 mpidr); + int (*cpu_on)(u64 mpidr); +}; + +#ifdef CONFIG_HOTPLUG_CPU +int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +int spin_table_hold_parked(void); +void spin_table_unhold_parked(void); +int spin_table_stop_parked(void); +void spin_table_release_stopped(void); +void spin_table_stop_finished(void); +bool spin_table_cpus_stuck(void); +#else +static inline int spin_table_hold_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_unhold_parked(void) {} + +static inline int spin_table_stop_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_release_stopped(void) {} +static inline void spin_table_stop_finished(void) {} + +static inline int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return -EOPNOTSUPP; +} + +static inline bool spin_table_cpus_stuck(void) +{ + return false; +} +#endif + int __init init_cpu_ops(int cpu); extern const struct cpu_operations *get_cpu_ops(int cpu); diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 0a421a88b3012..65982bd36a11f 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -1089,6 +1089,7 @@ static inline bool cpu_has_lpa2(void) } void __init init_cpucap_indirect_list_impdef(void); +void arm64_impdef_cpu_reset_restore(void); void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index e133011f64b5b..7e88e4b6ca501 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -15,6 +15,9 @@ #include extern const struct cpu_operations smp_spin_table_ops; +#ifdef CONFIG_HOTPLUG_CPU +extern const struct cpu_operations smp_rvbar_spin_table_ops; +#endif #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL extern const struct cpu_operations acpi_parking_protocol_ops; #endif @@ -24,6 +27,9 @@ static const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; static const struct cpu_operations *const dt_supported_cpu_ops[] __initconst = { &smp_spin_table_ops, +#ifdef CONFIG_HOTPLUG_CPU + &smp_rvbar_spin_table_ops, +#endif &cpu_psci_ops, NULL, }; @@ -109,6 +115,19 @@ int __init init_cpu_ops(int cpu) return -EOPNOTSUPP; } +#ifdef CONFIG_HOTPLUG_CPU + /* + * A boot object that sends a reset CPU back through its release address + * says so with /chosen/aurora,s2r-mailbox. Its spin-table CPUs can then + * leave the kernel, through the parking spin table. + */ + if (cpu_ops[cpu] == &smp_spin_table_ops && + of_property_present(of_chosen, "aurora,s2r-mailbox")) { + cpu_ops[cpu] = &smp_rvbar_spin_table_ops; + pr_info_once("spin-table CPUs can park: the boot object brings them back\n"); + } +#endif + return 0; } diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c index 4078f95146c32..f08e210bcb813 100644 --- a/arch/arm64/kernel/cpufeature_impdef.c +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -363,6 +364,23 @@ static const struct arm64_cpu_capabilities arm64_impdef_features[] = { {}, }; +/* + * Restore what these features keep per CPU, on a CPU that was reset without + * going through CPU hotplug: its enable callbacks do not run again, and a + * context switch only writes ACTLR when the thread's value changes. + */ +void arm64_impdef_cpu_reset_restore(void) +{ +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + write_sysreg(current->thread.actlr, actlr_el1); +#endif +#ifdef CONFIG_ARM64_APPLE_AMX + if (cpus_have_cap(ARM64_HAS_AMX_APPLE)) + cpu_disable_el0_amx(NULL); +#endif +} + void __init init_cpucap_indirect_list_impdef(void) { init_cpucap_indirect_list_from_array(arm64_impdef_features); diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 424291c547f02..9398c8d0ecb03 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -336,7 +337,7 @@ int swsusp_arch_suspend(void) unsigned long flags; struct sleep_stack_data state; - if (cpus_are_stuck_in_kernel()) { + if (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck()) { pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); return -EBUSY; } diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index b8e6b6c8aa653..1bbc27e9be509 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -68,7 +68,8 @@ int machine_kexec_prepare(struct kimage *kimage) return -EOPNOTSUPP; } - if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { + if (kimage->type != KEXEC_TYPE_CRASH && + (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck())) { pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); return -EBUSY; } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 49029eace3ad9..2d7660bd22f80 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include extern void secondary_holding_pen(void); @@ -125,3 +127,286 @@ const struct cpu_operations smp_spin_table_ops = { .cpu_prepare = smp_spin_table_cpu_prepare, .cpu_boot = smp_spin_table_cpu_boot, }; + +/* + * A spin table whose CPUs can leave the kernel. + * + * Plain spin-table has no way out of the kernel, so disabling the non-boot + * CPUs for system sleep fails with -EOPNOTSUPP. Some boot monitors leave a + * CPU's reset vector locked to their own image and, after a reset, send the + * CPU back through the same release address. Such a monitor publishes + * /chosen/aurora,s2r-mailbox, and its "spin-table" CPUs then get these ops + * (cpu_ops.c), so the device tree keeps the method every kernel knows. One can + * also name "aurora,rvbar-spin-table" as a CPU's method. + * + * A CPU that really powers off can stop the rest of the system on such a chip, + * so a CPU leaving the kernel parks here instead: still coherent, interrupts + * masked. From there it goes back through secondary_holding_pen, or, when the + * platform suspends the system, it stops and takes its last WFI after the commit. + * It is then started again at its reset vector. Stopping and starting a CPU is + * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug + * fails cleanly, as on plain spin-table. + * + * A CPU left awake at the commit can hang the machine, and the commit can't be + * undone. So each parked CPU first answers a hold, while the suspend can still + * refuse, and only a held CPU stops. + */ +#ifdef CONFIG_HOTPLUG_CPU +static const struct spin_table_power_ops *spin_table_power __read_mostly; +static struct cpumask spin_table_cpus_parked; +static struct cpumask spin_table_cpus_held; +static struct cpumask spin_table_cpus_unpark; +static struct cpumask spin_table_cpus_off; +static bool spin_table_hold_requested; +static bool spin_table_stop_requested; +static bool spin_table_stop_released; + +#define SPIN_TABLE_WAIT_US 100000 + +int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; +} + +/* A refused suspend: the held CPUs go back to the park. */ +void spin_table_unhold_parked(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + dsb(ish); + sev(); +} + +/* + * Every CPU not online is held or off. One that is neither, parked or not (a start that + * failed, a CPU never started), may be awake anywhere. + */ +static bool spin_table_all_held(bool report) +{ + unsigned int cpu; + bool all = true; + + for_each_present_cpu(cpu) { + if (cpu_online(cpu) || cpumask_test_cpu(cpu, &spin_table_cpus_held) || + cpumask_test_cpu(cpu, &spin_table_cpus_off)) + continue; + if (!report) + return false; + pr_err("CPU%u did not answer the hold for a suspend\n", cpu); + all = false; + } + + return all; +} + +/* + * Before a suspend's last refusal, interrupts off, other CPUs offline: every parked CPU + * answers and waits, still parked, for spin_table_stop_parked() or + * spin_table_unhold_parked(). Nothing is written to the hardware. Returns -ETIMEDOUT, + * with the hold undone, if any CPU not online is neither held nor off. + */ +int spin_table_hold_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_hold_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (spin_table_all_held(false)) + return 0; + udelay(1); + } + + if (spin_table_all_held(true)) + return 0; + spin_table_unhold_parked(); + + return -ETIMEDOUT; +} + +/* + * At the commit, after spin_table_hold_parked(): the held CPUs stop and wait for + * spin_table_release_stopped(). Returns -ETIMEDOUT if any didn't stop. The request + * stays up until the wake, so a late one still stops. + */ +int spin_table_stop_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_stop_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (cpumask_empty(&spin_table_cpus_parked)) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +/* After the commit: the stopped CPUs take their last WFI. */ +void spin_table_release_stopped(void) +{ + WRITE_ONCE(spin_table_stop_released, true); + dsb(ish); + sev(); +} + +/* + * Back from a suspend, before starting any CPU. Every CPU lost its power with the + * SoC, so one that never saw the stop is started at its reset vector too. + */ +void spin_table_stop_finished(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + WRITE_ONCE(spin_table_stop_requested, false); + WRITE_ONCE(spin_table_stop_released, false); + cpumask_or(&spin_table_cpus_off, &spin_table_cpus_off, &spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_held); + cpumask_clear(&spin_table_cpus_unpark); +} + +static int smp_rvbar_spin_table_cpu_boot(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + int ret; + + /* The CPU may run straight into the pen: release it before it starts. */ + write_pen_release(mpidr); + + if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { + ret = spin_table_power->cpu_on(mpidr); + if (ret) { + cpumask_set_cpu(cpu, &spin_table_cpus_off); + return ret; + } + } else if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) { + cpumask_set_cpu(cpu, &spin_table_cpus_unpark); + } + + dsb(ish); + sev(); + + return 0; +} + +static bool smp_rvbar_spin_table_cpu_can_disable(unsigned int cpu) +{ + return cpu != 0; +} + +static int smp_rvbar_spin_table_cpu_disable(unsigned int cpu) +{ + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + return -EOPNOTSUPP; + + return READ_ONCE(spin_table_power) ? 0 : -EOPNOTSUPP; +} + +#ifdef CONFIG_KEXEC_CORE +static void smp_rvbar_spin_table_restart(void) +{ + typeof(cpu_soft_restart) *restart; + + cpu_install_idmap(); + restart = (void *)__pa_symbol(cpu_soft_restart); + restart(0, __pa_symbol(secondary_holding_pen), 0, 0, 0); +} +#else +/* Never reached: cpu_die() doesn't leave its loop without KEXEC_CORE. */ +static void smp_rvbar_spin_table_restart(void) +{ +} +#endif + +/* Held until the commit's stop, or until a refusal sends it back to the park. */ +static void smp_rvbar_spin_table_hold(unsigned int cpu, u64 mpidr) +{ + cpumask_set_cpu(cpu, &spin_table_cpus_held); + + while (READ_ONCE(spin_table_hold_requested)) { + if (READ_ONCE(spin_table_stop_requested)) { + spin_table_power->cpu_stop(mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_off); + cpumask_clear_cpu(cpu, &spin_table_cpus_held); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + while (!READ_ONCE(spin_table_stop_released)) + wfe(); + spin_table_power->cpu_off(mpidr); + } + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_held); +} + +static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + + /* The early-boot and crash paths get here without hotplug support. */ + if (!READ_ONCE(spin_table_power) || !IS_ENABLED(CONFIG_KEXEC_CORE)) { + while (1) + wfe(); + } + + cpumask_set_cpu(cpu, &spin_table_cpus_parked); + + while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { + if (READ_ONCE(spin_table_hold_requested)) + smp_rvbar_spin_table_hold(cpu, mpidr); + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_unpark); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + smp_rvbar_spin_table_restart(); +} + +static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) +{ + int us; + + if (!READ_ONCE(spin_table_power)) + return 0; + + for (us = 0; us < SPIN_TABLE_WAIT_US; us += 10) { + if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) + return 0; + udelay(10); + } + + return -ETIMEDOUT; +} + +const struct cpu_operations smp_rvbar_spin_table_ops = { + .name = "aurora,rvbar-spin-table", + .cpu_init = smp_spin_table_cpu_init, + .cpu_prepare = smp_spin_table_cpu_prepare, + .cpu_boot = smp_rvbar_spin_table_cpu_boot, + .cpu_can_disable = smp_rvbar_spin_table_cpu_can_disable, + .cpu_disable = smp_rvbar_spin_table_cpu_disable, + .cpu_die = smp_rvbar_spin_table_cpu_die, + .cpu_kill = smp_rvbar_spin_table_cpu_kill, +}; + +/* + * An offlined parking CPU waits in this kernel's text and stack, which a kexec or a + * hibernation restore overwrites: it is stuck for those, power ops or not. + */ +bool spin_table_cpus_stuck(void) +{ + unsigned int cpu; + + for_each_possible_cpu(cpu) + if (get_cpu_ops(cpu) == &smp_rvbar_spin_table_ops) + return true; + return false; +} +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index c41724a40b756..2a69c79b4ed91 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,13 @@ void notrace __cpu_suspend_exit(void) /* Restore additional feature-specific configuration */ ptrauth_suspend_exit(); + + /* + * A firmware-driven system suspend really reset this CPU. Nothing else on + * this path restores implementation-defined state. + */ + if (pm_resume_via_firmware()) + arm64_impdef_cpu_reset_restore(); } /* diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index c1cc4487c97cb..6f447afa769e2 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "ane_fw_cold.h" @@ -75,6 +76,13 @@ struct ane_fw_backend { u8 data_digest[SHA256_DIGEST_SIZE]; /* Powered down for system sleep; a (cold or warm) boot is due. */ bool slept; + /* + * Parked: powered off with no firmware, as macOS keeps it without a + * client, until a request brings it up (ane_fw_session_wake()). + */ + bool parked; + /* Between PM_SUSPEND_PREPARE and PM_POST_SUSPEND. */ + bool in_sleep; /* The last cold boot after sleep: firmware ready, first replay, count. */ u64 resume_ready_ns, resume_first_ns; unsigned int resume_replayed; @@ -218,8 +226,8 @@ static void ane_fw_snapshot_pmgr(struct ane_fw_backend *b) s->sys = ane_rd32(b, b->pmgr + b->soc->pmgr_sys); s->cpu = ane_rd32(b, b->pmgr + b->soc->pmgr_cpu); s->subblock = ane_rd32(b, b->pmgr + b->soc->pmgr_subblock); - dev_info(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", - s->sys, s->cpu, s->subblock); + dev_dbg(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", + s->sys, s->cpu, s->subblock); } /* Step 2: PWGATE, with ANE_SYS on and before the child power step. */ @@ -229,7 +237,7 @@ static void ane_fw_snapshot_gate(struct ane_fw_backend *b) s->gate_request = ane_rd32(b, b->gate + 0xc); s->gate_ack = ane_rd32(b, b->gate + 0x30); - dev_info(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); + dev_dbg(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); } /* Step 3: the engine and its DARTs, after the child power step. */ @@ -247,8 +255,8 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->pending = ane_rd32(b, b->engine + b->soc->irq_status); for (i = 0; i < 8; i++) s->scratch[i] = ane_rd32(b, b->engine + b->soc->scratch + i * 4); - dev_info(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", - s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); + dev_dbg(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", + s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); for (i = 0; i < ANE_FW_DARTS; i++) { s->dart[i].version = ane_rd32(b, b->dart[i] + 8); s->dart[i].protect = ane_rd32(b, b->dart[i] + 0x200); @@ -257,20 +265,20 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->dart[i].error = ane_rd32(b, b->dart[i] + 0x100); for (j = 0; j < 16; j++) s->dart[i].enabled[j] = ane_rd32(b, b->dart[i] + 0xc00 + 4 * j); - dev_info(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", - i, s->dart[i].version, s->dart[i].protect, - s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); + dev_dbg(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", + i, s->dart[i].version, s->dart[i].protect, + s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); for (j = 0; j < 16; j++) if (s->dart[i].enabled[j]) - dev_info(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, - s->dart[i].enabled[j]); + dev_dbg(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, + s->dart[i].enabled[j]); } for (i = 0; i < b->soc->dapf_count; i++) { void __iomem *r = b->engine + 0x1810000 + i * 0x40; - dev_info(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", - i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), - ane_rd32(b, r + 0x20)); + dev_dbg(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", + i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), + ane_rd32(b, r + 0x20)); } } @@ -508,10 +516,42 @@ static int ane_fw_suspend_firmware(struct ane_fw_backend *b) static int ane_fw_boot(struct ane_fw_backend *b); static int ane_fw_startup(struct ane_fw_backend *b); +static int ane_fw_first_bringup(struct ane_fw_backend *b); +static ssize_t ane_fw_stage_action(struct device *dev, const char *buf, size_t count); static int ane_fw_rvbar_ready(struct ane_fw_backend *b); +/* A power word goes off whole, and off means target, actual and BUSY clear. */ +#define ANE_FW_PS_OFF 0x300 +#define ANE_FW_PS_OFF_MASK 0x8ff +#define ANE_FW_PS_BUSY BIT(11) +#define ANE_FW_PS_UNLATCH BIT(12) + +/* A word left BUSY stalls a wake that keeps DRAM up: clear it, or send this sleep deep. */ +static int ane_fw_ps_off(struct ane_fw_backend *b, u32 offset) +{ + void __iomem *reg = b->pmgr + offset; + int ret; + + ane_wr32(b, ANE_FW_PS_OFF, reg); + ret = ane_poll32(b, reg, ANE_FW_PS_OFF_MASK, 0, 1, 100000); + if (!ret) + return 0; + dev_warn(&b->pdev->dev, "H17 ANE: power word %#x stuck at %#x\n", offset, ane_rd32(b, reg)); + if (!(ane_rd32(b, reg) & 0xff)) { + ane_wr32(b, ANE_FW_PS_OFF | ANE_FW_PS_UNLATCH, reg); + ane_poll32(b, reg, ANE_FW_PS_BUSY, 0, 1, 1000); + ane_wr32(b, ANE_FW_PS_OFF, reg); + if (!ane_poll32(b, reg, ANE_FW_PS_OFF_MASK, 0, 1, 1000)) + return 0; + } + if (!apple_sleep_request_deep_once()) + return ret; + dev_warn(&b->pdev->dev, "H17 ANE: power word %#x still stuck, taking the deep path\n", offset); + return 0; +} + /* Reverse of the proven power-up: sub-block, PWGATE, CPU, then ANE_SYS. */ -static int ane_fw_power_down(struct ane_fw_backend *b) +static int ane_fw_power_down_staged(struct ane_fw_backend *b) { void __iomem *cpu = b->pmgr + b->soc->pmgr_cpu, *sub = b->pmgr + b->soc->pmgr_subblock; u32 sys; @@ -549,6 +589,41 @@ static int ane_fw_power_down(struct ane_fw_backend *b) return 0; } +/* The order macOS uses for this engine: sub-block, PWGATE, its ack, CPU, then ANE_SYS. */ +static int ane_fw_power_down(struct ane_fw_backend *b) +{ + int ret; + + if (!b->soc->macos_sleep) + return ane_fw_power_down_staged(b); + ret = ane_fw_ps_off(b, b->soc->pmgr_subblock); + if (ret) + return ret; + ane_wr32(b, 3, b->gate + 0xc); + ret = ane_poll32(b, b->gate + 0xc, 3, 3, 1, 100000); + if (ret) + return ret; + ane_wr32(b, 0, b->gate + 0x30); + ret = ane_poll32(b, b->gate + 0x30, 1, 0, 1, 100000); + if (ret) + return ret; + ret = ane_fw_ps_off(b, b->soc->pmgr_cpu); + if (ret) + return ret; + /* No engine or DART access from here until ANE_SYS is back on. */ + b->powered = false; + b->power_ref = false; + /* + * Our reference was the last one on ANE_SYS: genpd turns it off once the + * ANE DARTs are idle, or at the latest in the noirq phase of the system + * suspend. ane_fw_genpd_event() records that, and the resume reports a + * domain that stayed on. + */ + pm_runtime_put_sync(b->power); + dev_info(&b->pdev->dev, "H17 ANE: powered down for system sleep\n"); + return 0; +} + /* * After resume: power up from the cold child state, give SID0 its * translation back and the DAPF records, bring CE up, then release the CPU @@ -679,10 +754,10 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) * although genpd really turned ANE_SYS off says whether they live in * an always-on block. */ - dev_info(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", - ane_fw_cold_engine_valid(&b->cold, b->soc) ? "at its reset values" : - "with registers kept", b->sys_offs ? "really went off" : "stayed on", - b->cold.status, ane_rd32(b, b->engine + 0x1810000)); + dev_dbg(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", + ane_fw_cold_engine_valid(&b->cold, b->soc) ? "at its reset values" : + "with registers kept", b->sys_offs ? "really went off" : "stayed on", + b->cold.status, ane_rd32(b, b->engine + 0x1810000)); if (!ane_fw_cold_engine_restartable(&b->cold, b->soc)) return -EPERM; ret = ane_fw_rvbar_ready(b); @@ -719,8 +794,8 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) return ret; b->transport.allow_free = true; b->resume_ready_ns = ktime_get_ns(); - dev_info(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", - div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); + dev_dbg(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", + div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); /* Every open program back behind its handle, as the application left it. */ ane_h16_session_restart(&b->session); ret = ane_h16_session_replay(&b->session, &b->resume_replayed, &b->resume_first_ns); @@ -732,20 +807,235 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) return ret; } -/* The cold way down: nothing in flight, then power off under the firmware. */ +/* + * macOS stops this firmware before its power goes (ANE_deInit, 25G83): command + * 1, then SCRATCH0 armed and POWER_DOWN, which is answered in SCRATCH0 and not + * on the ring. SUSPEND (0x24) is for chips with a warm sleep, not this one. + */ +#define ANE_FW_CMD_STOP 0x1 +#define ANE_FW_STOP_SIZE 12 +#define ANE_FW_CMD_POWER_DOWN 0xb +#define ANE_FW_POWER_DOWN_SIZE 8 +#define ANE_FW_STOP_ARM 0xf7fbdff9 +#define ANE_FW_STOP_DONE 0x08042006 + +static bool clean_stop = true; +module_param(clean_stop, bool, 0644); +MODULE_PARM_DESC(clean_stop, "Stop the H17 firmware before powering it off for system sleep"); + +#define ane_fw_clean_stop(b) (clean_stop && (b)->soc->exclave_mode_stop) + +/* Until SCRATCH0 answers, keep serving what the firmware posts on its way down. */ +static int ane_fw_stop_wait(struct ane_fw_backend *b) +{ + struct ane_fw_transport *t = &b->transport; + void __iomem *scratch0 = b->engine + b->soc->scratch; + u64 deadline = ktime_get_ns() + NSEC_PER_SEC; + u8 report[ANE_FW_SUSPEND_REPORT_MAX]; + u32 pending; + size_t size; + int ret; + + while (ane_rd32(b, scratch0) != ANE_FW_STOP_DONE) { + if (ktime_get_ns() >= deadline) + return -ETIMEDOUT; + ret = ane_fw_transport_consume(t, report, sizeof(report), &size); + if (ret && ret != -EAGAIN) + return ret; + ret = ane_fw_transport_irq(t, &pending); + if (ret) + return ret; + ret = ane_fw_transport_shared(t); + if (ret && ret != -EAGAIN) + return ret; + usleep_range(100, 200); + } + return 0; +} + +/* Every port of every ANE DART idle: what macOS checks before it powers one down. */ +static int ane_fw_dart_idle(struct ane_fw_backend *b) +{ + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) { + u32 params = ane_rd32(b, b->dart[i] + 4); + unsigned int ports = max((params >> 16) & 7, (params >> 20) & 7); + + for (unsigned int p = 0; p < min(ports, 4U); p++) { + int ret = ane_poll32(b, b->dart[i] + 0x230 + 4 * p, U32_MAX, 0, 1, 10000); + + if (ret) { + dev_err(&b->pdev->dev, "H17 ANE: DART %u port %u not idle: %#x\n", + i, p, ane_rd32(b, b->dart[i] + 0x230 + 4 * p)); + return ret; + } + } + } + return 0; +} + +/* + * Each DART as macOS leaves it before the power words (recorded, 25G83): its counters + * cleared, error status acknowledged, every enabled stream disabled one at a time, then + * the TLB flushed. + */ +static void ane_fw_dart_deactivate(struct ane_fw_backend *b) +{ + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) { + void __iomem *dart = b->dart[i]; + u32 enabled; + + ane_wr32(b, U32_MAX, dart + 0x4060); + ane_rd32(b, dart + 0x4060); + ane_rd32(b, dart + 0x4000); + ane_wr32(b, 0, dart + 0x4000); + ane_wr32(b, U32_MAX, dart + 0x704); + ane_rd32(b, dart + 0x704); + enabled = ane_rd32(b, dart + 0xc00); + for (unsigned int bit = 0; bit < 16; bit++) { + if (!(enabled & BIT(bit)) && bit != 11 && bit != 15) + continue; + ane_wr32(b, BIT(bit), dart + 0xc20); + ane_rd32(b, dart + 0xc20); + } + /* Flush all: V2 | STT, as macOS writes it. */ + for (unsigned int j = 0; j < ANE_FW_DARTS; j++) + ane_wr32(b, 0xa000, b->dart[j] + 0x80); + for (unsigned int j = 0; j < ANE_FW_DARTS; j++) + ane_rd32(b, b->dart[j] + 0x80); + } +} + +/* h17 push telemetry control; macOS sets bit 0 at start-up. */ +#define ANE_FW_TELEMETRY 0x196c000 + +/* + * macOS's step right before the stop on this chip (ANE_deInit, its h17 case): + * push telemetry off once bit 4 reads clear, then SCRATCH14 = 0 and wait for + * SCRATCH15 bit 0, then SCRATCH16 = 1 and wait for SCRATCH17 bit 0. Each wait + * is 50 ms and macOS goes on after a miss. + */ +static void ane_fw_stop_handshake(struct ane_fw_backend *b) +{ + void __iomem *scratch = b->engine + b->soc->scratch; + void __iomem *telemetry = b->engine + ANE_FW_TELEMETRY; + struct device *dev = &b->pdev->dev; + + if (ane_poll32(b, telemetry, BIT(4), 0, 10, 50000)) + dev_warn(dev, "H17 ANE: telemetry busy before the stop: %#x\n", ane_rd32(b, telemetry)); + ane_wr32(b, ane_rd32(b, telemetry) & ~BIT(0), telemetry); + ane_wr32(b, 0, scratch + 0x38); + if (ane_poll32(b, scratch + 0x3c, 1, 1, 10, 50000)) + dev_warn(dev, "H17 ANE: SCRATCH15 not set: %#x\n", ane_rd32(b, scratch + 0x3c)); + ane_wr32(b, 1, scratch + 0x40); + if (ane_poll32(b, scratch + 0x44, 1, 1, 10, 50000)) + dev_warn(dev, "H17 ANE: SCRATCH17 not set: %#x\n", ane_rd32(b, scratch + 0x44)); +} + +static int ane_fw_firmware_stop(struct ane_fw_backend *b) +{ + struct ane_fw_transport *t = &b->transport; + struct device *dev = &b->pdev->dev; + void __iomem *status = b->engine + b->soc->wrapper + 0x48; + u8 stop[ANE_FW_STOP_SIZE] = {}; + u8 down[ANE_FW_POWER_DOWN_SIZE] = {}; + u32 val; + int ret; + + /* A core in WFI can't finish an allocation request, so answer those first. */ + ret = ane_fw_transport_settle(t, 2 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC); + if (ret < 0) + return ret; + /* + * No EXCLAVE_MODE_STOP here: macOS's AP never rings that channel around its sleep + * (recorded), and after it this firmware doesn't answer the stop. + */ + /* Without the PMU base the stop can't finish: its DM-bus idle query needs it. */ + if (!b->pmu_base_set) + return -ENODEV; + ane_fw_stop_handshake(b); + put_unaligned_le16(ANE_FW_CMD_STOP, stop + 4); + ret = ane_fw_command(b, ANE_FW_IO, stop, sizeof(stop), ktime_get_ns() + ANE_FW_TIMEOUT_NS); + if (ret) { + dev_err(dev, "H17 ANE: STOP failed: %d\n", ret); + return ret; + } + put_unaligned_le16(ANE_FW_CMD_POWER_DOWN, down + 4); + ane_wr32(b, ANE_FW_STOP_ARM, b->engine + b->soc->scratch); + ret = ane_fw_transport_publish(t, ANE_FW_IO, down, sizeof(down), + ktime_get_ns() + ANE_FW_TIMEOUT_NS); + if (!ret) + ret = ane_fw_stop_wait(b); + if (ret) { + ane_fw_sleep_trace(b, "POWER_DOWN failed", ane_fw_transport_slot(t, ANE_FW_IO)); + ane_h16_session_dump_fwlog(&b->session); + return ret; + } + /* It never completes on the ring; the boot after the wake starts a new one. */ + t->pending[ANE_FW_IO].active = false; + /* macOS waits up to 200 ms for WFI and goes on without it. */ + if (readl_poll_timeout(status, val, val & 3, 10000, 200000)) + dev_warn(dev, "H17 ANE: no WFI after POWER_DOWN, status %#x\n", ane_rd32(b, status)); + dev_info(dev, "H17 ANE: firmware stopped, status %#x\n", ane_rd32(b, status)); + ane_wr32(b, 0xfeedbabe, b->engine + b->soc->scratch + 4); + ret = ane_fw_dart_idle(b); + if (!ret) + ane_fw_dart_deactivate(b); + return ret; +} + +/* The cold way down: nothing in flight, stop the firmware, then power off. */ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) { + int ret; + for (unsigned int ch = 0; ch < ANE_FW_CHANNELS; ch++) if (b->transport.pending[ch].active) return -EBUSY; + if (ane_fw_clean_stop(b)) { + ret = ane_fw_firmware_stop(b); + /* A refused sleep is a failed suspend: the deep path cuts its power cleanly. */ + if (ret && apple_sleep_request_deep_once()) + dev_warn(&b->pdev->dev, "H17 ANE: no clean stop (%d), taking the deep path\n", ret); + else if (ret) + return ret == -EBUSY ? -EIO : ret; + } return ane_fw_power_down(b); } +/* A request for a parked engine: boot it from the preload, as after a sleep. */ +static int ane_fw_session_wake(void *ctx) +{ + struct ane_fw_backend *b = ctx; + int ret; + + if (READ_ONCE(b->stage) == ANE_STAGE_PROBED) + return READ_ONCE(b->in_sleep) ? -EAGAIN : ane_fw_first_bringup(b); + guard(mutex)(&b->lock); + if (!b->parked) + return 0; + /* The system is going to sleep or coming back: PM_POST_SUSPEND wakes waiters. */ + if (b->in_sleep) + return -EAGAIN; + ret = ane_fw_cold_cycle_boot(b); + WRITE_ONCE(b->parked, false); + WRITE_ONCE(b->session.parked, false); + if (ret) { + ane_fw_quarantine(b, ret); + b->session.quarantined = true; + } + ane_h16_session_sleep(&b->session, false); + dev_info(&b->pdev->dev, "H17 ANE: up for a request%s\n", ret ? " FAILED" : ""); + return ret; +} + static int ane_fw_sleep_prepare(struct ane_fw_backend *b) { int ret; guard(mutex)(&b->lock); + /* Parked: already off, with nothing to stop. */ + if (b->parked) + return 0; /* * A quarantined engine that never came up has nothing to lose; give * back an ANE_SYS reference a failed power-up kept. @@ -804,13 +1094,37 @@ static int ane_fw_sleep_prepare(struct ane_fw_backend *b) return 0; } -/* Called by genpd under its lock, in the noirq phase too: record only. */ +static bool dart_off = true; +module_param(dart_off, bool, 0644); +MODULE_PARM_DESC(dart_off, "Deactivate the H17 ANE DARTs before ANE_SYS powers off, as macOS does"); + +/* + * Called by genpd under its lock, in the noirq phase too: nothing that sleeps. + * + * macOS deactivates the three ANE DARTs before every ANE_SYS power-off, with + * or without firmware: it also powers ANE_SYS up for DART work alone and down + * again this way (recorded, 25G83). Here ANE_SYS goes off in genpd, at + * runtime once the ANE and its DARTs are idle or in the noirq phase of a + * system suspend, with the DARTs' streams still enabled; without this step + * it reached off with BUSY latched on most sleeps. Do macOS's step right + * before the off. + */ static int ane_fw_genpd_event(struct notifier_block *nb, unsigned long action, void *data) { struct ane_fw_backend *b = container_of(nb, struct ane_fw_backend, genpd_nb); u64 now = ktime_get_boottime_ns(); - if (action == GENPD_NOTIFY_OFF) { + if (action == GENPD_NOTIFY_PRE_OFF && dart_off && b->soc->exclave_mode_stop && + b->dart[0] && b->dart[1] && b->dart[2]) { + u32 streams[ANE_FW_DARTS]; + + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) + streams[i] = ane_rd32(b, b->dart[i] + 0xc00); + ane_fw_dart_deactivate(b); + dev_info(&b->pdev->dev, + "H17 ANE: DARTs off before ANE_SYS: streams were %#x %#x %#x, PMGR %#x\n", + streams[0], streams[1], streams[2], ane_rd32(b, b->pmgr + b->soc->pmgr_sys)); + } else if (action == GENPD_NOTIFY_OFF) { b->sys_offs++; b->sys_off_pmgr = ane_rd32(b, b->pmgr + b->soc->pmgr_sys); b->sys_off_at = now; @@ -834,6 +1148,13 @@ static void ane_fw_sleep_resume(struct ane_fw_backend *b) if (!b->slept) return; b->slept = false; + /* Stay off until a request needs the engine, as macOS does. */ + if (b->soc->macos_sleep && !b->session.quarantined) { + WRITE_ONCE(b->parked, true); + WRITE_ONCE(b->session.parked, true); + dev_info(&b->pdev->dev, "H17 ANE: resume ok: parked until the next request\n"); + return; + } start = ktime_get_ns(); b->resume_ready_ns = 0; b->resume_first_ns = 0; @@ -854,10 +1175,15 @@ static void ane_fw_sleep_resume(struct ane_fw_backend *b) } ane_h16_session_sleep(&b->session, false); /* genpd's record of the domain across this sleep (boot time, so it counts s2idle). */ - dev_info(&b->pdev->dev, - "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", - b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), - b->sys_off_pmgr, b->sys_on_pmgr); + if (!b->sys_offs || (b->sys_off_pmgr & 0xf0)) + dev_warn(&b->pdev->dev, + "H17 ANE: ANE_SYS stayed on through the sleep: genpd off %u, on %u; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, b->sys_off_pmgr, b->sys_on_pmgr); + else + dev_dbg(&b->pdev->dev, + "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), + b->sys_off_pmgr, b->sys_on_pmgr); /* One line per wake; times from PM_POST_SUSPEND, which follows the thaw. */ ready = b->resume_ready_ns ? div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC) : -1; first = b->resume_first_ns ? div_u64(b->resume_first_ns - start, NSEC_PER_MSEC) : -1; @@ -874,8 +1200,16 @@ static int ane_fw_pm_event(struct notifier_block *nb, unsigned long event, void if (event == PM_POST_SUSPEND) { ane_fw_sleep_resume(b); + WRITE_ONCE(b->in_sleep, false); + WRITE_ONCE(b->session.wake_blocked, false); + /* Requests that waited out the sleep now wake the parked engine. */ + wake_up_all(&b->session.wake); return NOTIFY_DONE; } + if (event == PM_SUSPEND_PREPARE) { + WRITE_ONCE(b->in_sleep, true); + WRITE_ONCE(b->session.wake_blocked, true); + } if (event == PM_SUSPEND_PREPARE && retained) { if (!ane_fw_sleep_prepare(b)) return NOTIFY_DONE; @@ -894,6 +1228,9 @@ static int ane_fw_retain(struct ane_fw_backend *b) { int ret; + if (b->retained) + return 0; + /* * The kexec blocker takes built-in users only. A module, like the H16 * backend, pins itself and refuses system sleep, but cannot veto kexec. @@ -991,7 +1328,7 @@ static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface * ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); if (!ret) { b->next_dva += size; - dev_info(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); } return ret; } @@ -1061,8 +1398,8 @@ static void ane_fw_cold_shared_note(void *ctx, const char *event, u64 dva, u64 s { struct ane_fw_backend *b = ctx; - dev_info(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", - event, tag, size, dva); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", + event, tag, size, dva); } static const struct ane_fw_transport_ops ane_fw_ops = { @@ -1158,7 +1495,7 @@ static int ane_fw_power_up(struct ane_fw_backend *b, bool raw_boot, bool cold_en if ((ane_rd64(b, b->engine + 0x18a4010) & 3) != 3 || ane_rd64(b, b->engine + 0x18a4018)) return -EIO; - dev_info(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); + dev_dbg(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); return 0; } @@ -1369,7 +1706,7 @@ static int ane_fw_boot(struct ane_fw_backend *b) ret = ane_fw_scratch_wait(b, 7, 0x08042006); if (ret) return ret; - dev_info(&b->pdev->dev, "ANE boot: READY\n"); + dev_dbg(&b->pdev->dev, "ANE boot: READY\n"); if (ane_rd32(b, scratch) != 8 || ane_rd32(b, scratch + 4) != 0xc440 || ane_rd32(b, scratch + 12) != 0xf00000 || ane_rd32(b, scratch + 16)) return -EPROTO; @@ -1403,14 +1740,14 @@ static int ane_fw_boot(struct ane_fw_backend *b) ane_wr32(b, 0x08042006, scratch + 3 * 4); ret = ane_fw_scratch_wait(b, 3, 0); if (!ret) - dev_info(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); + dev_dbg(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); /* The firmware's table order, which the macOS channel selectors index. */ for (i = 0; !ret && i < ANE_FW_CHANNELS; i++) { const struct ane_fw_channel *c = &b->transport.channels[i]; - dev_info(&b->pdev->dev, - "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", - ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); + dev_dbg(&b->pdev->dev, + "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", + ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); } return ret; } @@ -1431,22 +1768,24 @@ static int ane_fw_command(struct ane_fw_backend *b, unsigned int channel, } /* - * SET_SNE_PMU_BASE2 (0x29) on the control ring with the DT "pmgr" range - * base, once per cold firmware boot. Without it the 0x24 handler's DM-bus - * idle query never completes (C2-C7; SPEC-ANE-SLEEP-H17-r2.md). + * SET_SNE_PMU_BASE2 (0x29) with the DT "pmgr" range base, once per firmware + * boot, right after START, on the IO ring. macOS's command helper takes a + * queue, not a table position: 0 is IO, 1 DEBUG, 2 DATA_CHAIN_H2T + * (aneFirmwareCommandSend, 25G83), and 0x29 goes on queue 0, never DEBUG. */ static int ane_fw_pmu_base(struct ane_fw_backend *b) { - struct ane_fw_transport *t = &b->transport; + u8 command[ANE_FW_PMU_BASE_SIZE] = {}; u64 base = b->window[1].pa; int ret; b->pmu_base_set = false; - ret = ane_fw_transport_publish_pmu_base(t, base, ktime_get_ns() + ANE_FW_TIMEOUT_NS); - if (!ret) - ret = ane_fw_transport_wait(t, t->control_ch); - dev_info(&b->pdev->dev, "H17 ANE: SET_SNE_PMU_BASE2 (0x29) %#llx on ring %s: %d\n", - base, ane_fw_channel_name(t->control_ch), ret); + put_unaligned_le16(ANE_FW_CMD_SET_PMU_BASE, command + 4); + put_unaligned_le64(base, command + 8); + ret = ane_fw_command(b, ANE_FW_IO, command, sizeof(command), + ktime_get_ns() + ANE_FW_TIMEOUT_NS); + dev_info(&b->pdev->dev, "H17 ANE: SET_SNE_PMU_BASE2 (0x29) %#llx on ring IO: %d\n", + base, ret); if (!ret) b->pmu_base_set = true; return ret; @@ -1472,19 +1811,24 @@ static int ane_fw_startup(struct ane_fw_backend *b) return -EPROTO; if (commands[i].id == 3 && ane_fw_config_status(command, 16)) return -EPROTO; - dev_info(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); + dev_dbg(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); if (commands[i].id == 3) - dev_info(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", - get_unaligned_le32(command + 8)); + dev_dbg(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", + get_unaligned_le32(command + 8)); if (commands[i].id == 0x22) - dev_info(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); + dev_dbg(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); /* macOS order: START, then SET_SNE_PMU_BASE2, then CONFIG_GET. */ - if (commands[i].id == 0 && b->soc->sleep_warm) { + if (commands[i].id == 0 && (b->soc->sleep_warm || ane_fw_clean_stop(b))) { ret = ane_fw_pmu_base(b); if (ret) return ret; } } + /* The host flags macOS sets once the firmware is up (recorded, 25G83). */ + if (b->soc->macos_sleep) { + ane_wr32(b, 1, b->engine + b->soc->scratch + 0x38); + ane_wr32(b, 0, b->engine + b->soc->scratch + 0x40); + } return 0; } @@ -1535,6 +1879,7 @@ static const struct ane_h16_session_ops ane_fw_session_ops = { .log = ane_fw_program_log, .sync = ane_fw_program_sync, .busy = ane_fw_program_busy, + .wake = ane_fw_session_wake, }; /* @@ -1620,6 +1965,68 @@ static int ane_fw_program_stage(struct ane_fw_backend *b) return 0; } +/* + * The node without the firmware: macOS has no ANE running until its first + * client, so the domains stay off and a sleep has nothing to stop. The first + * request maps, boots and starts the engine (ane_fw_first_bringup()). + */ +static int ane_fw_register_parked(struct ane_fw_backend *b) +{ + int ret; + + ret = ane_fw_retain(b); + if (ret) + return ret; + b->transport.soc = b->soc; + ret = ane_h16_session_init_parked(&b->session, &b->transport, &ane_fw_session_ops, b, + ANE_FW_TIMEOUT_NS); + if (ret) + return ret; + ret = apple_clpc_ane_get(&b->pdev->dev, &b->clpc); + if (ret) + return ret; + b->session.gate_hold_ns = ANE_FW_GATE_HOLD_NS; + ret = ane_h16_accel_register(&b->pdev->dev, &b->session); + if (ret) { + apple_clpc_ane_put(b->clpc); + b->clpc = NULL; + return ret; + } + ane_fw_debugfs(b); + WRITE_ONCE(b->parked, true); + return 0; +} + +/* The first request: what the boot used to do, map, boot and start-up, then admit it. */ +static int ane_fw_first_bringup(struct ane_fw_backend *b) +{ + static const char *const steps[] = { "map", "boot", "startup" }; + unsigned int flags, i; + ssize_t ret = 0; + + flags = lock_system_sleep(); + if (READ_ONCE(b->stage) != ANE_STAGE_PROBED) + goto out; + for (i = 0; ret >= 0 && i < ARRAY_SIZE(steps); i++) + ret = ane_fw_stage_action(&b->pdev->dev, steps[i], strlen(steps[i])); + mutex_lock(&b->lock); + if (ret >= 0) { + b->transport.allow_free = true; + b->stage = ANE_STAGE_PROGRAM; + ret = 0; + } else { + b->session.quarantined = true; + } + WRITE_ONCE(b->parked, false); + WRITE_ONCE(b->session.parked, false); + ane_h16_session_sleep(&b->session, false); + mutex_unlock(&b->lock); + dev_info(&b->pdev->dev, "H17 ANE: first bring-up for a request: %zd\n", ret); +out: + unlock_system_sleep(flags); + return ret; +} + static ssize_t firmware_stage_show(struct device *dev, struct device_attribute *attr, char *buf) { static const char * const names[] = { @@ -1762,10 +2169,10 @@ static bool ane_fw_handoff_present(struct platform_device *pdev, * on every boot with a loader hand-off. A failure stops at that stage; after * acquisition it quarantines, as from the stage file. */ -static void ane_fw_start_work(struct work_struct *work) +/* The staged bring-up at boot, for SoCs without macos_sleep. */ +static void ane_fw_start_staged(struct ane_fw_backend *b) { static const char *const steps[] = { "probe", "map", "boot", "startup", "program" }; - struct ane_fw_backend *b = container_of(work, struct ane_fw_backend, start); unsigned int i, flags; ssize_t ret; @@ -1783,6 +2190,32 @@ static void ane_fw_start_work(struct work_struct *work) dev_info(&b->pdev->dev, "H17 ANE: up at boot, accel node ready\n"); } +static void ane_fw_start_work(struct work_struct *work) +{ + struct ane_fw_backend *b = container_of(work, struct ane_fw_backend, start); + unsigned int flags; + ssize_t ret; + + if (!b->soc->macos_sleep) { + ane_fw_start_staged(b); + return; + } + flags = lock_system_sleep(); + ret = ane_fw_stage_action(&b->pdev->dev, "probe", strlen("probe")); + if (ret >= 0) { + mutex_lock(&b->lock); + ret = ane_fw_register_parked(b); + mutex_unlock(&b->lock); + } + unlock_system_sleep(flags); + /* Nothing holds ANE_SYS now: with no firmware, genpd turns it off, as macOS keeps it. */ + ane_fw_bind_release(b); + if (ret < 0) + dev_err(&b->pdev->dev, "H17 ANE: node registration stopped: %zd\n", ret); + else + dev_info(&b->pdev->dev, "H17 ANE: accel node ready; firmware starts on the first request\n"); +} + static void ane_fw_observer_free(struct ane_fw_backend *b) { unsigned int i; diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index 222da774e6ded..5639f340d0a61 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -118,6 +118,7 @@ const struct ane_fw_platform ane_h17_platform = { .safe_reset = false, /* 25G83 boots this firmware in Exclave mode (SPEC-ANE-SLEEP-H17.md). */ .exclave_mode_stop = true, + .macos_sleep = true, /* * Off: on 25G83 SUSPEND is still not taken after 0x31 (C7), so sleep * cycles the engine cold and replays every program behind its handle. diff --git a/drivers/accel/ane/ane_fw_platform.h b/drivers/accel/ane/ane_fw_platform.h index 5f59952fb123c..e4afe348b30c1 100644 --- a/drivers/accel/ane/ane_fw_platform.h +++ b/drivers/accel/ane/ane_fw_platform.h @@ -66,6 +66,12 @@ struct ane_fw_platform { * (EXCLAVE_MODE_STOP on the control ring) so that SUSPEND can run. */ bool exclave_mode_stop; + /* + * The sleep macOS was recorded doing on this SoC: the engine stays off with + * no firmware until its first request, and powers down in macOS's word + * order. Other SoCs keep the staged backend's boot-time start. + */ + bool macos_sleep; /* * System sleep keeps the firmware (EXCLAVE_MODE_STOP, SUSPEND, warm * boot); otherwise the engine is cycled cold and the programs replayed. diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index 3f2de0cfe288a..1a5373b787c1d 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -512,12 +512,12 @@ static void h16_drain(struct ane_h16_session *s) wake_up_all(&s->wake); } -int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, - const struct ane_h16_session_ops *ops, void *ctx, - u64 timeout_ns) +static int h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns, bool parked) { - if (!s || !transport || !transport->armed || !ops || !ops->map || !ops->log || - !timeout_ns) + if (!s || !transport || !transport->soc || (!parked && !transport->armed) || !ops || + !ops->map || !ops->log || !timeout_ns || (parked && !ops->wake)) return -EINVAL; memset(s, 0, sizeof(*s)); s->transport = transport; @@ -535,10 +535,28 @@ int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *tra spin_lock_init(&s->export_lock); INIT_LIST_HEAD(&s->exports_done); INIT_WORK(&s->export_work, h16_export_work); + s->asleep = parked; + s->parked = parked; return 0; } + +int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns) +{ + return h16_session_init(s, transport, ops, ctx, timeout_ns, false); +} EXPORT_SYMBOL_GPL(ane_h16_session_init); +/* Closed and parked, its transport armed only by ops->wake on the first request. */ +int ane_h16_session_init_parked(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns) +{ + return h16_session_init(s, transport, ops, ctx, timeout_ns, true); +} +EXPORT_SYMBOL_GPL(ane_h16_session_init_parked); + static int h16_fail(struct ane_h16_session *s, int error) { if (!s->error) @@ -1756,10 +1774,18 @@ int ane_h16_session_enter(struct ane_h16_session *s, bool nonblock) int ret; for (;;) { + /* A parked engine comes up for its first request. */ + if (READ_ONCE(s->parked) && s->ops->wake) { + ret = s->ops->wake(s->ctx); + if (ret && ret != -EAGAIN) + return ret; + } /* O_NONBLOCK: a clean -EAGAIN while the engine is cycled. */ if (nonblock && READ_ONCE(s->asleep)) return -EAGAIN; - ret = wait_event_freezable(s->wake, !READ_ONCE(s->asleep)); + ret = wait_event_freezable(s->wake, !READ_ONCE(s->asleep) || + (READ_ONCE(s->parked) && s->ops->wake && + !READ_ONCE(s->wake_blocked))); if (ret) return ret; mutex_lock(s->lock); diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h index b26537be6de1d..a40051abd9b37 100644 --- a/drivers/accel/ane/ane_h16_program.h +++ b/drivers/accel/ane/ane_h16_program.h @@ -168,6 +168,11 @@ struct ane_h16_session_ops { * hold a performance vote while any request is in flight. */ void (*busy)(void *ctx, bool busy); + /* + * Optional. The engine is parked (off, no firmware) until a request needs + * it: bring it up, or return an error if it can't come up now. + */ + int (*wake)(void *ctx); }; /* @@ -301,6 +306,10 @@ struct ane_h16_session { * ane_h16_session_enter() and closes are deferred to the reopen. */ bool asleep; + /* Off until the next request, which calls ops->wake first. */ + bool parked; + /* A system sleep is under way: a parked engine can't come up until it ends. */ + bool wake_blocked; struct list_head deferred; /* * Firmware boot count: a cold restart across system sleep loses every @@ -354,6 +363,10 @@ struct ane_h16_session { int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, const struct ane_h16_session_ops *ops, void *ctx, u64 timeout_ns); +int ane_h16_session_init_parked(struct ane_h16_session *s, + struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns); int ane_h16_session_load(struct ane_h16_session *s, struct ane_h16_hwx_snapshot *snapshot, const void *const aux[ANE_H16_AUX_COUNT], diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 72910ef976f8e..d60458d69c590 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -65,6 +65,8 @@ /* Vendor command telling the controller that the host goes to sleep. */ #define BT7932_OP_HOST_SLEEP 0xfc6f #define BT7932_HOST_SLEEP_PLEN 7 +/* How long a wake reason may take to arrive once the transport is back. */ +#define BT7932_WAKE_REASON_MS 200 #define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" #define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" @@ -153,6 +155,8 @@ struct bt7932 { bool tx_blocked; /* nothing is sent until resume */ bool irq_held; /* IRQ off until the PCI state is restored */ bool pci_saved; /* saved PCI state not yet restored */ + bool wake_quiesced; /* in host sleep with HCI set up to wake, deep sleep */ + bool wake_expected; /* the next wake report is what woke the host */ enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; }; @@ -300,6 +304,70 @@ static bool bt7932_wake_report(struct bt7932 *bt, const u8 *data, size_t len) } } +/* + * Names a wake report (see bt7932_wake_event()) for the log, and points @addr + * at the remote address it carries, or NULL. + */ +static const char *bt7932_wake_name(const u8 *data, size_t len, const u8 **addr) +{ + /* Where the address starts: after status and handle, or LE report fields. */ + size_t offset = 0; + const char *name = "event"; + + switch (data[0]) { + case HCI_EV_CONN_REQUEST: + name = "connection request"; + offset = HCI_EVENT_HDR_SIZE; + break; + case HCI_EV_CONN_COMPLETE: + name = "connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_SYNC_CONN_COMPLETE: + name = "synchronous connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_HARDWARE_ERROR: + name = "hardware error"; + break; + case HCI_EV_LE_META: + if (len < HCI_EVENT_HDR_SIZE + 1) + break; + /* Subevent, report count, event type (two bytes if extended), address type. */ + switch (data[HCI_EVENT_HDR_SIZE]) { + case HCI_EV_LE_ADVERTISING_REPORT: + name = "LE advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_DIRECT_ADV_REPORT: + name = "LE directed advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_EXT_ADV_REPORT: + name = "LE extended advertising report"; + offset = HCI_EVENT_HDR_SIZE + 5; + break; + } + break; + } + *addr = offset && len >= offset + sizeof(bdaddr_t) ? data + offset : NULL; + return name; +} + +/* Caller owns lock and expects the wake reason. */ +static void bt7932_log_wake(struct bt7932 *bt, const u8 *data, size_t len) +{ + const u8 *addr; + const char *name = bt7932_wake_name(data, len, &addr); + + if (addr) + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s from %pMR\n", name, addr); + else + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s\n", name); + bt->wake_expected = false; + wake_up_all(&bt->wake_wait); +} + static int bt7932_validate_ptx(const u8 *data, size_t size) { static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; @@ -520,11 +588,13 @@ static int bt7932_sleep_locked(struct bt7932 *bt) } if (value > 1) return -EPROTO; - /* A previous host-sleep notification must not cancel control 3. */ - if (bt->pm_state == BT7932_QUIESCING) + /* + * A previous host-sleep notification must not cancel control 3. Once + * quiesced, only the host ends the host sleep, so leave the rest to + * bt7932_restore_transport(), as macOS ignores it then too. + */ + if (bt->pm_state == BT7932_QUIESCING || bt->pm_state == BT7932_QUIESCED) return 0; - if (bt->pm_state == BT7932_QUIESCED) - return -EPROTO; if (bt->pm_state == BT7932_RESTORING) { if (value) { if (!bt->waking) { @@ -725,8 +795,11 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) * remember that it arrived. */ if (geo->packet == HCI_EVENT_PKT && bt->wake_tracking && - bt7932_wake_report(bt, descriptor + 16, length)) + bt7932_wake_report(bt, descriptor + 16, length)) { + if (bt->wake_expected) + bt7932_log_wake(bt, descriptor + 16, length); pm_wakeup_dev_event(&bt->pdev->dev, 0, true); + } hci_recv_frame(bt->hdev, skb); } slot = (slot + 1) & BT7932_MASK; @@ -743,9 +816,9 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) return 0; } -static irqreturn_t bt7932_irq_thread(int irq, void *data) +/* Takes what the firmware published: sleep notifications, completions, packets. */ +static void bt7932_service(struct bt7932 *bt) { - struct bt7932 *bt = data; u16 producer; int channel, ret; @@ -785,6 +858,11 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) queue_work(bt->tx_wq, &bt->tx_work); out: mutex_unlock(&bt->lock); +} + +static irqreturn_t bt7932_irq_thread(int irq, void *data) +{ + bt7932_service(data); return IRQ_HANDLED; } @@ -1125,7 +1203,7 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) * Follows the wakeup setting of the PCI function, see bt7932_probe(). * Only while it is on does hci_suspend_sync() leave page scan and LE passive * scan set up for the devices flagged for remote wakeup, and only then does - * bt7932_suspend() keep the transport running with its interrupt armed. + * bt7932_suspend() leave the controller able to wake the system. */ static bool bt7932_wakeup(struct hci_dev *hdev) { @@ -1487,29 +1565,33 @@ static bool bt7932_hci_pm(struct bt7932 *bt) } /* - * Suspend-to-idle can keep the transport running so that the controller can - * wake the system: with wakeup enabled on the PCI function, the controller - * up and HCI suspended by this driver. + * The controller can wake the system with wakeup enabled on the PCI function, the controller + * up and HCI suspended by this driver. In suspend-to-idle the transport stays up and the wake + * is its MSI. A deep sleep takes the link down: the controller goes into host sleep, which is + * the quiesce, and wakes the system through the SMC, as macOS does. */ static bool bt7932_may_wake(struct bt7932 *bt) { - return device_may_wakeup(&bt->pdev->dev) && pm_suspend_no_platform() && - bt7932_hci_pm(bt) && test_bit(HCI_UP, &bt->hdev->flags); + return device_may_wakeup(&bt->pdev->dev) && bt7932_hci_pm(bt) && + test_bit(HCI_UP, &bt->hdev->flags); } static void bt7932_track_wake(struct bt7932 *bt, bool on) { mutex_lock(&bt->lock); bt->wake_tracking = on; + if (!on) + bt->wake_expected = false; mutex_unlock(&bt->lock); } /* - * Before system sleep, the host tells the controller that it is going to - * sleep with a vendor command whose parameters are all ones; nothing is sent - * when the host is back. It is answered by a vendor event, if at all, so - * place it on the command ring directly rather than through the HCI command - * flow, and wait until the controller has taken it. + * For suspend-to-idle with the transport kept running: a vendor command whose + * parameters are all ones. macOS uses the same bytes, but only as a request to + * its own transport, which turns them into the host sleep of a deep sleep (see + * bt7932_suspend()) and never sends them on. It is answered by a vendor event, + * if at all, so place it on the command ring directly rather than through the + * HCI command flow, and wait until the controller has taken it. * * Returns -EAGAIN if nothing was published (the controller could not be * reached without a fault, or the ring was full), -EBUSY if the command was @@ -1583,6 +1665,7 @@ static int bt7932_arm_wakeup(struct bt7932 *bt) ret = bt->fault ? -EIO : 0; if (!ret) { bt->wake_armed = true; + bt->wake_expected = true; /* A failed wake handshake would fault the transport. */ bt->tx_blocked = true; } @@ -1607,7 +1690,6 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) mutex_lock(&bt->lock); bt->wake_armed = false; bt->tx_blocked = false; - bt->wake_tracking = false; bt->pci_saved = false; mutex_unlock(&bt->lock); wake_up_all(&bt->wake_wait); @@ -1617,6 +1699,28 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) } } +/* + * After a sleep set up for Bluetooth wake, the controller delivers what woke + * the host once the transport runs again. Take it while HCI is still + * suspended, so that the core records the wake reason too, and log it. + * Ends wake tracking. + */ +static void bt7932_take_wake_reason(struct bt7932 *bt) +{ + /* The interrupt thread leaves the rings alone until the restore ends. */ + bt7932_service(bt); + wait_event_timeout(bt->wake_wait, + !READ_ONCE(bt->wake_expected) || READ_ONCE(bt->fault), + msecs_to_jiffies(BT7932_WAKE_REASON_MS)); + mutex_lock(&bt->lock); + if (bt->wake_expected && !bt->fault) + dev_dbg(&bt->pdev->dev, "Bluetooth did not wake the host\n"); + mutex_unlock(&bt->lock); + bt7932_track_wake(bt, false); + /* The core takes a wake reason only until hci_resume_dev(). */ + flush_work(&bt->hdev->rx_work); +} + /* * With wakeup enabled, the transport stays up through suspend-to-idle * instead of being quiesced: @@ -1637,6 +1741,12 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) * The PCIe link and host bridge stay up in suspend-to-idle anyway, as the * Wi-Fi function on the same link is never stopped, and ASPM stays off on * this link as the transport was admitted. + * + * A deep sleep with wakeup enabled quiesces the transport after HCI has set + * up its wake scans. The quiesce (sleep control 3, notification 3) is the + * host sleep that macOS enters at S2R. The controller keeps scanning and + * wakes the system through the SMC. bt7932_resume() ends the host sleep + * with control 0, as macOS does, and then takes the wake reason. */ static int bt7932_suspend(struct device *dev) { @@ -1644,8 +1754,9 @@ static int bt7932_suspend(struct device *dev) bool wake; int ret; + /* A transport that's down has nothing to stop, and it can't cost a deep sleep. */ if (READ_ONCE(bt->fault)) - return -EIO; + return pm_suspend_no_platform() ? -EIO : 0; if (!bt->registered) return 0; wake = bt7932_may_wake(bt); @@ -1656,16 +1767,19 @@ static int bt7932_suspend(struct device *dev) ret = hci_suspend_dev(bt->hdev); if (ret) { bt7932_track_wake(bt, false); - return ret; + if (pm_suspend_no_platform()) + return ret; + dev_warn(dev, "HCI did not suspend: %d\n", ret); } } - if (wake) { + if (wake && pm_suspend_no_platform()) { ret = bt7932_arm_wakeup(bt); if (!ret) { - dev_dbg(dev, "system sleep with wakeup armed\n"); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } bt7932_track_wake(bt, false); + wake = false; if (ret == -EBUSY) { /* * The host sleep command may still take effect, so the @@ -1676,8 +1790,7 @@ static int bt7932_suspend(struct device *dev) hci_resume_dev(bt->hdev); bt7932_reset(bt->hdev); return ret; - } - if (ret != -EAGAIN) { + } else if (ret != -EAGAIN) { dev_err(dev, "transport failed while arming wakeup\n"); hci_resume_dev(bt->hdev); return ret; @@ -1686,9 +1799,28 @@ static int bt7932_suspend(struct device *dev) ret = bt7932_quiesce(bt); if (!ret) { dev_dbg(dev, "system sleep quiesce acknowledged\n"); + if (!wake) + return 0; + /* hci_suspend_dev() may have stopped short of the wake setup. */ + if (!bt->hdev->suspended || + bt->hdev->suspend_state != BT_SUSPEND_CONFIGURE_WAKE) { + bt7932_track_wake(bt, false); + dev_warn(dev, "no Bluetooth wake for this sleep\n"); + return 0; + } + mutex_lock(&bt->lock); + bt->wake_quiesced = true; + bt->wake_expected = true; + mutex_unlock(&bt->lock); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } + if (wake) + bt7932_track_wake(bt, false); dev_warn(dev, "system sleep quiesce failed: %d\n", ret); + /* The wake finds out what's left. A deep sleep takes the power regardless. */ + if (!pm_suspend_no_platform()) + return 0; /* Undo our request before the PM core rolls back the rest of the bus. */ if (!READ_ONCE(bt->fault)) bt7932_restore_transport(bt); @@ -1732,23 +1864,43 @@ static int bt7932_resume_noirq(struct device *dev) static int bt7932_resume(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + bool wake_quiesced = bt->wake_quiesced; int ret; + bt->wake_quiesced = false; + /* A deep sleep retrains the link. With the chip gone, nothing here may touch it. */ + if (!pci_device_is_present(bt->pdev)) { + bt7932_track_wake(bt, false); + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "radio gone after the sleep"); + mutex_unlock(&bt->lock); + apple_piodma_radio_recover(); + return -ENODEV; + } + if (bt->wake_armed) { bt7932_disarm_wakeup(bt); + bt7932_take_wake_reason(bt); if (READ_ONCE(bt->fault)) return -EIO; return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } - if (READ_ONCE(bt->fault)) + if (READ_ONCE(bt->fault)) { + bt7932_track_wake(bt, false); return -EIO; + } if (!bt->registered) return 0; if (READ_ONCE(bt->pm_state) != BT7932_RUNNING) { ret = bt7932_restore_transport(bt); - if (ret) + if (ret) { + bt7932_track_wake(bt, false); + apple_piodma_radio_recover(); return ret; + } } + if (wake_quiesced) + bt7932_take_wake_reason(bt); dev_dbg(dev, "transport restored after system sleep\n"); return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } @@ -1778,9 +1930,15 @@ static void bt7932_remove(struct pci_dev *pdev) u16 command = U16_MAX; bool quiesced, drained; - /* Closing HCI may send Reset: keep the transport and IRQ alive for it. */ - if (bt->registered) + /* + * The HCI core stops its command and event work before it closes a device it + * unregisters, so a Reset on that close goes unsent and waits out its 2 s timeout. + * The quiesce below stops the firmware. Close still runs with the transport alive. + */ + if (bt->registered) { + hci_clear_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_unregister_dev(bt->hdev); + } quiesced = bt->ipc_ready && !READ_ONCE(bt->fault) && !bt7932_quiesce(bt); mutex_lock(&bt->lock); WRITE_ONCE(bt->stopping, true); diff --git a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c index 61e93ce1f31db..8a57f9f30ca1a 100644 --- a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c +++ b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c @@ -65,11 +65,57 @@ static void bt7932_wake_event_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, bt7932_wake_event(error, sizeof(error)), BT7932_WAKE_ERROR); } +/* The wake log names the event and the remote address it carries. */ +static void bt7932_wake_name_bredr_test(struct kunit *test) +{ + static const u8 request[] = { HCI_EV_CONN_REQUEST, 10, + 1, 2, 3, 4, 5, 6, 0x40, 0x25, 0x00, 0x01 }; + static const u8 complete[] = { HCI_EV_CONN_COMPLETE, 11, 0x00, 0x01, 0x00, + 1, 2, 3, 4, 5, 6, 0x01, 0x00 }; + static const u8 sync_short[] = { HCI_EV_SYNC_CONN_COMPLETE, 1, 0x00 }; + static const u8 error[] = { HCI_EV_HARDWARE_ERROR, 1, 0x00 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(request, sizeof(request), &addr), + "connection request"); + KUNIT_EXPECT_PTR_EQ(test, addr, &request[2]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(complete, sizeof(complete), &addr), + "connection complete"); + KUNIT_EXPECT_PTR_EQ(test, addr, &complete[5]); + bt7932_wake_name(sync_short, sizeof(sync_short), &addr); + KUNIT_EXPECT_NULL(test, addr); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(error, sizeof(error), &addr), "hardware error"); + KUNIT_EXPECT_NULL(test, addr); +} + +static void bt7932_wake_name_le_test(struct kunit *test) +{ + /* Subevent, one report, event type, address type, address, data length, RSSI. */ + static const u8 adv[] = { HCI_EV_LE_META, 12, HCI_EV_LE_ADVERTISING_REPORT, 1, + 0x00, 0x01, 1, 2, 3, 4, 5, 6, 0x00, 0xc0 }; + /* The extended report's event type takes two bytes. */ + static const u8 ext[] = { HCI_EV_LE_META, 11, HCI_EV_LE_EXT_ADV_REPORT, 1, + 0x13, 0x00, 0x01, 1, 2, 3, 4, 5, 6 }; + static const u8 adv_short[] = { HCI_EV_LE_META, 2, HCI_EV_LE_ADVERTISING_REPORT, 1 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(adv, sizeof(adv), &addr), + "LE advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &adv[6]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(ext, sizeof(ext), &addr), + "LE extended advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &ext[7]); + bt7932_wake_name(adv_short, sizeof(adv_short), &addr); + KUNIT_EXPECT_NULL(test, addr); +} + static struct kunit_case bt7932_wake_test_cases[] = { KUNIT_CASE(bt7932_wake_event_bredr_test), KUNIT_CASE(bt7932_wake_event_le_test), KUNIT_CASE(bt7932_wake_event_replies_test), KUNIT_CASE(bt7932_wake_event_error_test), + KUNIT_CASE(bt7932_wake_name_bredr_test), + KUNIT_CASE(bt7932_wake_name_le_test), {} }; diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index fb87ae6ae137a..379fa031ce599 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -837,6 +838,19 @@ static unsigned int apple_soc_cpufreq_pstate_freq(struct cpufreq_policy *policy, static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; + struct apple_fast_die *fd = priv->fast_die; + + /* + * A firmware-driven sleep stops the PMP, and its driver starts it again only after the + * CPUs are back: the first samples after the wake find it down. Give it the same time to + * come back as at boot. Jiffies don't count the sleep. + */ + if (fd && fd->report && pm_suspend_via_firmware()) { + spin_lock_bh(&fd->lock); + fd->start = jiffies; + fd->warned = false; + spin_unlock_bh(&fd->lock); + } if (priv->info->lazy_verify) { int pstate = apple_soc_cpufreq_drain(priv); @@ -856,6 +870,37 @@ static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) return cpufreq_generic_suspend(policy); } +/* + * A firmware-driven sleep powers the cluster down, and it comes back in a performance state + * of the hardware's choosing, not the one asked for at suspend. Take what it reads as the + * expected state again, before the governor starts, rather than stopping every request as if + * someone else owned it. + */ +static int apple_soc_cpufreq_resume(struct cpufreq_policy *policy) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned long flags; + u64 reg; + + if (!priv->info->verify_transition || !pm_resume_via_firmware()) + return 0; + + raw_spin_lock_irqsave(&priv->lock, flags); + reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + if (!(reg & APPLE_DVFS_CMD_BUSY)) { + unsigned int now = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); + + if (now != priv->expected_pstate) + dev_info(priv->cpu_dev, "DVFS state %u after the sleep, %u before\n", + now, priv->expected_pstate); + priv->expected_pstate = now; + priv->transition_failed = false; + } + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { @@ -1571,6 +1616,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, .suspend = apple_soc_cpufreq_suspend, + .resume = apple_soc_cpufreq_resume, }; static int __init apple_soc_cpufreq_module_init(void) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index 939e5d5e7bdb9..3f21320463ccb 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dmaengine.h" @@ -141,6 +142,12 @@ struct admac_chan { struct list_head issued; struct list_head to_free; + + /* The client's last configuration, set up again after a system sleep. */ + struct dma_slave_config cfg; + bool cfg_valid; + /* A system sleep took the channel's registers; set it up before a transfer. */ + bool lost; }; struct admac_sram { @@ -640,11 +647,30 @@ static void admac_reset_rings(struct admac_chan *adchan) writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, adchan->no)); } +static int admac_device_config(struct dma_chan *chan, struct dma_slave_config *config); + +/* Registers can't be reached before a client powers the fabric, so this waits for it. */ +static void admac_restore_chan(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + + if (!adchan->lost) + return; + adchan->lost = false; + + if (ad->split_channels) + writel_relaxed(0x18080001, ad->base + REG_CTL); + writel_relaxed(adchan->carveout, ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); + if (adchan->cfg_valid) + admac_device_config(&adchan->chan, &adchan->cfg); +} + static void admac_start_current_tx(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; int ch = adchan->no; + admac_restore_chan(adchan); admac_reset_rings(adchan); writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, ch)); if (ad->split_channels) { @@ -764,12 +790,21 @@ static int admac_resume(struct dma_chan *chan) int ret = 0; spin_lock_irqsave(&adchan->lock, flags); - if (adchan->current_tx && adchan->current_tx->error) + if (adchan->current_tx && adchan->current_tx->error) { ret = -EIO; - else if (admac_resume_realigns(adchan)) + } else if (adchan->lost && adchan->current_tx) { + struct admac_tx *tx = adchan->current_tx; + + /* The rings went with the registers: start again from the last period reclaimed. */ + tx->submitted_pos = tx->reclaimed_pos; + tx->nperiods_outstanding = 0; + adchan->nperiod_acks = 0; + admac_start_current_tx(adchan); + } else if (admac_resume_realigns(adchan)) { admac_cyclic_realign(adchan); - else + } else { admac_start_chan(adchan); + } spin_unlock_irqrestore(&adchan->lock, flags); return ret; @@ -1146,6 +1181,9 @@ static int admac_device_config(struct dma_chan *chan, | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), ad->base + REG_CHAN_FIFOCTL(ad, adchan->no)); + adchan->cfg = *config; + adchan->cfg_valid = true; + return 0; } @@ -1175,8 +1213,36 @@ static int admac_runtime_resume(struct device *dev) return 0; } +/* + * A firmware-driven sleep, refused or not, takes the registers of a channel a + * client still holds (genpd gates the domain at noirq), and its client won't + * configure it again: only a new hw_params does that. + */ +static int __maybe_unused admac_resume_early(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + unsigned long flags; + int i; + + if (!pm_suspend_via_firmware()) + return 0; + + mutex_lock(&ad->cache_alloc_lock); + ad->ctl_programmed = false; + mutex_unlock(&ad->cache_alloc_lock); + + for (i = 0; i < ad->nchannels; i++) { + spin_lock_irqsave(&ad->channels[i].lock, flags); + ad->channels[i].lost = true; + spin_unlock_irqrestore(&ad->channels[i].lock, flags); + } + + return 0; +} + /* Allocated channels must stay powered, including paused cyclic transfers. */ static const struct dev_pm_ops admac_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, admac_resume_early) RUNTIME_PM_OPS(admac_runtime_suspend, admac_runtime_resume, NULL) }; diff --git a/drivers/dma/apple-sio-boot-test.c b/drivers/dma/apple-sio-boot-test.c index 3d8f93d11d8bb..5994518f619b0 100644 --- a/drivers/dma/apple-sio-boot-test.c +++ b/drivers/dma/apple-sio-boot-test.c @@ -187,12 +187,36 @@ static void sio_boot_dma_arrays_test(struct kunit *test) } } +/* Only a channel with nothing on it may pass through a firmware restart. */ +static void sio_sleep_busy_test(struct kunit *test) +{ + struct sio_chan *chan = kunit_kzalloc(test, sizeof(*chan), GFP_KERNEL); + struct sio_tx *tx = kunit_kzalloc(test, sizeof(*tx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, chan); + KUNIT_ASSERT_NOT_NULL(test, tx); + spin_lock_init(&chan->vc.lock); + + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); + chan->current_tx = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->current_tx = NULL; + chan->idle = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->idle = NULL; + chan->ring = true; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->ring = false; + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); +} + static struct kunit_case sio_boot_test_cases[] = { KUNIT_CASE(sio_boot_order_test), KUNIT_CASE(sio_boot_optional_tables_test), KUNIT_CASE(sio_boot_bad_params_test), KUNIT_CASE(sio_boot_control_ack_failure_test), KUNIT_CASE(sio_boot_dma_arrays_test), + KUNIT_CASE(sio_sleep_busy_test), {} }; diff --git a/drivers/dma/apple-sio-probe-test.c b/drivers/dma/apple-sio-probe-test.c index c486c5072f07c..9b9c5a0774320 100644 --- a/drivers/dma/apple-sio-probe-test.c +++ b/drivers/dma/apple-sio-probe-test.c @@ -72,6 +72,8 @@ static void sio_match_data_test(struct kunit *test) KUNIT_EXPECT_EQ(test, hw->fifo_threshold, 0x800U); KUNIT_EXPECT_FALSE(test, hw->clear_period_desc); KUNIT_EXPECT_FALSE(test, hw->check_boot_tables); + KUNIT_EXPECT_FALSE(test, hw->firmware_only); + KUNIT_EXPECT_FALSE(test, sio_hw_t8140.firmware_only); } static struct kunit_case sio_probe_test_cases[] = { diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index e1a8ec545b8c6..330bb7fa6b753 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include "dmaengine.h" @@ -160,6 +162,15 @@ struct sio_hw_data { u32 cfg_word4; /* configuration word 4 */ bool cfg_six_words; /* the configuration ends in a zero sixth word */ bool clear_period_desc; + /* + * Only run the firmware, so that it can be stopped for a system sleep. Nothing + * uses its DMA here, and its service parameters haven't been traced. + */ + bool firmware_only; + /* Without the boot tables for DMA, still run the firmware so a sleep can stop it. */ + bool firmware_without_tables; + /* macOS idles this firmware in the sleep's last step: leave it running until then. */ + bool stop_at_commit; /* DMA arrays owned by the driver, if not protocol 9's */ const struct sio_dma_arrays *arrays; int (*check_boot_tables)(struct device *dev); @@ -173,6 +184,11 @@ struct sio_data { struct dma_device dma; struct device *dev; struct apple_rtkit *rtk; + /* The firmware is stopped for a system sleep. */ + bool asleep; + bool firmware_only; + bool commit_registered; + bool commit_used; u32 protocol; void *shmem; dma_addr_t shmem_iova; @@ -538,7 +554,9 @@ static int sio_t6041_check_boot_tables(struct device *dev) u32 size[ARRAY_SIZE(sio_t6041_params)] = {}; if (sio_t6041_check_node(dev->of_node, addr, size)) { - dev_err(dev, "boot tables are not T6041 SIO tables; leaving the SIO stopped\n"); + /* T8140 still runs its firmware, and says so once it does. */ + if (!of_device_is_compatible(dev->of_node, "apple,t8140-sio")) + dev_err(dev, "boot tables are not T6041 SIO tables; leaving the SIO stopped\n"); return -ENODEV; } return 0; @@ -643,6 +661,8 @@ static const struct sio_hw_data sio_hw_t6041 = { * (ADT dp-audioN dma-channels): [2, 0, 0x800, 0x800, 0x800, 0]. */ static const struct sio_hw_data sio_hw_t8140 = { + .firmware_without_tables = true, + .stop_at_commit = true, .protocol = 10, .fifo_threshold = 0x800, .cfg_word4 = 0x800, @@ -1916,6 +1936,19 @@ static int sio_send_dt_params(struct sio_data *sio) return dev_err_probe(sio->dev, err, "failed to read '%s'\n", propname); } +/* Interrupts off, one CPU: the idle pair and the closed mailbox. The caller clears the run bit. */ +static int sio_commit_stop(void *data) +{ + struct sio_data *sio = data; + unsigned int us; + int ret; + + ret = apple_rtkit_sleep_idle_polled(sio->rtk, sio->base + REG_CPU_CONTROL, &us); + apple_rtkit_mailbox_recv_off(sio->rtk); + + return ret ? 1 : 0; +} + static int sio_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -1923,6 +1956,7 @@ static int sio_probe(struct platform_device *pdev) const struct sio_hw_data *hw; struct sio_data *sio; struct dma_device *dma; + bool firmware_only; int nchannels; int err, i; @@ -1931,10 +1965,13 @@ static int sio_probe(struct platform_device *pdev) return -ENODEV; hw = match->data; /* Refuse incompatible firmware tables before DMA or hardware setup. */ + firmware_only = hw->firmware_only; if (hw->check_boot_tables) { err = hw->check_boot_tables(&pdev->dev); - if (err) + if (err && !hw->firmware_without_tables) return err; + if (err) + firmware_only = true; } err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); @@ -1954,6 +1991,7 @@ static int sio_probe(struct platform_device *pdev) sio->dev = &pdev->dev; sio->nchannels = nchannels; sio->hw = hw; + sio->firmware_only = firmware_only; sio->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sio->base)) @@ -2014,6 +2052,11 @@ static int sio_probe(struct platform_device *pdev) if (err) return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + if (sio->firmware_only) { + dev_info(&pdev->dev, "firmware running, no DMA service\n"); + goto rpm_put; + } + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); if (err) return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); @@ -2048,6 +2091,9 @@ static int sio_probe(struct platform_device *pdev) } rpm_put: + if (!err && hw->stop_at_commit) + sio->commit_registered = !apple_sleep_coproc_register(&pdev->dev, sio->base, false, + sio_commit_stop, sio); pm_runtime_put(&pdev->dev); return err; @@ -2057,6 +2103,11 @@ static void sio_remove(struct platform_device *pdev) { struct sio_data *sio = platform_get_drvdata(pdev); + if (sio->commit_registered) + apple_sleep_coproc_unregister(&pdev->dev); + if (sio->firmware_only) + return; + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&sio->dma); } @@ -2074,7 +2125,118 @@ static __maybe_unused int sio_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); +/* A channel with a transfer can't be carried through a firmware restart. */ +static bool sio_chan_busy(struct sio_chan *siochan) +{ + unsigned long flags; + bool busy; + + spin_lock_irqsave(&siochan->vc.lock, flags); + busy = siochan->current_tx || siochan->idle || siochan->ring; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return busy; +} + +static __maybe_unused int sio_suspend_late(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int i, ret; + + if (!pm_suspend_via_firmware() || sio->asleep) + return 0; + + for (i = 0; i < sio->nchannels; i++) { + if (sio_chan_busy(&sio->channels[i])) { + dev_err(dev, "channel %d is busy, refusing the sleep\n", i); + return -EBUSY; + } + } + + if (sio->commit_registered && apple_sleep_commit_stops_coprocs()) { + apple_sleep_coproc_set_ready(dev, true); + sio->commit_used = true; + sio->asleep = true; + return 0; + } + + /* Idled any earlier, a refusal after it would leave it stopped. */ + if (sio->hw->stop_at_commit && apple_sleep_commit_stops_coprocs()) { + dev_err(dev, "no stop at the sleep's commit: refusing the sleep\n"); + return -EBUSY; + } + + ret = apple_rtkit_sleep_stop(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) { + dev_err(dev, "firmware did not stop for sleep: %d\n", ret); + if (apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) + dev_err(dev, "firmware did not come back\n"); + return ret; + } + sio->asleep = true; + + return 0; +} + +static __maybe_unused int sio_resume_early(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int ret; + + if (!sio->asleep) + return 0; + /* Left for the commit: if the sleep was refused before it, the firmware never idled. */ + if (sio->commit_used) { + bool stopped = apple_sleep_coproc_stopped(dev); + + sio->commit_used = false; + apple_sleep_coproc_set_ready(dev, false); + if (!pm_resume_via_firmware() && !stopped) { + sio->asleep = false; + return 0; + } + } + /* + * A refused sleep: an idled firmware doesn't wake with its power kept, and SIO_CPU shares + * SIO's switch, so it can't be power-cycled alone. The sleep engine's own PMGR reset of it + * restarts it (S); if that fails it stays stopped until the next real sleep. + */ + if (!pm_resume_via_firmware()) { + if (!apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) { + sio->asleep = false; + return 0; + } + ret = apple_sleep_engine_reset(dev); + if (!ret) + ret = apple_rtkit_power_cycled_return(sio->rtk, + sio->base + REG_CPU_CONTROL); + if (ret) { + dev_warn(dev, "sleep refused: firmware not back after a reset (%d), stopped until a real sleep\n", + ret); + return 0; + } + sio->asleep = false; + dev_info(dev, "sleep refused: firmware back after a reset\n"); + return 0; + } + /* Endpoints and parameters stay with the firmware: nothing is sent again. */ + ret = apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) { + /* Still down: the next sleep restarts it, or resets it if refused. */ + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + return 0; + } + sio->asleep = false; + dev_dbg(dev, "firmware back from sleep\n"); + + return 0; +} + +static const struct dev_pm_ops sio_pm_ops = { + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + LATE_SYSTEM_SLEEP_PM_OPS(sio_suspend_late, sio_resume_early) + RUNTIME_PM_OPS(sio_suspend, sio_resume, NULL) +}; static struct platform_driver apple_sio_driver = { .driver = { diff --git a/drivers/edac/apple_soc_edac.c b/drivers/edac/apple_soc_edac.c index a72e7f4dc351a..a4f717029cc5c 100644 --- a/drivers/edac/apple_soc_edac.c +++ b/drivers/edac/apple_soc_edac.c @@ -5,6 +5,7 @@ * Copyright The Aurora Silicon Contributors */ +#include #include #include #include @@ -36,9 +37,23 @@ struct apple_soc_edac_range { const char *name; }; +/* + * A fabric NI whose error status sits in its aperture: a block of status words, + * one per error type, whose set bits name the port. They're acknowledged by + * writing the set bits back. + */ +struct apple_soc_edac_ni { + u8 index; /* interrupt index */ + u8 reg; /* reg index of the NI's aperture */ + u16 status; /* offset of the first status word */ + u32 words; /* bit n: a status word at status + 4 * n */ +}; + struct apple_soc_edac_hw { const struct apple_soc_edac_range *ranges; size_t nranges; + const struct apple_soc_edac_ni *ni; + size_t nni; unsigned int num_irqs; unsigned int num_status_regs; bool event_log; @@ -51,6 +66,8 @@ struct apple_soc_edac_irq { const char *source; unsigned int index; int irq; + const struct apple_soc_edac_ni *ni; + void __iomem *ni_base; unsigned int retries; unsigned int backoff_ms; unsigned long armed_jiffies; @@ -126,6 +143,16 @@ static const struct apple_soc_edac_range t8140_error_ranges[] = { { 25, 1, "pio" }, }; +/* From the ADT's niN-metadata and macOS's NI decoder tables */ +static const struct apple_soc_edac_ni t8140_ni[] = { + { 18, 9, 0x1900, 0x1ffd7ff }, /* SoC NI 0 */ + { 19, 10, 0x1b00, 0x1ffd7ff }, /* SoC NI 1 */ + { 20, 11, 0x1500, 0x1ffd7ff }, /* SoC NI 2 */ + { 21, 12, 0x2500, 0x1ffd7ff }, /* MCC NI */ + { 22, 13, 0x2100, 0x7fefff }, /* AFC MCC NI */ + { 23, 14, 0x1d00, 0x3ff7ff }, /* GFX NI */ +}; + static const struct apple_soc_edac_hw t6000_error_hw = { .ranges = t6000_error_ranges, .nranges = ARRAY_SIZE(t6000_error_ranges), @@ -148,6 +175,8 @@ static const struct apple_soc_edac_hw t8112_error_hw = { static const struct apple_soc_edac_hw t8140_error_hw = { .ranges = t8140_error_ranges, .nranges = ARRAY_SIZE(t8140_error_ranges), + .ni = t8140_ni, + .nni = ARRAY_SIZE(t8140_ni), .num_irqs = 26, }; @@ -214,6 +243,38 @@ static bool apple_soc_edac_t6041_status(struct apple_soc_edac_irq *irq_data, return false; } +/* + * Log and acknowledge an NI's set status words. Returns whether any was set; + * @cleared says whether all of them read back clear. + */ +static bool apple_soc_edac_ni_status(struct apple_soc_edac_irq *irq_data, + bool *cleared) +{ + const struct apple_soc_edac_ni *ni = irq_data->ni; + unsigned long words = ni->words; + unsigned int bit, offset; + bool found = false; + u32 status, left; + + *cleared = true; + for_each_set_bit(bit, &words, 32) { + offset = ni->status + 4 * bit; + status = readl(irq_data->ni_base + offset); + if (!status) + continue; + writel(status, irq_data->ni_base + offset); + left = readl(irq_data->ni_base + offset); + dev_err_ratelimited(irq_data->edac->dev, + "%s: status %#x at %#x, %#x after the acknowledge\n", + irq_data->source, status, offset, left); + found = true; + if (left) + *cleared = false; + } + + return found; +} + static void apple_soc_edac_rearm(struct work_struct *work) { struct apple_soc_edac_irq *irq_data = @@ -244,12 +305,27 @@ static irqreturn_t apple_soc_edac_irq_thread(int irq, void *data) return IRQ_HANDLED; } + snprintf(msg, sizeof(msg), "%s source %u", irq_data->source, + irq_data->index); + if (irq_data->ni) { + bool cleared; + + if (apple_soc_edac_ni_status(irq_data, &cleared)) { + edac_device_handle_ue(edac->dci, 0, 0, msg); + dev_err_ratelimited(edac->dev, "hardware error: %s (IRQ %d)\n", + msg, irq); + if (cleared) + return IRQ_HANDLED; + } else { + dev_err_ratelimited(edac->dev, "%s asserted with no status set\n", + msg); + } + } + /* No status was pending, or this source has no clear contract. */ disable_irq_nosync(irq); - snprintf(msg, sizeof(msg), "%s source %u", irq_data->source, - irq_data->index); - if (!edac->hw->num_status_regs || irq_data->index < 45) { + if (!irq_data->ni && (!edac->hw->num_status_regs || irq_data->index < 45)) { edac_device_handle_ue(edac->dci, 0, 0, msg); dev_err_ratelimited(edac->dev, "hardware error: %s (IRQ %d)\n", msg, irq); @@ -280,6 +356,24 @@ static irqreturn_t apple_soc_edac_irq_thread(int irq, void *data) return IRQ_HANDLED; } +static int apple_soc_edac_map_ni(struct platform_device *pdev, + struct apple_soc_edac_irq *irq_data) +{ + const struct apple_soc_edac_hw *hw = irq_data->edac->hw; + size_t i; + + for (i = 0; i < hw->nni; i++) { + if (hw->ni[i].index != irq_data->index) + continue; + irq_data->ni_base = devm_platform_ioremap_resource(pdev, hw->ni[i].reg); + if (IS_ERR(irq_data->ni_base)) + return PTR_ERR(irq_data->ni_base); + irq_data->ni = &hw->ni[i]; + } + + return 0; +} + static int apple_soc_edac_probe(struct platform_device *pdev) { const struct apple_soc_edac_hw *hw; @@ -356,6 +450,9 @@ static int apple_soc_edac_probe(struct platform_device *pdev) irq_data->irq = irq; irq_data->backoff_ms = APPLE_SOC_EDAC_BACKOFF_MS; irq_data->armed_jiffies = jiffies; + ret = apple_soc_edac_map_ni(pdev, irq_data); + if (ret) + goto err_del_device; INIT_DELAYED_WORK(&irq_data->rearm, apple_soc_edac_rearm); ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 459d5da71fbc5..a24b557119b35 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -47,6 +47,7 @@ #include #include #include +#include #include "apple_boot_display.h" #include "apple_resume.h" @@ -67,8 +68,15 @@ struct apple_drm_private { * the screen, the DRM device stays unregistered and its DCPs running. */ bool aborted; + /* This suspend found every firmware already stopped. */ + bool s2r_skipped; + /* A dark wake left the outputs as the suspend did, for the next resume. */ + bool dark_held; + struct notifier_block pm_nb; }; +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data); + /* Component readiness is later than platform probe/bind. */ static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; @@ -1783,6 +1791,9 @@ static int apple_drm_init(struct device *dev) goto err_unbind; } + apple->pm_nb.notifier_call = apple_drm_pm_notify; + register_pm_notifier(&apple->pm_nb); + return 0; err_unlock: @@ -1806,6 +1817,8 @@ static void apple_drm_uninit(struct device *dev) if (apple->aborted) return; + unregister_pm_notifier(&apple->pm_nb); + /* Before anything of the DRM device is torn down. */ t8132_ext_drm_fini(&apple->drm); apple_drm_quiesce_connectors(&apple->drm); @@ -1924,12 +1937,38 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); #ifdef CONFIG_PM_SLEEP +/* + * Left stopped by a suspend that never reached the commit. Nothing can commit to + * them, and a refused sleep has to be able to try again. + */ +static bool apple_drm_firmware_stopped(struct apple_drm_private *apple) +{ + struct drm_crtc *crtc; + bool stopped = false; + + drm_for_each_crtc(crtc, &apple->drm) { + if (t8132_ext_crtc(crtc) || + !dcp_s2r_firmware_stopped(to_apple_crtc(crtc)->dcp)) + return false; + stopped = true; + } + + return stopped; +} + static int apple_platform_suspend(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple && !apple->aborted) - return drm_mode_config_helper_suspend(&apple->drm); + /* Still suspended from before a dark wake. */ + if (apple && apple->dark_held) + return 0; + + if (apple && !apple->aborted) { + apple->s2r_skipped = apple_drm_firmware_stopped(apple); + if (!apple->s2r_skipped) + return drm_mode_config_helper_suspend(&apple->drm); + } return 0; } @@ -1951,29 +1990,71 @@ static bool apple_drm_hold_resume_events(struct apple_drm_private *apple, u32 cr return held; } +static void apple_drm_resume_outputs(struct apple_drm_private *apple) +{ + u32 dropped = 0; + + apple->dark_held = false; + if (apple->drm.mode_config.suspend_state) + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped && apple_drm_hold_resume_events(apple, dropped)) + dropped = 0; + drm_mode_config_helper_resume(&apple->drm); + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - u32 dropped = 0; - if (apple && !apple->aborted) { - if (apple->drm.mode_config.suspend_state) - dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); - /* Have userspace set the mode of a CRTC resumed disabled again. */ - if (dropped && apple_drm_hold_resume_events(apple, dropped)) - dropped = 0; - drm_mode_config_helper_resume(&apple->drm); - if (dropped) - drm_kms_helper_hotplug_event(&apple->drm); + if (!apple || apple->aborted || apple->s2r_skipped) + return 0; + + /* A dark wake goes back to sleep with user space frozen: nothing is lit for it. */ + if (pm_dark_wake()) { + apple->dark_held = true; + dev_info(dev, "dark wake: outputs stay off\n"); + return 0; } + apple_drm_resume_outputs(apple); return 0; } +/* The suspend after a dark wake failed before it suspended this device. */ +static void apple_platform_complete(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple && apple->dark_held && !pm_dark_wake()) + apple_drm_resume_outputs(apple); +} + +/* It failed before ->prepare() reached this device, so ->complete() didn't run either. */ +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_drm_private *apple = container_of(nb, struct apple_drm_private, pm_nb); + + if (action == PM_POST_SUSPEND && apple->dark_held) { + dev_warn(apple->drm.dev, "dark wake over without a resume: outputs back now\n"); + apple_drm_resume_outputs(apple); + } + + return NOTIFY_DONE; +} + static const struct dev_pm_ops apple_platform_pm_ops = { .suspend = apple_platform_suspend, .resume = apple_platform_resume, + .complete = apple_platform_complete, }; +#else +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + return NOTIFY_DONE; +} #endif static struct platform_driver apple_platform_driver = { diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index df32ea6268d24..06a2b8f71b02d 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1883,7 +1884,23 @@ static __maybe_unused int dcpaud_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(dcpaud_pm_ops, dcpaud_suspend, dcpaud_resume, NULL); +/* The SIO channel and the sink's link both end in a sleep that reaches the hardware. */ +static __maybe_unused int dcpaud_system_suspend(struct device *dev) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + + if (pm_suspend_via_firmware() && dcpaud->substream && dcpaud->substream->ref_count) { + dev_err(dev, "a stream is open, refusing the sleep\n"); + return -EBUSY; + } + + return pm_runtime_force_suspend(dev); +} + +static const struct dev_pm_ops dcpaud_pm_ops = { + SYSTEM_SLEEP_PM_OPS(dcpaud_system_suspend, pm_runtime_force_resume) + RUNTIME_PM_OPS(dcpaud_suspend, dcpaud_resume, NULL) +}; static const struct of_device_id dcpaud_of_match[] = { { .compatible = "apple,dpaudio" }, diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index de538ed801846..1e4ed493a1fab 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -654,6 +654,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, scoped_guard(mutex, &dcp->tb_lock) { route->deactivate_xbar_done = false; route->deactivate_error = 0; + route->sleep_parked = false; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -703,7 +704,10 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return route->deactivate_error; if (!route->deactivate_xbar_done) { dcp_typec_route_phy_off(route); - if (!route->tunnel || route->xbar_up) { + /* Parked for a sleep, its output is already down and not held. */ + if (route->sleep_parked) { + ret = 0; + } else if (!route->tunnel || route->xbar_up) { ret = mux_control_deselect(route->active_xbar ?: route->xbar); } else { ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); @@ -718,6 +722,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) } /* Deselect always unlocks: never consume that selection twice. */ route->deactivate_xbar_done = true; + route->sleep_parked = false; route->xbar_up = false; if (ret) { /* Idle is unproven; retain the owner and the real error. */ @@ -4600,6 +4605,53 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) return dcp_fabric_run_probe(!!routes, dcp_register_typec_routes_present, &ctx); } +/* + * A sleep that reaches its commit powers the crossbar off. Take the output of a direct + * route down while it still has power, as a released display's is, and select it again + * after the wake. The route stays, and with it the display on the port. + */ +void dcp_typec_route_park(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route || route->tunnel || !route->selected) + return; + guard(mutex)(&dcp->tb_lock); + if (route->deactivate_xbar_done || route->sleep_parked) + return; + /* Deselect always unlocks. */ + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + route->sleep_parked = true; + dev_dbg(dcp->dev, "display route parked for the sleep: crossbar output down: %d\n", ret); +} + +void dcp_typec_route_unpark(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route) + return; + guard(mutex)(&dcp->tb_lock); + if (!route->sleep_parked) + return; + route->sleep_parked = false; + ret = mux_control_try_select(route->active_xbar ?: route->xbar, route->mux_index); + if (ret) { + /* Nothing is held, so the route's release has nothing to give back. */ + route->deactivate_xbar_done = true; + dev_err(dcp->dev, "display route not back after the sleep: crossbar output: %d\n", + ret); + return; + } + dev_dbg(dcp->dev, "display route back after the sleep: crossbar output up\n"); +} + /* * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY * (see dcp_typec_route_deactivate()). Point it at the HDMI output again diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 0805c54d39051..edca8065bd21a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -55,6 +55,8 @@ struct apple_dcp_typec_route { /* Lower mux release is consumed even when idle programming fails. */ bool deactivate_xbar_done; int deactivate_error; + /* Its crossbar output is down for a sleep, and not held. */ + bool sleep_parked; }; int dcp_typec_phy_on(struct apple_dcp *dcp); @@ -117,6 +119,8 @@ bool dcp_is_t8132_ext(const struct apple_dcp *dcp); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); +void dcp_typec_route_park(struct apple_dcp *dcp); +void dcp_typec_route_unpark(struct apple_dcp *dcp); void dcp_typec_routes_disable(struct apple_dcp *dcp); void dcp_typec_routes_disable_dev(struct device *dev); void dcp_typec_retrain_work(struct work_struct *work); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ac355dcd2c5cc..f2ab6f88567f5 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -264,6 +265,13 @@ struct apple_dcp_hw_data { * (desktop 26.x), see dcp_sink_metadata(). */ bool sink_metadata_edid; + /* + * With H17P firmware: a firmware-driven system sleep stops the internal processor's + * firmware and powers it down, as DPMS off does, then restarts it warm, see + * dcp_s2r_suspend(). @s2r_ext: the same for an external one, see dcp_s2r_ext_suspend(). + */ + bool s2r; + bool s2r_ext; }; enum dcp_test_point { @@ -368,6 +376,8 @@ const struct drm_edid *dcp_panel_edid_snapshot(struct apple_dcp *dcp, unsigned int count, u64 *generation); bool dcp_panel_metadata_current(struct apple_dcp *dcp, u64 generation); +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on); + struct apple_dcp { /* Generation-specific wire/transaction state, sharing the common KMS owner. */ struct iomfb_m4 *v26_6; @@ -665,6 +675,38 @@ struct apple_dcp { /* Messages from the coprocessor are held in the mailbox for sleep. */ bool rx_held; + /* A firmware-driven suspend stops the firmware, see dcp_s2r_suspend(). */ + bool s2r_active; + bool s2r_stopped; + /* DPMS off stopped the firmware and lowered the domains; DPMS on brings them back. */ + bool dpms_powered_off; + /* Refused its idle with the display off: the firmware still runs, the display is dark. */ + bool s2r_refused_off; + /* The firmware's secure mailbox, where it has one. */ + struct apple_rtkit *secure_rtk; + bool secure_answered; + bool secure_polling; + /* Polled every jiffy until then, see dcp_secure_poll(). */ + unsigned long secure_fast_until; + int secure_map_ret; + s64 secure_map_us; + struct delayed_work secure_poll_wq; + /* Stopped and not restarted, which outlives a suspend that gets refused. */ + bool s2r_fw_down; + /* DCPEXT kept down through a dark wake, see dcp_s2r_dark_finish(). */ + bool s2r_dark_held; + struct notifier_block s2r_pm_nb; + /* This driver raised the display domains, and nothing has lowered them since. */ + bool s2r_domains_up; + /* Counted in pmc-dcp, the number of powered DCPs */ + bool pmc_up; + bool s2r_power_on; + bool s2r_power_oob; + u8 s2r_step; + struct completion s2r_dart_done; + struct device *s2r_disp_dart; + struct device *s2r_dcp_dart; + /* eDP display without DP-HDMI conversion */ bool main_display; @@ -992,6 +1034,7 @@ int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, const void *data, size_t size, void *cookie); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); +void iomfb_reset_transport(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout); void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bc452a840da83..05c22685393ef 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -19,13 +19,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -149,6 +152,135 @@ static void dcp_release_context(void *data) kfree(dcp); } +#define DCP_SECURE_EP 0x30 +#define DCP_SECURE_TIMEOUT_MS 200 + +/* + * The firmware's secure mailbox. No other software answers it under Linux, so the + * driver does, as macOS's secure side does: 3 is answered with 2, and 4 needs nothing. + * What these messages mean isn't known. + */ +static void dcp_secure_recv(void *cookie, u8 ep, u64 msg) +{ + struct apple_dcp *dcp = cookie; + int ret; + + dev_dbg(dcp->dev, "secure port: ep %#x msg %#llx\n", ep, msg); + if (ep != DCP_SECURE_EP || (msg >> 60) != 3) + return; + + ret = apple_rtkit_send_message(dcp->secure_rtk, ep, 2ULL << 60, NULL, false); + if (ret) + dev_warn(dcp->dev, "secure port: no reply sent: %d\n", ret); +} + +static const struct apple_rtkit_ops dcp_secure_rtkit_ops = { + .recv_message = dcp_secure_recv, +}; + +/* + * The port's interrupts aren't in the DT. Its handshakes, idle and restart poll it themselves; + * this answers the rest. Every jiffy for a second after its start or a message, as more tend to + * follow, otherwise every 100 ms: woken every jiffy, an idle CPU never reaches deep idle. + */ +#define DCP_SECURE_POLL_FAST_MS 1000 +#define DCP_SECURE_POLL_SLOW_MS 100 + +static void dcp_secure_poll(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + secure_poll_wq); + bool fast = time_before(jiffies, dcp->secure_fast_until); + + if (!READ_ONCE(dcp->secure_polling)) + return; + + if (apple_rtkit_poll(dcp->secure_rtk) > 0) { + if (!fast) + dev_info_once(dcp->dev, "secure port: a message while idle\n"); + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + fast = true; + } + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, + fast ? 1 : msecs_to_jiffies(DCP_SECURE_POLL_SLOW_MS)); +} + +static void dcp_secure_poll_start(struct apple_dcp *dcp) +{ + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + WRITE_ONCE(dcp->secure_polling, true); + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, 0); +} + +static void dcp_secure_poll_stop(struct apple_dcp *dcp) +{ + WRITE_ONCE(dcp->secure_polling, false); + cancel_delayed_work_sync(&dcp->secure_poll_wq); +} + +/* After the firmware's HELLO on its main mailbox. */ +static void dcp_secure_port_start(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->secure_rtk) + return; + + ret = apple_rtkit_port_handshake(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) { + dev_warn(dcp->dev, "secure port handshake not finished: %d\n", ret); + return; + } + dcp->secure_answered = true; + dcp_secure_poll_start(dcp); + dev_info(dcp->dev, "secure port up\n"); +} + +/* Between the main mailbox's AP idle and its IOP idle. */ +static void dcp_secure_port_idle(void *data) +{ + struct apple_dcp *dcp = data; + int ret; + + if (!dcp->secure_rtk || !dcp->secure_answered) + return; + + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_idle(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP idle not acknowledged: %d\n", ret); +} + +static void dcp_release_secure_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + dcp_secure_poll_stop(dcp); + apple_rtkit_free(dcp->secure_rtk); + dcp->secure_rtk = NULL; +} + +/* Nothing is read from the port until its handshake is wanted. */ +static int dcp_secure_port_init(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + + if (of_property_match_string(dev->of_node, "mbox-names", "secure") < 0) + return 0; + + dcp->secure_rtk = apple_rtkit_init(dev, dcp, "secure", 0, &dcp_secure_rtkit_ops); + if (IS_ERR(dcp->secure_rtk)) { + int ret = PTR_ERR(dcp->secure_rtk); + + dcp->secure_rtk = NULL; + return dev_err_probe(dev, ret, "Failed to initialize the secure port\n"); + } + + INIT_DELAYED_WORK(&dcp->secure_poll_wq, dcp_secure_poll); + + return devm_add_action_or_reset(dev, dcp_release_secure_rtkit, dcp); +} + static void dcp_release_rtkit(void *data) { struct apple_dcp *dcp = data; @@ -1627,11 +1759,16 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) return iomfb_v14_7_atomic_check(dcp, crtc, commit); - if (dcp->crashed) - return -EINVAL; - crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + /* + * A crashed DCP can't be lit, but a commit that leaves its CRTC off has to pass: + * the system suspend's disable-all includes every CRTC, and a stopped external DCP + * would refuse every sleep after the first. + */ + if (dcp->crashed) + return crtc_state && crtc_state->enable ? -EINVAL : 0; + /* External H17P does not program adaptive sync. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && @@ -3947,6 +4084,7 @@ int dcp_start(struct platform_device *pdev) /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ init_completion(&dcp->start_done); + init_completion(&dcp->s2r_dart_done); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) return iomfb_v26_6_start(dcp); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) @@ -4259,6 +4397,9 @@ static bool dcp_h17g_panel(struct apple_dcp *dcp) dcp_has_panel(dcp); } +static void dcp_dpms_power_off(struct apple_dcp *dcp); +static void dcp_dpms_power_on(struct apple_dcp *dcp); + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { /* The 14.7 panel has its own known-level blank and restore hooks. */ @@ -4328,6 +4469,7 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { + dcp_dpms_power_on(dcp); if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { @@ -4480,11 +4622,15 @@ void dcp_poweroff(struct platform_device *pdev) cancel_delayed_work(&dcp->swap_watchdog_wq); - /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + /* + * Internal H17P DPMS is a brightness present, never a pipe stop. With hw.s2r the + * display then powers off as well, after the backlight. + */ if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + dcp_dpms_power_off(dcp); return; } @@ -5703,6 +5849,12 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; + /* After a warm restart its log counters need echoing, or it refuses its next idle. */ + if (dcp->hw.s2r || dcp->hw.s2r_ext) + apple_rtkit_set_oslog_echo(dcp->rtk, true); + ret = dcp_secure_port_init(dcp); if (ret) goto err_piodma; @@ -5721,6 +5873,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } + dcp_secure_port_start(dcp); enable_work(&dcp->dimensions_wq); schedule_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); @@ -6100,6 +6253,8 @@ static void dcp_t8132_ext_routes(struct apple_dcp *dcp) dcp->nr_typec_routes); } +static int dcp_s2r_pm_register(struct apple_dcp *dcp); + static int dcp_platform_probe_pipe(struct platform_device *pdev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); @@ -6289,6 +6444,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) } platform_set_drvdata(pdev, dcp); + ret = dcp_s2r_pm_register(dcp); + if (ret) + return ret; if (iomfb_is_m4(dcp)) { dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; @@ -6909,10 +7067,13 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) */ static void dcp_hold_for_sleep(struct apple_dcp *dcp) { - /* A failed component bind leaves no coprocessor session to hold. */ + /* + * A failed component bind leaves no coprocessor session to hold. Firmware that DPMS + * stopped sends nothing, and its presentation stays paused until DPMS restarts it. + */ if (IS_ERR_OR_NULL(dcp->rtk) || !dcp_uses_soft_dpms(dcp) || !dcp_presents_nits(dcp) || - READ_ONCE(dcp->crashed)) + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->s2r_fw_down)) return; iomfb_pause_presentation(dcp, true); @@ -7012,10 +7173,13 @@ static int dcp_platform_prepare(struct device *dev) return 0; } +static void dcp_s2r_dark_finish(struct apple_dcp *dcp); + static void dcp_platform_complete(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_s2r_dark_finish(dcp); if (!dcp->external) return; mutex_lock(&dcp->hpd_mutex); @@ -7023,105 +7187,1247 @@ static void dcp_platform_complete(struct device *dev) mutex_unlock(&dcp->hpd_mutex); } -static int dcp_platform_suspend(struct device *dev) +/* + * Suspend-to-RAM with hw.s2r: the sleep engine waits for the DCP's domain to be off, so + * the firmware has to stop first. Measured from macOS on T8140, in order: + * + * 1. A414(0), A478 x2, A476(1), A478, A412, A473(0). + * 2. D572(0) from the firmware: the display DART's streams go off. + * 3. A500, unnamed, then D107s and D575(0) from the firmware. + * 4. RTKit idle, then the ASC's run bit off. + * 5. The DCP DART's streams off. That's the coprocessor's own shutdown, apart + * from step 2. The sleep driver then lowers the domains. + * + * macOS also restarts the firmware 893 ms before the commit, calls A478, A473(0), + * A115, A500, and stops it again. Why isn't known, and that's not done here. + * + * The firmware takes IOP idle only with its secure mailbox answered, see + * dcp_s2r_answer_secure_port(). + */ +#define DCP_S2R_DISP_STREAMS (BIT(0) | BIT(3) | BIT(4) | BIT(16)) +#define DCP_S2R_DCP_STREAMS (BIT(23) | BIT(24)) +#define DCP_S2R_D572_TIMEOUT_MS 1000 /* macOS: about 170 ms after A473(0) */ +#define DCP_PS_MIN GENMASK(19, 16) + +static struct device *dcp_s2r_find_dart(struct device_node *np) { - struct apple_dcp *dcp = dev_get_drvdata(dev); + struct device_node *dart; + struct platform_device *pdev; - /* Its power sequences may crash the DCP and blank the panel. */ - if (dcp_takeover_was_aborted(dcp)) - return -EBUSY; + dart = of_parse_phandle(np, "iommus", 0); + if (!dart) + return NULL; + pdev = of_find_device_by_node(dart); + of_node_put(dart); - /* - * T8132: setPowerState powers the display down; the coprocessor keeps - * running and its power domains stay on, as below, unless the internal - * display is blanked with appledrm.power_gate_ms set: then the - * coprocessor and its domains are powered down too (iomfb_m4_suspend()). - */ - if (iomfb_is_m4(dcp)) { - int ret = iomfb_m4_suspend(dcp); + return pdev ? &pdev->dev : NULL; +} - if (!ret) - device_set_wakeup_path(dev); - return ret; +static void dcp_s2r_put_darts(struct apple_dcp *dcp) +{ + if (dcp->s2r_dcp_dart) + put_device(dcp->s2r_dcp_dart); + if (dcp->s2r_disp_dart) + put_device(dcp->s2r_disp_dart); + dcp->s2r_dcp_dart = NULL; + dcp->s2r_disp_dart = NULL; +} + +static void dcp_s2r_find_darts(struct apple_dcp *dcp) +{ + struct device_node *piodma; + + /* A suspend that failed after an earlier lookup still holds those. */ + dcp_s2r_put_darts(dcp); + dcp->s2r_dcp_dart = dcp_s2r_find_dart(dcp->dev->of_node); + piodma = of_get_child_by_name(dcp->dev->of_node, "piodma"); + if (piodma) { + dcp->s2r_disp_dart = dcp_s2r_find_dart(piodma); + of_node_put(piodma); } - if (READ_ONCE(dcp->ext_dead)) - return 0; - iomfb_v14_7_backlight_suspend(dcp); +} - /* Serialize PM against explicit bring-up, including queued startup work. - * Before a startup attempt the device remains suspendable. After any - * attempt we cannot prove DMA/firmware quiescence, even on failure. - */ - if (dcp->external) { - const char *why; +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on) +{ + if (!READ_ONCE(dcp->s2r_active)) + return; - mutex_lock(&dcp->hpd_mutex); - why = dcp_external_sleep_vetoed(dcp); - if (why) { - mutex_unlock(&dcp->hpd_mutex); - if (dcp->external_native) - dev_warn(dev, "suspend refused: %s\n", why); - else - dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); - return -EBUSY; - } - WRITE_ONCE(dcp->external_suspended, true); - mutex_unlock(&dcp->hpd_mutex); + if (dcp->s2r_disp_dart) + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, on, false); + complete(&dcp->s2r_dart_done); +} + +static bool dcp_s2r_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.s2r; +} + +static void dcp_s2r_cpu_stop(struct apple_dcp *dcp) +{ + /* macOS writes it twice. */ + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); +} + +/* Up from bind, unless its handshake failed then. A bound firmware needs it to take IOP idle. */ +static void dcp_s2r_answer_secure_port(struct apple_dcp *dcp) +{ + if (!dcp->secure_answered) + dcp_secure_port_start(dcp); +} + +/* After a refused idle: AP on to both mailboxes, as the GPU and the AOP send it. */ +static void dcp_s2r_idle_undo(struct apple_dcp *dcp) +{ + int ret; + + ret = apple_rtkit_ap_power(dcp->rtk, true); + if (ret) + dev_warn(dcp->dev, "AP on after the refused idle: %d\n", ret); + if (!dcp->secure_rtk || !dcp->secure_answered || READ_ONCE(dcp->secure_polling)) + return; + ret = apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP on after the refused idle: %d\n", ret); + dcp_secure_poll_start(dcp); +} + +/* + * macOS's order up to RTKit idle. On success the firmware is idle and its run bit + * is still set. + */ +static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) +{ + ktime_t start = ktime_get(); + /* Off since a refused idle: no second set_power_state(0), whose D572 may never come. */ + bool display_off = dcp->s2r_refused_off; + bool idle_sent = false; + s64 power_ms; + bool idle; + int ret; + + reinit_completion(&dcp->s2r_dart_done); + dcp_s2r_find_darts(dcp); + WRITE_ONCE(dcp->s2r_active, true); + + if (display_off) { + dev_info(dcp->dev, "display still off from a refused idle\n"); + goto display_is_off; } - if (dcp_is_external_h17p(dcp)) { - int ret; + /* In band, 3 of 3 on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, false, true, false); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(0): %d\n", ret); + goto err; + } + power_ms = ktime_ms_delta(ktime_get(), start); - dcp_disable_typec_work(dcp, false); - ret = dcp_external_disconnect(dcp, false); - if (ret) { - dcp_enable_typec_work(dcp); - return ret; - } - device_set_wakeup_path(dev); - return 0; + /* Without D572 the display isn't known to be off, and nothing below can undo this. */ + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) { + dev_err(dcp->dev, "no D572 %lld ms after set_power_state(0)\n", + ktime_ms_delta(ktime_get(), start)); + ret = -ETIMEDOUT; + goto err; } + dev_dbg(dcp->dev, "display off: set_power_state(0) %lld ms, D572 %lld ms\n", power_ms, + ktime_ms_delta(ktime_get(), start)); + display_off = true; - dcp_drain_for_sleep(dcp); - /* - * The Type-C route reports cable removal through - * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays - * connected, and resume powers the CRTC back up through dcp_poweron() - * as after DPMS off. - */ - dcp_disable_typec_work(dcp, false); - cancel_delayed_work_sync(&dcp->swap_watchdog_wq); - if (dcp->external_native) - disable_work_sync(&dcp->external_audio_work); +display_is_off: + /* D572 itself is still being answered, and a busy channel can't be sent on. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before A500\n"); - if (dcp->avep) - av_service_disconnect(dcp); + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + goto err; + } + dev_dbg(dcp->dev, "A500 replied after %lld ms\n", ktime_ms_delta(ktime_get(), start)); - if (dcp->hdmi_hpd_irq) { - disable_irq(dcp->hdmi_hpd_irq); - if (!dcp->active_typec_route) { - disconnected_hpd_event(dcp->connector); - dcp_dptx_disconnect(dcp, 0); - } + /* The firmware's callbacks after A500 have to be answered first. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before idle\n"); + + idle_sent = true; + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); + if (ret) { + /* Answer what it still has queued, its log counters among them, and ask again. */ + dev_warn(dcp->dev, "firmware refused idle after %lld ms: %d, control %#x status %#x: retrying once\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + apple_rtkit_poll(dcp->rtk); + msleep(30); + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); } + if (ret) { + dev_err(dcp->dev, "firmware did not go idle after %lld ms: %d, control %#x status %#x\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + goto err; + } + + /* Both set once the firmware's last message has been read; macOS waits for them. */ + if (readl_poll_timeout(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS, *status, + (*status & (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) == + (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED), 100, 5000)) + dev_warn(dcp->dev, "idle status not set: %#x\n", *status); + dev_dbg(dcp->dev, "firmware idle after %lld ms, status %#x\n", + ktime_ms_delta(ktime_get(), start), *status); + + WRITE_ONCE(dcp->s2r_active, false); + dcp->s2r_refused_off = false; + + return 0; + +err: + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); /* - * Set the device as a wakeup device, which forces its power - * domains to stay on. We need this as we do not support full - * shutdown properly yet. + * Refused with the display off: leave the firmware running, never stop it under its + * own feet. DPMS on lights the display again on the running firmware. */ - device_set_wakeup_path(dev); - /* Last: the steps above still talk to the coprocessor. */ - dcp_hold_for_sleep(dcp); + if (display_off) { + if (idle_sent) + dcp_s2r_idle_undo(dcp); + dcp->s2r_refused_off = true; + dev_warn(dcp->dev, "display left off with the firmware running\n"); + return ret; + } + /* The display may be off and the firmware half idle. Nothing here brings it back. */ + WRITE_ONCE(dcp->crashed, true); - return 0; + return ret; } -static int dcp_platform_resume(struct device *dev) +/* Its power state can only be lowered once this has finished. */ +static int dcp_s2r_stop_streams(struct apple_dcp *dcp) { - struct apple_dcp *dcp = dev_get_drvdata(dev); + if (!dcp->s2r_dcp_dart) + return -ENODEV; - if (dcp_takeover_was_aborted(dcp)) - return 0; + return apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, false, true); +} + +static int dcp_s2r_stop_firmware(struct apple_dcp *dcp) +{ + u32 status; + int ret; + + ret = dcp_s2r_quiesce(dcp, &status); + if (ret) + return ret; + + apple_rtkit_mailbox_off(dcp->rtk); + dcp_s2r_cpu_stop(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + return dcp_s2r_stop_streams(dcp); +} + +/* + * macOS stops this firmware with one call the size of A500, then about 4 ms later AP + * then IOP idle and the run bit off twice. Nothing has shown it needing a secure + * port. + * + * An attached display goes first, as macOS's DP port releases its display at system + * sleep: the pipe off, HPD low and the display released, as DPMS off does, then the + * crossbar output. The route and the connector stay, and after the wake the link + * connects again, as macOS replays the port's HPD. + */ +static bool dcp_s2r_ext_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.s2r_ext; +} + +/* + * Taken as stopped, but a start since raised its domains and failed after the run bit, + * as one a signal ended. Left running, it refuses every sleep; stopped, it comes back + * from the sleep's power cycle like any other. + */ +static void dcp_s2r_stop_failed_start(struct apple_dcp *dcp) +{ + if (!dcp->s2r_domains_up || + !(readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL) & + APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return; + dev_warn(dcp->dev, "a failed start left the firmware running: stopping it for the sleep\n"); + dcp_s2r_cpu_stop(dcp); +} + +/* + * pmc-dcp counts the powered DCPs, as macOS keeps it: 0, 1, 2, 1, 0 across a wake. A DCP + * counts from when this driver powers it up until it powers it off or a sleep stops it. + */ +static DEFINE_MUTEX(dcp_pmc_lock); +static unsigned int dcp_pmc_count; + +static void dcp_pmc_set(struct apple_dcp *dcp, void __iomem *pmc, bool up) +{ + guard(mutex)(&dcp_pmc_lock); + if (up != dcp->pmc_up) { + dcp->pmc_up = up; + if (up) + dcp_pmc_count++; + else if (dcp_pmc_count) + dcp_pmc_count--; + } + writel(dcp_pmc_count, pmc); +} + +/* The sleep controller lowers the domains and pmc-dcp itself. */ +static void dcp_pmc_forget(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_pmc_lock); + if (dcp->pmc_up && dcp_pmc_count) + dcp_pmc_count--; + dcp->pmc_up = false; +} + +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp); + +/* + * Crashed firmware takes no stop sequence, and refusing would hold every later sleep off until + * reboot. Its CPU stops cold, as for a failed start, and the wake starts it again, as macOS + * recovers a crashed DCP. A system sleep only: DPMS off still refuses it. + */ +static bool dcp_s2r_stops_crashed(struct apple_dcp *dcp) +{ + return READ_ONCE(dcp->crashed) && !IS_ERR_OR_NULL(dcp->rtk) && + pm_suspend_target_state != PM_SUSPEND_ON; +} + +static int dcp_s2r_stop_crashed(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + dev_warn(dcp->dev, "firmware crashed: stopped cold for the sleep, started again after it\n"); + apple_rtkit_mailbox_off(dcp->rtk); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) +{ + int ret; + + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + dcp_typec_route_park(dcp); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + return 0; + } + + /* No A500 and no disconnect: the link and its crossbar output just go down. */ + if (dcp_s2r_stops_crashed(dcp)) { + dcp_disable_typec_work(dcp, false); + dcp_typec_route_park(dcp); + return dcp_s2r_stop_crashed(dcp); + } + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) + dev_dbg(dcp->dev, "a display is attached: released for the sleep, connected again after it\n"); + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dcp_enable_typec_work(dcp); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* The display's link went above: its crossbar output follows, as on macOS. */ + dcp_typec_route_park(dcp); + + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + usleep_range(4000, 5000); + + ret = apple_rtkit_sleep_stop(dcp->rtk, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (ret) { + dev_err(dcp->dev, "firmware did not go idle: %d, control %#x status %#x\n", ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +/* A failure before the firmware has stopped leaves it running, and refuses cleanly. */ +static int dcp_s2r_suspend(struct apple_dcp *dcp) +{ + bool was_off; + int ret; + + /* Still stopped from a suspend that never reached the commit. */ + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + ret = dcp_s2r_stop_streams(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + return 0; + } + + if (dcp_s2r_stops_crashed(dcp)) + return dcp_s2r_stop_crashed(dcp); + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + dcp_s2r_answer_secure_port(dcp); + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* Presentation stays paused if this fails: see dcp_s2r_quiesce(). */ + was_off = dcp->s2r_refused_off; + ret = dcp_s2r_stop_firmware(dcp); + if (ret) { + if (READ_ONCE(dcp->s2r_fw_down)) + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + /* Dark since a refused idle, as after DPMS off: it stays so until DPMS on. */ + if (was_off) + dcp->dpms_powered_off = true; + + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static void dcp_s2r_ps_lower(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & BIT(28)) { + val &= ~(BIT(28) | GENMASK(9, 8)); + writel(val, reg); + } + writel((val & ~GENMASK(3, 0)) | GENMASK(9, 8), reg); +} + +/* macOS's power-off, measured 3 of 3. The firmware is stopped and its DARTs are off. */ +static int dcp_s2r_domains_off(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + dcp->s2r_domains_up = false; + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + dcp_pmc_set(dcp, pmc, false); + val = readl(ps + 0x10000); + if (val & BIT(28)) + writel(val & ~(BIT(28) | GENMASK(9, 8)), ps + 0x10000); + writel(3, pwrgate + 0xc8c); + writel(0, pwrgate + 0xcd0); + dcp_s2r_ps_lower(ps + 0x10000); + dcp_s2r_ps_lower(ps + 0x318); + dcp_s2r_ps_lower(ps + 0x280); + + ret = readl_poll_timeout(ps + 0x10000, val, !FIELD_GET(GENMASK(7, 4), val), 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power off: %#x\n", val); + dev_dbg(dcp->dev, "display domains off: CPU %#x FE %#x SYS %#x\n", readl(ps + 0x10000), + readl(ps + 0x318), readl(ps + 0x280)); +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * Exact mirror of the writes in macOS's power-off, measured 3 of 3. DISP_SYS and + * DISP_FE first, then DISP_CPU with AUTO_ENABLE last. + */ +static int dcp_s2r_domains_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + val = readl(ps + 0x280) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x280); + writel(val | 0xf | BIT(28), ps + 0x280); + val = readl(ps + 0x318) & ~(GENMASK(9, 8) | GENMASK(3, 0)); + writel(val | 0xf, ps + 0x318); + val = readl(ps + 0x10000) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x10000); + writel(0, pwrgate + 0xc8c); + writel(1, pwrgate + 0xcd0); + writel(val | 0xf | BIT(28), ps + 0x10000); + dcp_pmc_set(dcp, pmc, true); + + ret = readl_poll_timeout(ps + 0x10000, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power on: %#x\n", val); + else + dcp->s2r_domains_up = true; +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * DISPEXT0 up as macOS does it after a wake: SYS and FE on, then the CPU with PS_MIN 4 first, + * since the wake reset it, then on, and pmc-dcp up once it reads on. + */ +static int dcp_s2r_ext_domains_on(struct apple_dcp *dcp) +{ + static const u16 sys_fe[] = { 0x288, 0x320 }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + u32 val; + int i, ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + for (i = 0; i < ARRAY_SIZE(sys_fe); i++) { + val = readl(ps + sys_fe[i]) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + sys_fe[i]); + writel(val | 0xf | BIT(28), ps + sys_fe[i]); + } + val = readl(ps + 0x328) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28) | DCP_PS_MIN); + val |= FIELD_PREP(DCP_PS_MIN, 4); + writel(val, ps + 0x328); + writel(val | 0xf, ps + 0x328); + writel(val | 0xf | BIT(28), ps + 0x328); + + ret = readl_poll_timeout(ps + 0x328, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) { + dev_err(dcp->dev, "DISPEXT0_CPU did not power on: %#x\n", val); + } else { + dcp_pmc_set(dcp, pmc, true); + dcp->s2r_domains_up = true; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * A firmware stopped while crashed is about to start over: forget the calls and the + * transaction the old one left unanswered, and let the new one's messages through. A failed + * start marks it crashed again. + */ +static void dcp_s2r_forget_crash(struct apple_dcp *dcp) +{ + if (!READ_ONCE(dcp->crashed)) + return; + iomfb_reset_transport(dcp); + WRITE_ONCE(dcp->crashed, false); +} + +/* + * DCPEXT back after a real sleep, the same way in every macOS wake measured: its domains, + * its DART's streams, and a warm restart, with no secure port and no display-on call. After + * its own power cycle the start is the same, with its display DART lit again. + */ +static int dcp_s2r_ext_start(struct apple_dcp *dcp, bool cycled) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + dcp_s2r_forget_crash(dcp); + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (!ret) + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + /* After the domains, whose writes it would reject, the DART protects them again. */ + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + if (!cycled) + return apple_rtkit_sleep_return(dcp->rtk, control); + + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, false); + return apple_rtkit_power_cycled_return(dcp->rtk, control); +} + +/* DISPEXT0 off in its own order, CPU first. Only the CPU has to power-gate to start again. */ +static int dcp_s2r_ext_domains_off(struct apple_dcp *dcp) +{ + static const struct { + u16 offset; + u32 first, second; + bool member; + } steps[] = { + { 0x328, 0xf0400ff, 0xf0403f0 }, { 0x320, 0, 0x23f0, true }, { 0x288, 0, 0x23f0 }, + }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + int i, ret = -ENODEV; + u32 val; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + /* + * Its display DART goes off with DISPEXT0, and every framebuffer is mapped into it: a + * map during the hold would reach its registers. Dark until the domains are back. + */ + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, true); + dcp->s2r_domains_up = false; + dcp_pmc_set(dcp, pmc, false); + ret = 0; + for (i = 0; i < ARRAY_SIZE(steps); i++) { + int gated; + + if (steps[i].first) + writel(steps[i].first, ps + steps[i].offset); + writel(steps[i].second, ps + steps[i].offset); + gated = readl_poll_timeout(ps + steps[i].offset, val, + !(val & GENMASK(7, 4)) && !(val & BIT(11)) && + (steps[i].member || (val & BIT(8))), 100, 50000); + dev_dbg(dcp->dev, "DISPEXT0 power word %#x reads %#x: %d\n", steps[i].offset, val, + gated); + if (i == 0 && gated) + ret = gated; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +#define DCP_S2R_EXT_CYCLE_HOLD_MS 1000 + +static bool dcp_s2r_ext_cpu_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps; + bool on; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return false; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + of_node_put(np); + if (!ps) + return false; + on = FIELD_GET(GENMASK(7, 4), readl(ps + 0x328)); + iounmap(ps); + + return on; +} + +/* + * After a refused sleep DCPEXT has idled with its power kept, and an idled firmware doesn't + * wake again: power-cycle it through DISPEXT0's own sequence, then start it as after a sleep. + * The sleep driver lowers DISPEXT0 before refusals of its own, and with it off the CPU and the + * display DART don't answer, so it's raised first. + */ +static int dcp_s2r_ext_cycle(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (ret) + return ret; + if (!dcp_s2r_ext_cpu_on(dcp)) { + dev_info(dcp->dev, "DISPEXT0 lowered by the refused sleep: raised for the cycle\n"); + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + } + + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + ret = dcp_s2r_ext_domains_off(dcp); + if (!ret) { + msleep(DCP_S2R_EXT_CYCLE_HOLD_MS); + ret = dcp_s2r_ext_start(dcp, true); + } + + return ret; +} + +/* Between the main mailbox's IOP report and its AP on, measured 2 of 2. */ +static void dcp_secure_port_map(void *data) +{ + struct apple_dcp *dcp = data; + ktime_t start = ktime_get(); + + dcp->secure_map_ret = apple_rtkit_port_map(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + dcp->secure_map_us = ktime_us_delta(ktime_get(), start); +} + +static int dcp_s2r_warm_restart(struct apple_dcp *dcp) +{ + int ret; + + dcp_s2r_forget_crash(dcp); + ret = dcp_s2r_domains_on(dcp); + if (ret) + return ret; + + /* After the domains, whose writes it would reject, the DART protects them again. */ + if (dcp->s2r_dcp_dart) + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + + if (dcp->secure_rtk) { + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_restart(dcp->secure_rtk); + if (ret) + return ret; + } + + ret = apple_rtkit_warm_restart(dcp->rtk); + if (ret) + return ret; + + writel(APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + ret = apple_rtkit_boot_with(dcp->rtk, dcp->secure_rtk ? dcp_secure_port_map : NULL, dcp); + if (ret) + return ret; + + if (dcp->secure_rtk) { + ret = dcp->secure_map_ret ?: + apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port handshake after restart: %d\n", ret); + else + dev_dbg(dcp->dev, "secure port up after restart, map took %lld us\n", + dcp->secure_map_us); + dcp_secure_poll_start(dcp); + } + + return 0; +} + +/* + * Display on after a sleep, and after DPMS off: the warm restart, then the named calls + * macOS sends first, A414(0), A478 x2, A476(1), A478, A412, then A473(1). + * + * Not sent: A439, which macOS overlaps with A473(1), A438 around swaps, A461 per frame + * and A352. They stay unnamed, and A439 and A438 hold macOS kernel pointers. Linux + * already lights the panel without A438 at boot. + */ +static int dcp_s2r_display_on(struct apple_dcp *dcp) +{ + int ret; + + ret = dcp_s2r_warm_restart(dcp); + if (ret) { + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + return ret; + } + + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + + /* Out of band; either channel answered on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(1): %d\n", ret); + goto out; + } + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); +out: + WRITE_ONCE(dcp->s2r_active, false); + + return ret; +} + +/* Power kept: lowered as DPMS off does. A failed flush left the GAPF slice faulting that. */ +static int dcp_s2r_refused_domains_off(struct apple_dcp *dcp) +{ + int ret = dcp->s2r_dcp_dart ? apple_dart_gapf_ready(dcp->s2r_dcp_dart) : -ENODEV; + + return ret ?: dcp_s2r_domains_off(dcp); +} + +static void dcp_s2r_restart_done(struct apple_dcp *dcp, int ret) +{ + bool ext = dcp_s2r_ext_capable(dcp); + + dcp_s2r_put_darts(dcp); + /* An attached display's crossbar output; its link reconnects from the Type-C work. */ + if (ext) + dcp_typec_route_unpark(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); + return; + } + /* Back from a full restart, even after a failed start had marked it crashed. */ + WRITE_ONCE(dcp->crashed, false); + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + if (ext && !READ_ONCE(dcp->typec_work_enabled)) + dcp_enable_typec_work(dcp); +} + +/* + * If the display doesn't come back, the panel stays dark until reboot, and the driver + * must not talk to the stopped firmware, so treat it as a crashed DCP. Nothing else in + * the system depends on it. + */ +static int dcp_s2r_restart(struct apple_dcp *dcp, bool real) +{ + int ret = -EAGAIN; + + /* + * After a refused sleep the firmware has idled with its power kept, and an idled firmware + * doesn't wake again: the internal DCP comes back as DPMS on brings it, from a power + * cycle, and DCPEXT through its own. + */ + if (dcp_s2r_capable(dcp)) { + ret = real ? 0 : dcp_s2r_refused_domains_off(dcp); + if (!ret) + ret = dcp_s2r_display_on(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware restarted as for DPMS: %d\n", + ret); + } else if (dcp_s2r_ext_capable(dcp)) { + ret = real ? dcp_s2r_ext_start(dcp, false) : dcp_s2r_ext_cycle(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware %s after a power cycle: %d\n", + ret ? "not back" : "back", ret); + else if (ret) + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + } + dcp_s2r_restart_done(dcp, ret); + + return ret; +} + +static int dcp_s2r_resume(struct apple_dcp *dcp) +{ + bool real = pm_resume_via_firmware(); + + dcp->s2r_stopped = false; + /* A real sleep took the domains' power. */ + if (real) + dcp->s2r_domains_up = false; + /* A dark wake sleeps again with the panel never lit, as if DPMS had turned it off. */ + if (pm_dark_wake() && dcp_s2r_capable(dcp) && !dcp->dpms_powered_off) { + dcp->dpms_powered_off = true; + dev_info(dcp->dev, "dark wake: firmware and panel stay off\n"); + } + /* DCPEXT too, its crossbar output parked: dcp_s2r_dark_finish() if no resume follows. */ + if (pm_dark_wake() && dcp_s2r_ext_capable(dcp)) { + dcp->s2r_dark_held = true; + dev_info(dcp->dev, "dark wake: firmware and output stay off\n"); + return 0; + } + dcp->s2r_dark_held = false; + /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ + if (dcp->dpms_powered_off) { + /* Off after a refused idle, its power may be on: DPMS on needs it lowered. */ + if (!real) + dcp_s2r_refused_domains_off(dcp); + dcp_s2r_put_darts(dcp); + return 0; + } + dcp_s2r_restart(dcp, real); + + return 0; +} + +/* + * The suspend after a dark wake failed before suspending DCPEXT, or before preparing it. It's + * still down from the real sleep, whose wake reset its CPU: start it as after one. + */ +static void dcp_s2r_dark_finish(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->s2r_dark_held || pm_dark_wake() || IS_ERR_OR_NULL(dcp->rtk)) + return; + dcp->s2r_dark_held = false; + dcp->s2r_stopped = false; + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_ext_start(dcp, true); + dev_info(dcp->dev, "dark wake over without a resume: firmware %s: %d\n", + ret ? "not back" : "back", ret); + dcp_s2r_restart_done(dcp, ret); +} + +static int dcp_s2r_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_dcp *dcp = container_of(nb, struct apple_dcp, s2r_pm_nb); + + if (action == PM_POST_SUSPEND) + dcp_s2r_dark_finish(dcp); + + return NOTIFY_DONE; +} + +static void dcp_s2r_pm_unregister(void *data) +{ + struct apple_dcp *dcp = data; + + unregister_pm_notifier(&dcp->s2r_pm_nb); +} + +static int dcp_s2r_pm_register(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp_s2r_ext_capable(dcp)) + return 0; + dcp->s2r_pm_nb.notifier_call = dcp_s2r_pm_notify; + ret = register_pm_notifier(&dcp->s2r_pm_nb); + if (ret) + return ret; + + return devm_add_action_or_reset(dcp->dev, dcp_s2r_pm_unregister, dcp); +} + +/* + * Its own suspend failed with the CPU stopped, and the PM core won't call .resume: start it + * again as a refused sleep does, or the display stays dark until a later sleep gets through. + */ +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp) +{ + dev_warn(dcp->dev, "stop failed with the CPU stopped: starting the firmware again\n"); + dcp_s2r_restart(dcp, false); + dcp->s2r_stopped = false; +} + +/* + * DPMS on an internal panel with hw.s2r as macOS does a display sleep: the backlight to 0, + * then the firmware stopped and the display domains off, and the reverse on wake. Not + * during a system sleep: that path stops the firmware itself. + */ +static bool dcp_dpms_power_cycles(struct apple_dcp *dcp) +{ + return dcp_s2r_capable(dcp) && pm_suspend_target_state == PM_SUSPEND_ON; +} + +static void dcp_dpms_power_off(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (!dcp_dpms_power_cycles(dcp) || READ_ONCE(dcp->s2r_fw_down)) + return; + + ret = dcp_s2r_suspend(dcp); + if (ret && !READ_ONCE(dcp->s2r_fw_down)) { + dev_warn(dcp->dev, "DPMS: firmware stop refused, backlight only: %d\n", ret); + return; + } + dcp->s2r_stopped = false; + if (!ret) + ret = dcp_s2r_domains_off(dcp); + dcp_s2r_put_darts(dcp); + dcp->dpms_powered_off = true; + dev_dbg(dcp->dev, "DPMS: display powered off in %lld ms: %d\n", + ktime_ms_delta(ktime_get(), start), ret); +} + +/* After a refused idle the firmware still runs with the display off: light it again there. */ +static void dcp_s2r_light_running(struct apple_dcp *dcp) +{ + int ret; + + dcp_s2r_find_darts(dcp); + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (!ret && !wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "display not back on the running firmware: %d\n", ret); + return; + } + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "display back on the running firmware\n"); +} + +static void dcp_dpms_power_on(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (dcp->s2r_refused_off) { + dcp->s2r_refused_off = false; + dcp_s2r_light_running(dcp); + return; + } + if (!dcp->dpms_powered_off) + return; + dcp->dpms_powered_off = false; + if (!READ_ONCE(dcp->s2r_fw_down)) + return; + + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_display_on(dcp); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "DPMS: display power-on failed, it stays off: %d\n", ret); + return; + } + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "DPMS: display powered on in %lld ms\n", + ktime_ms_delta(ktime_get(), start)); +} + +bool dcp_s2r_firmware_stopped(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return READ_ONCE(dcp->s2r_fw_down); +} + +static int dcp_platform_suspend(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + /* Its power sequences may crash the DCP and blank the panel. */ + if (dcp_takeover_was_aborted(dcp)) + return -EBUSY; + + if (pm_suspend_via_firmware() && dcp_s2r_capable(dcp)) + return dcp_s2r_suspend(dcp); + if (pm_suspend_via_firmware() && dcp_s2r_ext_capable(dcp)) + return dcp_s2r_ext_suspend(dcp); + + /* + * T8132: setPowerState powers the display down; the coprocessor keeps + * running and its power domains stay on, as below, unless the internal + * display is blanked with appledrm.power_gate_ms set: then the + * coprocessor and its domains are powered down too (iomfb_m4_suspend()). + */ + if (iomfb_is_m4(dcp)) { + int ret = iomfb_m4_suspend(dcp); + + if (!ret) + device_set_wakeup_path(dev); + return ret; + } + if (READ_ONCE(dcp->ext_dead)) + return 0; + iomfb_v14_7_backlight_suspend(dcp); + + /* Serialize PM against explicit bring-up, including queued startup work. + * Before a startup attempt the device remains suspendable. After any + * attempt we cannot prove DMA/firmware quiescence, even on failure. + */ + if (dcp->external) { + const char *why; + + mutex_lock(&dcp->hpd_mutex); + why = dcp_external_sleep_vetoed(dcp); + if (why) { + mutex_unlock(&dcp->hpd_mutex); + if (dcp->external_native) + dev_warn(dev, "suspend refused: %s\n", why); + else + dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); + return -EBUSY; + } + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); + } + + if (dcp_is_external_h17p(dcp)) { + int ret; + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + device_set_wakeup_path(dev); + return 0; + } + + dcp_drain_for_sleep(dcp); + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + if (dcp->external_native) + disable_work_sync(&dcp->external_audio_work); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->hdmi_hpd_irq) { + disable_irq(dcp->hdmi_hpd_irq); + if (!dcp->active_typec_route) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); + /* Last: the steps above still talk to the coprocessor. */ + dcp_hold_for_sleep(dcp); + + return 0; +} + +static int dcp_platform_resume(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp_takeover_was_aborted(dcp)) + return 0; + + if (dcp->s2r_stopped) + return dcp_s2r_resume(dcp); if (iomfb_is_m4(dcp)) return iomfb_m4_resume(dcp); @@ -7195,6 +8501,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .firmware_display_windows = 5, .firmware_scratch = 0x20000, .firmware_request = 0x68000, + .s2r = true, }; /* Separate from the established internal-panel H17P profile. */ @@ -7208,6 +8515,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { /* Native PMGR BWR agent 2 and boost agent 3 for DISPEXT0. */ .firmware_scratch = 0x28000, .firmware_request = 0x70000, + .s2r_ext = true, }; /* diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index c9a8aab1095a5..4a60ead3fe995 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -136,6 +136,7 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp); int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); +bool dcp_s2r_firmware_stopped(struct platform_device *pdev); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index df7ea57955f38..0a9e1cd356920 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -365,7 +365,7 @@ static bool iomfb_uses_queue(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } -static bool iomfb_channels_idle(struct apple_dcp *dcp) +bool iomfb_channels_idle(struct apple_dcp *dcp) { return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && @@ -673,6 +673,38 @@ void iomfb_queue_init(struct apple_dcp *dcp) atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } +/* + * For firmware stopped cold and started again: the calls and the transaction the old + * one never answered went with it, and would hold the channels and the queue forever. + */ +void iomfb_reset_transport(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction; + + if (!iomfb_uses_queue(dcp)) + return; + + cancel_delayed_work_sync(&dcp->iomfb.timeout); + mutex_lock(&dcp->iomfb.lock); + transaction = dcp->iomfb.active; + dcp->iomfb.active = NULL; + if (transaction) { + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; + transaction->release(transaction); + } + iomfb_discard_pending(dcp); + dcp->ch_cmd.depth = 0; + dcp->ch_oobcmd.depth = 0; + dcp->ch_cb.depth = 0; + dcp->ch_oobcb.depth = 0; + dcp->ch_async.depth = 0; + dcp->ch_oobasync.depth = 0; + dcp->present_state_h17p.pending = false; + dcp->present_state_h17p.accepted = false; + mutex_unlock(&dcp->iomfb.lock); +} + void iomfb_queue_stop(struct apple_dcp *dcp) { struct iomfb_callback *item; @@ -3182,6 +3214,10 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct platform_device *pdev = apple_crtc->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + /* Can't be lit: a resume then restores its CRTC disabled, not failing every output. */ + if (READ_ONCE(dcp->crashed)) + return false; + if (iomfb_is_m4(dcp)) { struct drm_display_mode fixed; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 1021825209fb3..958b3b889ed10 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -196,6 +196,7 @@ enum dcpep_method { dcpep_pipe_query_478, dcpep_pipe_query_474, dcpep_pipe_set_476, + dcpep_unnamed_500, iomfbep_set_brightness_correction, dcpep_num_methods }; @@ -216,6 +217,7 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_channels_idle(struct apple_dcp *dcp); bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, u32 out_len); /* Desktop 26.6: does the record hold what the tag's handler reads and writes? */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 1e24d0e617a79..eb3e7e0e84c05 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -241,6 +241,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A500", dcpep_unnamed_500), /* (0x4, 0) */ IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ IOMFB_METHOD("A428", iomfbep_set_brightness_correction), /* (0x4, 0x4) 0x10000 */ IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ @@ -312,6 +313,21 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +/* + * D572: the firmware controls the display DART's power stage with this. The + * argument is 0 when it's done with the display and 1 when it wants it back. + */ +static bool trampoline_dart_power_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_packet_header *header = in - sizeof(*header); + + if (header->in_len >= sizeof(u32)) + dcp_s2r_dart_power(dcp, get_unaligned_le32(in)); + + return trampoline_zero(dcp, tag, out, in); +} + static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -691,7 +707,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [563] = trampoline_true, [565] = trampoline_true, [567] = trampoline_true, - [572] = trampoline_zero, /* powerUpDART */ + [572] = trampoline_dart_power_h17p, /* powerUpDART */ [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 52235ecf334e3..cb4e6320751d8 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -540,6 +540,7 @@ DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_IN(dcp_unnamed_500, dcpep_unnamed_500, u32); #endif DCP_THUNK_INOUT(dcp_set_brightness_correction, iomfbep_set_brightness_correction, u32, u32); @@ -2487,6 +2488,121 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } } +/* V27 shares the h17p template prefix, but does not own its sleep RPCs. */ +#if DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) +/* + * set_power_state around a firmware-driven suspend. What macOS sends before it in + * either direction: A414(0), A478 twice, A476(1), A478, A412. All are named in + * this firmware's table, and their arguments are measured from macOS. + * + * For display on, macOS also sends A439 at the same moment. Its name isn't known, + * it carries a macOS kernel pointer and its reply wasn't captured, so it isn't + * sent here. + */ +enum dcp_s2r_step { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A412, DCP_S2R_A473, +}; + +static const u8 dcp_s2r_seq[] = { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A478, + DCP_S2R_A412, DCP_S2R_A473, +}; + +static void dcp_s2r_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned int step = dcp->s2r_step++; + + switch (dcp_s2r_seq[step]) { + case DCP_S2R_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A478: + dcp_pipe_query_478(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A476: { + u32 val = 1; + + dcp_pipe_set_476(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A412: + dcp_is_main_display(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A473: { + /* Measured: the state, then 00 00 00 00 00 aa aa aa. */ + struct DCP_FW_NAME(dcp_set_power_state_req) req = { + .unklong = dcp->s2r_power_on, + .padding = { 0xaa, 0xaa, 0xaa }, + }; + + dcp_set_power_state(dcp, dcp->s2r_power_oob, &req, + dcp->s2r_power_on ? dcp_on_final : complete_set_powerstate, + cookie); + break; + } + } +} + +static void dcp_s2r_500_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); +} + +/* Unnamed. macOS sends it after D572 on every display off, with these bytes. */ +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + u32 val = 0xaaaa0000; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_unnamed_500(dcp, false, &val, dcp_s2r_500_done, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} + +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->s2r_power_on = on; + dcp->s2r_power_oob = oob; + dcp->s2r_step = named_calls ? 0 : ARRAY_SIZE(dcp_s2r_seq) - 1; + dcp_s2r_run(dcp, NULL, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} +#endif + static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { /* H17P does not confirm that scanout has stopped. */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 3bd2b0d2c214b..93e8895e9fb48 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -389,5 +389,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); +#if DCP_FW_H17(DCP_FW_VER) +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp); +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob); +#endif #endif diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 1a7fed6c11403..e2faa77ef3767 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/pmpdash.o +aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ba6301fa1c595..12aa70c6bf750 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -247,12 +247,80 @@ asahi_of_device_table!( #[cfg(CONFIG_DRM_ASAHI_G15)] asahi_of_device_table!(); +struct SleepOps(bindings::dev_pm_ops); + +// SAFETY: Immutable: the callbacks are plain functions. +unsafe impl Sync for SleepOps {} + +/// # Safety +/// +/// `dev` must be a device this driver is bound to, as for a PM core callback. +unsafe fn sleep_data<'a>(dev: *mut bindings::device) -> &'a AsahiPlatformDriverData<'a> { + // SAFETY: Probe stored a `Pin>` here, and the PM core keeps the + // device bound for the duration of a callback. + unsafe { &*bindings::dev_get_drvdata(dev).cast() } +} + +/// Runs before any device suspends, so the GPU goes first and work still in flight can finish. +unsafe extern "C" fn sleep_prepare(dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: Called by the PM core. + let data = unsafe { sleep_data(dev) }; + match data.g17.as_ref() { + Some(gpu) => kernel::error::from_result(|| gpu.sleep_prepare().map(|_| 0)), + None => EBUSY.to_errno(), + } +} + +/// Runs after every device resumed. Does nothing unless [`sleep_prepare`] stopped the firmware. +unsafe extern "C" fn sleep_complete(dev: *mut bindings::device) { + // SAFETY: Called by the PM core. + if let Some(gpu) = unsafe { sleep_data(dev) }.g17.as_ref() { + gpu.sleep_complete(); + } +} + +static SLEEP_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_prepare), + complete: Some(sleep_complete), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// For a driver with no firmware restart: a sleep that powers the SoC down would lose its firmware. +unsafe extern "C" fn sleep_refuse(_dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if unsafe { bindings::pm_suspend_via_firmware() } { + EBUSY.to_errno() + } else { + 0 + } +} + +static SLEEP_REFUSE_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_refuse), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// The callbacks of a driver that refuses a sleep that powers the SoC down. +pub(crate) fn sleep_refuse_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_REFUSE_OPS.0) +} + /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { type IdInfo = ProbeConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_OPS.0) + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { // SAFETY: Takes no arguments, process context. Clears the T6001 AGX request in the PMP report if the pmp_report.agx_vote // parameter set it; a no-op everywhere else. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 861668429197d..4cb221fb0a5df 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -178,6 +178,10 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(driver) = &this._driver { driver.close_binding(); diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 38fd612d240da..41fdad4e4cdaa 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -145,6 +145,10 @@ impl platform::Driver for Driver { type Data<'bound> = Option>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 0b3b0641181b9..250264e01ce13 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -61,6 +61,7 @@ mod recovery; mod replay; mod reports; mod runtime; +mod sleep; pub(crate) mod stats; pub(crate) mod status; mod teardown; @@ -235,6 +236,9 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, + /// The firmware's warm restart across a system sleep is known for this SoC. Otherwise the + /// driver refuses a sleep that powers the SoC down. + pub(crate) sleep_restart: bool, /// The SoC's values of region C; `None` where the firmware graph has its own (M5). pub(crate) region_c: Option<&'static fw::initdata::RegionCConfig>, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, @@ -602,7 +606,11 @@ impl Coprocessor { ) -> Result> { for attempt in 0..RECEIVER_ATTEMPTS { match rtkit::RtKit::::new(dev.as_ref().as_ref(), None, index, state.clone()) { - Ok(receiver) => return Ok(receiver), + Ok(receiver) => { + // Its OSLOG counters stay unanswered until a soak shows echoing them is safe. + receiver.set_oslog_echo(false); + return Ok(receiver); + } Err(error) if error == EINVAL || error == ETIMEDOUT => { if attempt + 1 < RECEIVER_ATTEMPTS { fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); @@ -644,6 +652,15 @@ impl Coprocessor { self.rtkit.send(EP_DOORBELL, msg) } + /// Starts the CPU of firmware that lost power with its state intact, and waits for RTKit to + /// come up. No endpoint starts and no initialization: it already has both. + fn restart(&mut self) -> Result { + self.rtkit.with(|rtkit| rtkit.warm_restart())?; + self.cpu.start()?; + self.running = true; + self.rtkit.with(|rtkit| rtkit.boot()) + } + /// Hands the firmware the host's log buffer ([`MSG_LOG_BUFFER`]). fn publish_log_buffer(&self, va: u64, size: usize) -> Result { let msg = MSG_LOG_BUFFER @@ -660,6 +677,14 @@ impl Coprocessor { self.running = false; } } + + /// The books for a CPU that the sleep's commit stopped, with no register access. + fn stopped_at_commit(&mut self) { + if self.running { + self.cpu.stopped_elsewhere(); + self.running = false; + } + } } impl Drop for Coprocessor { @@ -692,6 +717,44 @@ impl Bridge { } } + /// The bridge across a power-off of the GPU's domain. Nothing on a target that runs without + /// it. Teardown still clears it afterwards. + fn set(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + return self.regs.enable_g17_bridge(); + } + self.regs.disable_g17_bridge(); + Ok(()) + } + + /// Core gating across a power-off of the GPU's domain. Nothing on a target that runs without + /// the bridge. + fn set_gating(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + self.regs.enable_g17_gating() + } else { + self.regs.hold_g17_gating() + } + } + + /// Both bridge words and the gating word, on a target that has them. + fn words(&self) -> Option<[u32; 3]> { + if !self.enabled { + return None; + } + self.regs.g17_fabric_words().ok() + } + + fn forget(&mut self) { + self.enabled = false; + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -780,7 +843,10 @@ impl Firmware { /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { use recovery::Memory; - if self.recovery.pending() || self.init.recovery_state()? != 0 { + if self.primary.state.shared.sleeping() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + { return Err(EAGAIN); } let target = channel::Rings::publish_control(&self.init, record)?; @@ -940,7 +1006,7 @@ impl Firmware { /// Observe a telemetry epoch boundary without changing GPU power policy. fn observe_statistics_power(&mut self) { - if self.rings.statistics_suspended() { + if self.rings.statistics_suspended() || self.primary.state.shared.sleeping() { return; } let roles = self.roles(); @@ -1117,6 +1183,14 @@ struct Shared { /// The M5 shared-queue resource policy (frozen device topology). render_policy: Option, crashed: AtomicBool, + /// The firmware is stopped, or about to be: nothing may read or publish to it. + sleeping: AtomicBool, + /// A sleep holds preparations closed, so a recovery settling meanwhile doesn't reopen them. + sleep_gate: AtomicBool, + /// The power state word as it was before the firmware stopped. + sleep_ps: AtomicU32, + /// The stop the sleep's commit makes, from `.prepare` to `.complete`; null otherwise. + sleep_commit: core::sync::atomic::AtomicPtr, feed: Arc, #[pin] feed_work: Work, @@ -1233,6 +1307,9 @@ impl_has_work! { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } if let Err(error) = this.grow_render() { if this.state.lock().is_some() { dev_err!( @@ -1256,6 +1333,9 @@ impl WorkItem for Shared { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } let result = { let mut state = this.state.lock(); match (*state).as_deref_mut() { @@ -1656,6 +1736,10 @@ impl Gpu { firmware_test: !submits, render_policy, crashed: AtomicBool::new(false), + sleeping: AtomicBool::new(false), + sleep_gate: AtomicBool::new(false), + sleep_ps: AtomicU32::new(0), + sleep_commit: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index be27fedaca625..f9bd31f13bc7d 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -541,6 +541,11 @@ impl Port { self.demand.set_active(full); } + /// The next statistics drain and reader discard what the decoder holds. + pub(super) fn statistics_epoch_bump(&self) { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } + pub(super) fn statistics_epoch(&self) -> u64 { self.statistics_epoch.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 27e2cb008bacf..b4abbc2ebac2c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2752,6 +2752,10 @@ impl Pair { } } } + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } + /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. /// The steady state in which a command may publish without the device diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 83a4bcb2f3022..4b8a2d2f64ee8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -292,6 +292,12 @@ impl ParkedPair { } } +impl ParkedPair { + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } +} + impl Prepared { /// Called after the registry revalidates its preparation epoch and parked pair identity. pub(crate) fn finish_install( diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 72771e5c4fe5f..db7fd40bdda40 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -109,7 +109,7 @@ fn write_power( firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>, ) -> kernel::fmt::Result { - if firmware.rings.statistics_suspended() { + if firmware.rings.statistics_suspended() || firmware.primary.state.shared.sleeping() { return writeln!(f, "power_management=firmware suspended=1"); } let core_power = firmware.bridge.regs.g17_core_power(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 10ba2927f7fbe..93ebb0e93feb1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -347,6 +347,59 @@ impl Firmware { } } + /// A step of the event pass is waiting on the firmware with no event to wake it. + fn runtime_polling(&self, now: u64) -> bool { + self.render_control_backpressured() + || self.init.outer_waiting() + || self.queues.replay_outer_wait.is_some() + || self.queues.replay_consumer_wait.is_some() + || self.queues.render.replay_polling() + || self.queues.teardown.polling(now) + || self.compute_exit_polling(now) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()) + } + + /// Nothing is published that hasn't retired, nothing is waiting on the firmware, its rings + /// are empty, and the idle check has taken the effort and power holds down. + pub(in crate::g17) fn sleep_quiet(&self, now: u64) -> Result { + use crate::g17::recovery::Memory; + let compute = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + if compute + || self.queues.render_in_flight() + || !self.queues.render_settled() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + || self.runtime_polling(now) + || self.hold.active() + || self.registration_power_release_pending + || self.effort.applied() != 0 + { + return Ok(false); + } + let [consumer, unknown, producer] = self.init.control_counters()?; + if consumer != producer || unknown != producer { + return Ok(false); + } + for &role in self.roles() { + if channel::Events::pending(&self.init, role)? { + return Ok(false); + } + } + Ok(true) + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -423,6 +476,10 @@ impl crate::g17::Shared { /// callbacks are finished. Shutdown can detach Firmware during an off-lock /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { + // The way back queues a pass, which also handles a crash noticed meanwhile. + if self.sleeping() { + return; + } if self.crashed.load(core::sync::atomic::Ordering::Acquire) { self.stop_runtime(); return; @@ -492,7 +549,9 @@ impl crate::g17::Shared { } if recover { firmware.service_runtime_recovery(deferred)?; - if !firmware.recovery.pending() { + if !firmware.recovery.pending() + && !self.sleep_gate.load(core::sync::atomic::Ordering::Acquire) + { if !self.preparations.reopen() { return Ok(()); } @@ -571,21 +630,7 @@ impl crate::g17::Shared { // The firmware may consume M5 work-ring slots without a completion interrupt: while // a preparation waits for unclaimed room, every pass polls the consumers. let outer = firmware.init.poll_outer_progress()?; - let backpressured = firmware.render_control_backpressured(); - let polling = backpressured - || firmware.init.outer_waiting() - || firmware.queues.replay_outer_wait.is_some() - || firmware.queues.replay_consumer_wait.is_some() - || firmware.queues.render.replay_polling() - || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now) - || firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); + let polling = firmware.runtime_polling(now); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index aadf83cc792a7..4dac5f5230237 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -1154,6 +1154,28 @@ impl Registry { } Ok(()) } + /// No graph is being built or grown. + pub(in crate::g17) fn render_settled(&self) -> bool { + self.render + .entries + .iter() + .flatten() + .all(|entry| entry.preparing.is_none() && !entry.growing) + } + + /// The firmware restarted with no power on the cores: nothing it was told about a pair through + /// their registers can be relied on, so the next command installs it again. + pub(in crate::g17) fn forget_render_registrations(&mut self) { + for entry in self.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + pair.forget_registration(); + } + if let Some(parked) = entry.parked.as_mut() { + parked.forget_registration(); + } + } + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c new file mode 100644 index 0000000000000..6207c99613172 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The GPU's power state word across a system sleep. The domain is always on as far + * as genpd knows, so nothing else lowers it. The word and its parent come from the + * device tree. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sleep.h" + +#define PS_AUTO_ENABLE BIT(28) +#define PS_AUTO GENMASK(27, 24) +#define PS_BUSY BIT(11) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTIVE 0xf +#define PS_POLL_US 100 +#define PS_TIMEOUT_US 50000 + +/* In the "sgx" range, as regs.rs has them. */ +#define SGX_GATING 0xd06030 +#define SGX_GATING_HOLD BIT(2) +#define SGX_BRIDGE 0x1000104 +#define SGX_BRIDGE_WORDS 2 +#define SGX_BRIDGE_ENABLE BIT(0) + +#define ASC_CPU_STATUS 0x4 + +/* The PMP's request for the GPU, where the device tree names one. */ +int asahi_g17_pmp_request(struct device *dev, bool enable) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (!np) + return 0; + ret = apple_pmp_report_entry_request(np, enable); + of_node_put(np); + + return ret; +} + +static int asahi_g17_ps(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -EINVAL; + ret = of_property_read_u32(np, "reg", offset); + if (!ret) { + *regmap = np->parent ? syscon_node_to_regmap(np->parent) : ERR_PTR(-EINVAL); + ret = PTR_ERR_OR_ZERO(*regmap); + } + of_node_put(np); + + return ret; +} + +/* Automatic power management off. Returns the word as it was in @saved. */ +int asahi_g17_power_auto_off(struct device *dev, u32 *saved) +{ + struct regmap *regmap; + u32 offset; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, saved); + if (ret) + return ret; + + return regmap_write(regmap, offset, *saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS)); +} + +/* The target off, then wait for the domain to finish going off. */ +int asahi_g17_power_lower(struct device *dev) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + regmap_write(regmap, offset, + (val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS); + + ret = regmap_read_poll_timeout(regmap, offset, cur, + !FIELD_GET(PS_ACTUAL, cur) && !(cur & PS_BUSY), + PS_POLL_US, PS_TIMEOUT_US); + dev_dbg(dev, "GPU domain %#x, now %#x\n", val, cur); + + return ret; +} + +static u32 asahi_g17_ps_on(u32 val, u32 saved) +{ + val &= ~(PS_AUTO_ENABLE | PS_AUTO | PS_BUSY | PS_FLAGS | PS_ACTUAL | PS_TARGET); + + return val | (saved & PS_AUTO) | FIELD_PREP(PS_TARGET, PS_ACTIVE); +} + +/* The target on, and wait for it. */ +int asahi_g17_power_raise(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + dev_dbg(dev, "GPU domain before raising it: %#x\n", val); + + regmap_write(regmap, offset, asahi_g17_ps_on(val, saved)); + + return regmap_read_poll_timeout(regmap, offset, cur, + FIELD_GET(PS_ACTUAL, cur) == PS_ACTIVE, + PS_POLL_US, PS_TIMEOUT_US); +} + +/* Automatic power management as it was. */ +int asahi_g17_power_auto_on(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + return regmap_write(regmap, offset, asahi_g17_ps_on(val, saved) | (saved & PS_AUTO_ENABLE)); +} + +/* + * What the sleep's commit needs to stop the GPU with interrupts off, taken in .prepare, and + * what it saw, for the log after the sleep. + */ +struct asahi_g17_commit { + struct device *dev; + unsigned int count; + struct apple_rtkit *rtk[2]; + struct apple_mbox *mbox[2]; + void __iomem *cpu[2]; + void __iomem *gating; + void __iomem *bridge; + void __iomem *ps; + struct device_node *request; + + bool ran; + int idle[2]; + unsigned int idle_us[2]; + u32 status[2]; + u32 words[SGX_BRIDGE_WORDS + 1]; + int request_ret; + u32 saved; + u32 lowered; + int lower_ret; +}; + +/* + * down() at the commit, every refusal behind it: both instances idled by polling, mailboxes + * and run bits off, the gating hold, the PMP request withdrawn while the PMP still runs, then + * the bridge and the domain. Told to idle, the firmware can't carry on, so the sleep goes on + * whatever it answers. + */ +static int asahi_g17_commit_stop(void *data) +{ + struct asahi_g17_commit *c = data; + unsigned int i; + u32 val; + + c->ran = true; + for (i = 0; i < c->count; i++) + c->idle[i] = apple_rtkit_sleep_idle_polled(c->rtk[i], c->cpu[i], &c->idle_us[i]); + for (i = 0; i < c->count; i++) { + c->status[i] = readl(c->cpu[i] + ASC_CPU_STATUS); + apple_rtkit_mailbox_recv_off(c->rtk[i]); + apple_mbox_ascwrap_v6_stop_cpu_atomic(c->mbox[i]); + if (!i && c->bridge) + writel(readl(c->gating) | SGX_GATING_HOLD, c->gating); + } + c->request_ret = c->request ? apple_pmp_report_entry_withdraw_polled(c->request) : 0; + + c->saved = readl(c->ps); + writel(c->saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS), c->ps); + if (c->bridge) { + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + c->words[i] = readl(c->bridge + 4 * i); + c->words[i] = readl(c->gating); + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + writel(c->words[i] & ~SGX_BRIDGE_ENABLE, c->bridge + 4 * i); + } + val = readl(c->ps); + writel((val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS, c->ps); + c->lower_ret = readl_poll_timeout_atomic(c->ps, c->lowered, + !FIELD_GET(PS_ACTUAL, c->lowered) && + !(c->lowered & PS_BUSY), + 10, PS_TIMEOUT_US); + + return (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) ? 1 : 0; +} + +static void __iomem *asahi_g17_map(struct device_node *np, const char *name, int index, + u32 offset, u32 size) +{ + struct resource res; + + if (name) + index = of_property_match_string(np, "reg-names", name); + if (index < 0 || of_address_to_resource(np, index, &res) || + offset + size > resource_size(&res)) + return NULL; + + return ioremap(res.start + offset, size); +} + +static void asahi_g17_commit_free(struct asahi_g17_commit *c) +{ + if (c->request) { + apple_pmp_report_entry_commit_owned(c->request, false); + of_node_put(c->request); + } + if (c->ps) + iounmap(c->ps); + if (c->bridge) + iounmap(c->bridge); + if (c->gating) + iounmap(c->gating); + kfree(c); +} + +/* + * Leaves the GPU's whole stop to the sleep's commit: maps what the stop touches, keeps the + * PMP request for it, and registers it with the sleep driver, ready. Nothing is stopped + * here. @secondary_rtk is NULL on a single instance, and @bridge false on a target that runs + * without the bridge. + */ +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge) +{ + struct asahi_g17_commit *c; + struct device_node *np; + unsigned int i; + u32 offset; + int ret; + + c = kzalloc_obj(*c); + if (!c) + return ERR_PTR(-ENOMEM); + c->dev = dev; + c->rtk[0] = primary_rtk; + c->mbox[0] = primary_mbox; + c->rtk[1] = secondary_rtk; + c->mbox[1] = secondary_mbox; + c->count = secondary_rtk ? 2 : 1; + + ret = -EOPNOTSUPP; + for (i = 0; i < c->count; i++) { + c->cpu[i] = apple_mbox_ascwrap_v6_cpu_control(c->mbox[i]); + if (!c->rtk[i] || !c->cpu[i]) + goto err; + } + + ret = -ENODEV; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + goto err; + if (!of_property_read_u32(np, "reg", &offset) && np->parent) + c->ps = asahi_g17_map(np->parent, NULL, 0, offset, 4); + of_node_put(np); + c->gating = asahi_g17_map(dev->of_node, "sgx", 0, SGX_GATING, 4); + if (bridge) + c->bridge = asahi_g17_map(dev->of_node, "sgx", 0, SGX_BRIDGE, + 4 * SGX_BRIDGE_WORDS); + if (!c->ps || !c->gating || (bridge && !c->bridge)) + goto err; + + c->request = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (c->request) + apple_pmp_report_entry_commit_owned(c->request, true); + + ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + if (ret) + goto err; + apple_sleep_coproc_set_ready(dev, true); + + return c; + +err: + asahi_g17_commit_free(c); + return ERR_PTR(ret); +} + +/* Whether only the sleep's commit may stop the GPU: then nothing else may. */ +bool asahi_g17_commit_required(void) +{ + return apple_sleep_commit_stops_coprocs(); +} + +/* + * After the sleep, refused or not: whether the commit stopped the GPU, and in @saved the + * domain's word from before, if it did. Frees @c. + */ +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved) +{ + bool stopped = apple_sleep_coproc_stopped(c->dev) || c->ran; + + apple_sleep_coproc_unregister(c->dev); + if (c->ran) { + if (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) + dev_warn(c->dev, "GPU stop at the commit: idle %d %d, PMP request %d, domain %#x (%d)\n", + c->idle[0], c->idle[1], c->request_ret, c->lowered, c->lower_ret); + dev_dbg(c->dev, "GPU stopped at the commit: idle in %u %u us, status %#x %#x\n", + c->idle_us[0], c->idle_us[1], c->status[0], c->status[1]); + dev_dbg(c->dev, "GPU at the commit: bridge %#x %#x gating %#x, domain %#x now %#x\n", + c->words[0], c->words[1], c->words[2], c->saved, c->lowered); + *saved = c->saved; + } + asahi_g17_commit_free(c); + + return stopped; +} diff --git a/drivers/gpu/drm/asahi/g17/sleep.h b/drivers/gpu/drm/asahi/g17/sleep.h new file mode 100644 index 0000000000000..e4473c73e19d3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SLEEP_H +#define ASAHI_G17_SLEEP_H +#include + +struct apple_mbox; +struct apple_rtkit; +struct asahi_g17_commit; +struct device; + +int asahi_g17_pmp_request(struct device *dev, bool enable); +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge); +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved); +bool asahi_g17_commit_required(void); +int asahi_g17_power_auto_off(struct device *dev, u32 *saved); +int asahi_g17_power_lower(struct device *dev); +int asahi_g17_power_raise(struct device *dev, u32 saved); +int asahi_g17_power_auto_on(struct device *dev, u32 saved); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sleep.rs b/drivers/gpu/drm/asahi/g17/sleep.rs new file mode 100644 index 0000000000000..fe5779ee5d5e1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.rs @@ -0,0 +1,820 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! System sleep: the firmware goes idle, its domain goes off, and it restarts warm. +//! +//! The firmware isn't initialized again. It keeps its rings and state in DRAM, so a restart is +//! the RTKit handshake and the end of the power-on transition. Publication stays closed from +//! before the idle until after the restart, and nothing is in flight across it: work that doesn't +//! finish in time refuses the sleep. +//! +//! Order, on both instances, is what macOS does on the chip whose [`super::Config`] sets +//! `sleep_restart`. Other chips refuse. +//! +//! Only a real wake restarts the firmware. A sleep refused after the firmware stopped leaves it +//! stopped and publication closed until the next real sleep, with the domain on so that everything +//! behind it stays reachable. That sleep then only has the domain left to lower. +//! +//! Where the sleep driver takes it, `.prepare` only drains and checks for quiet, and the sleep's +//! commit makes the whole stop in C, with interrupts off and every refusal behind it. A sleep +//! refused before then finds the firmware never stopped. Elsewhere `.prepare` stops it. + +use super::{Firmware, Role, Shared}; +use core::sync::atomic::Ordering; +use kernel::{ + bindings, + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, + workqueue::{DelayedWork, Work}, +}; + +/// Time work in flight gets to finish before the sleep is refused. +const DRAIN_NS: u64 = 3_000_000_000; +const DRAIN_TICK_MS: i64 = 2; + +unsafe extern "C" { + fn asahi_g17_pmp_request(dev: *mut core::ffi::c_void, enable: bool) -> core::ffi::c_int; + fn asahi_g17_power_auto_off(dev: *mut core::ffi::c_void, saved: *mut u32) -> core::ffi::c_int; + fn asahi_g17_power_lower(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_g17_power_raise(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_power_auto_on(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_commit_prepare( + dev: *mut core::ffi::c_void, + primary_rtk: *mut bindings::apple_rtkit, + primary_mbox: *mut core::ffi::c_void, + secondary_rtk: *mut bindings::apple_rtkit, + secondary_mbox: *mut core::ffi::c_void, + bridge: bool, + ) -> *mut core::ffi::c_void; + fn asahi_g17_commit_finish(commit: *mut core::ffi::c_void, saved: *mut u32) -> bool; + fn asahi_g17_commit_required() -> bool; +} + +/// What `.complete` does after a sleep whose stop was left to the commit. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum AfterCommit { + /// Refused before the commit: the firmware never stopped, so only publication reopens. + Running, + /// Stopped at the commit, or its power lost anyway: as after [`down`]. + Stopped, +} + +fn after_commit(stopped: bool, real: bool) -> AfterCommit { + if stopped || real { + AfterCommit::Stopped + } else { + AfterCommit::Running + } +} + +/// What stopping and restarting the firmware touches, in the order [`down`] and [`up`] use it. +pub(super) trait Hardware { + fn has_secondary(&self) -> bool; + /// RTKit AP and IOP idle, each acknowledged. + fn idle(&mut self, role: Role) -> Result; + /// Back from idle with no restart, after a refusal. One that fails loses the device. + fn wake(&mut self, role: Role); + /// Mailbox off, then run bit off, for an instance that acknowledged idle. + fn stop(&mut self, role: Role); + /// The PMP's request for the GPU. It may use a device it's told is in use, so the request is + /// only ever set with the domain on. + fn request(&mut self, enable: bool); + /// Core gating as it is while the firmware runs, or held for a domain about to go off. + fn gating(&mut self, enable: bool) -> Result; + /// The GPU-to-fabric bridge. + fn bridge(&mut self, enable: bool) -> Result; + /// Automatic power management off, keeping the word for the way back. + fn auto_off(&mut self) -> Result; + /// The domain off, waiting until it has settled. + fn lower(&mut self) -> Result; + /// The domain on, waiting until it is. + fn raise(&mut self) -> Result; + /// Automatic power management as it was. + fn auto_on(&mut self) -> Result; + /// Run bit on, RTKit handshake to AP on. + fn restart(&mut self, role: Role) -> Result; + /// Ends the power-on transition, on every instance. + fn power_on_done(&mut self) -> Result; +} + +/// Stops the firmware and lowers the domain. An error leaves the firmware running, so the caller +/// refuses the sleep. Both instances idle before either stops, so that a refusal by the second +/// still has a way back. A domain that is slow to go off isn't an error here: the sleep driver +/// checks it at the commit. +pub(super) fn down(hw: &mut H) -> Result { + if let Err(error) = hw.idle(Role::Primary) { + hw.wake(Role::Primary); + return Err(error); + } + if hw.has_secondary() { + if let Err(error) = hw.idle(Role::Secondary) { + hw.wake(Role::Secondary); + hw.wake(Role::Primary); + return Err(error); + } + } + hw.stop(Role::Primary); + let _ = hw.gating(false); + if hw.has_secondary() { + hw.stop(Role::Secondary); + } + hw.request(false); + lower_domain(hw); + Ok(()) +} + +/// Lowers the domain, with the firmware stopped. +fn lower_domain(hw: &mut H) { + let _ = hw.auto_off(); + let _ = hw.bridge(false); + let _ = hw.lower(); +} + +/// Brings the domain back for firmware that stays stopped after a refused sleep. +fn hold(hw: &mut H) -> Result { + hw.raise()?; + hw.auto_on() +} + +/// Restarts the firmware. Any error loses the device: there is no other way back. +pub(super) fn up(hw: &mut H) -> Result { + hw.raise()?; + hw.bridge(true)?; + hw.auto_on()?; + hw.request(true); + hw.gating(true)?; + if hw.has_secondary() { + hw.restart(Role::Secondary)?; + } + hw.restart(Role::Primary)?; + hw.power_on_done() +} + +impl Hardware for Firmware { + fn has_secondary(&self) -> bool { + self.secondary.is_some() + } + + fn idle(&mut self, role: Role) -> Result { + self.coprocessor(role)?.rtkit.with(|rtkit| rtkit.idle()) + } + + fn wake(&mut self, role: Role) { + let result = match self.coprocessor(role) { + Ok(coprocessor) => coprocessor.rtkit.with(|rtkit| rtkit.wake()), + Err(error) => Err(error), + }; + // EINVAL: it never left the on state. + if result.is_err_and(|error| error != EINVAL) { + self.primary.state.shared.lose_device_quietly(); + } + } + + fn stop(&mut self, role: Role) { + if let Ok(coprocessor) = self.coprocessor(role) { + coprocessor.rtkit.with(|rtkit| rtkit.mailbox_off()); + coprocessor.stop(); + } + } + + fn request(&mut self, enable: bool) { + self.primary.state.shared.pmp_request(enable); + } + + fn gating(&mut self, enable: bool) -> Result { + self.bridge.set_gating(enable) + } + + fn bridge(&mut self, enable: bool) -> Result { + if let Some([first, second, gating]) = self.bridge.words() { + dev_dbg!( + self.primary.state.shared.dev.as_ref(), + "GPU before bridge {}: {:#x} {:#x} gating {:#x}\n", + if enable { "on" } else { "off" }, + first, + second, + gating + ); + } + self.bridge.set(enable) + } + + fn auto_off(&mut self) -> Result { + self.primary.state.shared.power_auto_off() + } + + fn lower(&mut self) -> Result { + self.primary.state.shared.power_lower() + } + + fn raise(&mut self) -> Result { + let result = self.primary.state.shared.power_raise(); + if result.is_err() { + // The registers aren't reachable, so teardown can't clear the bridge. + self.bridge.forget(); + } + result + } + + fn auto_on(&mut self) -> Result { + self.primary.state.shared.power_auto_on() + } + + fn restart(&mut self, role: Role) -> Result { + let shared = self.primary.state.shared.clone(); + let coprocessor = self.coprocessor(role)?; + let result = coprocessor.restart(); + if let (Err(_), Ok((control, status))) = (&result, coprocessor.cpu.state()) { + dev_err!( + shared.dev.as_ref(), + "GPU firmware {:?}: control {:#x} status {:#x}\n", + role, + control, + status + ); + } + result + } + + fn power_on_done(&mut self) -> Result { + if !self.variant.power_on_done { + return Ok(()); + } + self.primary.notify(super::MSG_POWER_ON_DONE)?; + if let Some(secondary) = self.secondary.as_ref() { + secondary.notify(super::MSG_POWER_ON_DONE)?; + } + Ok(()) + } +} + +impl Firmware { + /// Hands the sleep's commit what its stop needs. Nothing stops here. + fn leave_for_commit(&mut self) -> Result<*mut core::ffi::c_void> { + let shared = self.primary.state.shared.clone(); + let primary_rtk = self.primary.rtkit.with(|rtkit| rtkit.as_raw()); + let primary_mbox = self.primary.cpu.as_raw().cast(); + let (secondary_rtk, secondary_mbox) = match self.secondary.as_ref() { + Some(secondary) => ( + secondary.rtkit.with(|rtkit| rtkit.as_raw()), + secondary.cpu.as_raw().cast(), + ), + None => (core::ptr::null_mut(), core::ptr::null_mut()), + }; + // SAFETY: `dev` is the live GPU device. The RTKit and mailbox pointers stay valid while + // the firmware exists, and `.complete` frees the result before anything drops them. + let commit = unsafe { + asahi_g17_commit_prepare( + shared.dev.as_raw().cast(), + primary_rtk, + primary_mbox, + secondary_rtk, + secondary_mbox, + self.bridge.enabled, + ) + }; + kernel::error::from_err_ptr(commit) + } + + /// The books for both instances, which the commit stopped. + fn stopped_at_commit(&mut self) { + if let Some(secondary) = self.secondary.as_mut() { + secondary.stopped_at_commit(); + } + self.primary.stopped_at_commit(); + } + + fn coprocessor(&mut self, role: Role) -> Result<&mut super::Coprocessor> { + match role { + Role::Primary => Ok(&mut self.primary), + Role::Secondary => self.secondary.as_mut().ok_or(ENODEV), + } + } +} + +impl Shared { + pub(super) fn sleeping(&self) -> bool { + self.sleeping.load(Ordering::Acquire) + } + + /// A PMP that doesn't answer in time only gets a warning: nothing better is left to do here. + fn pmp_request(&self, enable: bool) { + // SAFETY: `dev` is the live GPU device. + let result = kernel::error::to_result(unsafe { + asahi_g17_pmp_request(self.dev.as_raw().cast(), enable) + }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "PMP request for the GPU, {}: {:?}\n", enable, error); + } + } + + fn power_auto_off(&self) -> Result { + let mut saved = 0; + // SAFETY: `dev` is the live GPU device and `saved` is writable. + let result = kernel::error::to_result(unsafe { + asahi_g17_power_auto_off(self.dev.as_raw().cast(), &mut saved) + }); + self.sleep_ps.store(saved, Ordering::Relaxed); + result + } + + fn power_lower(&self) -> Result { + // SAFETY: `dev` is the live GPU device. + let result = + kernel::error::to_result(unsafe { asahi_g17_power_lower(self.dev.as_raw().cast()) }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "GPU domain slow to go off: {:?}\n", error); + } + result + } + + fn power_raise(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { asahi_g17_power_raise(self.dev.as_raw().cast(), saved) }) + } + + fn power_auto_on(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { + asahi_g17_power_auto_on(self.dev.as_raw().cast(), saved) + }) + } + + fn cancel_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: As in `disable_timer`: a pending callback owned one Arc that it can no longer + // reclaim. Future enqueues stay possible and run into the sleeping check. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, bindings::delayed_work, work); + bindings::cancel_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: See above. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// With the sleeping flag set, joins whatever may be reading firmware state. The growth work + /// isn't joined: it may be waiting on the closed gate, and can't publish while it is closed. + fn join_event_workers(self: &Arc) { + self.cancel_timer(&self.poll); + self.cancel_timer(&self.idle); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + } + + /// Waits for what's been published to retire. The event worker keeps running meanwhile, and + /// its own idle check takes the power and effort holds down. + fn wait_quiet(self: &Arc, deadline: u64) -> Result { + loop { + if self.crashed.load(Ordering::Acquire) { + return Err(ENODEV); + } + let now = super::now_ns(); + { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + // Nothing else retries this with no render publication coming. + if firmware.registration_power_release_pending { + let _ = firmware.release_registration_power(); + } + if firmware.sleep_quiet(now)? { + return Ok(()); + } + } + if now >= deadline { + return Err(EBUSY); + } + self.queue_events(); + fsleep(Delta::from_millis(DRAIN_TICK_MS)); + } + } + + /// Undoes everything short of a stopped firmware. A builder that never finishes would keep + /// publication closed for good, and that is a lost device. + fn sleep_cancel(self: &Arc) { + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + let drained = self + .preparations + .wait_drained_until(super::now_ns().saturating_add(DRAIN_NS)); + if (drained.is_err() || !self.preparations.reopen()) + && !self.crashed.load(Ordering::Acquire) + { + self.lose_device_quietly(); + } + self.queue_events(); + } + + /// Keeps the firmware stopped after a sleep that never reached the hardware. False if there's + /// no firmware to keep, or no domain to keep it in. + fn sleep_hold(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { + return false; + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return false; + }; + if hold(firmware).is_err() { + return false; + } + dev_warn!( + self.dev.as_ref(), + "sleep refused: GPU firmware stays stopped until a real sleep\n" + ); + true + } + + /// Leaves the stop to the commit. False, with the reason logged, if it can't be. Where only + /// the commit may stop it, that refuses the sleep instead: a refusal after an earlier stop + /// would leave it stopped. + fn sleep_leave_for_commit(&self, firmware: &mut Firmware) -> Result { + match firmware.leave_for_commit() { + Ok(commit) => { + self.sleep_commit.store(commit, Ordering::Release); + dev_dbg!(self.dev.as_ref(), "GPU firmware left running for the sleep's commit\n"); + Ok(true) + } + // SAFETY: Reads the sleep driver's state. + Err(error) if unsafe { asahi_g17_commit_required() } => { + dev_err!( + self.dev.as_ref(), + "no GPU stop at the commit ({:?}): refusing the sleep\n", + error + ); + Err(EBUSY) + } + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "no GPU stop at the commit ({:?}): stopping it now\n", + error + ); + Ok(false) + } + } + } + + /// Whether the commit stopped the firmware, if the stop was left to it. + fn sleep_commit_finish(&self) -> Option { + let commit = self.sleep_commit.swap(core::ptr::null_mut(), Ordering::AcqRel); + if commit.is_null() { + return None; + } + let mut saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `commit` came from `asahi_g17_commit_prepare` and is finished only once. + let stopped = unsafe { asahi_g17_commit_finish(commit, &mut saved) }; + self.sleep_ps.store(saved, Ordering::Relaxed); + Some(stopped) + } + + /// After the commit stopped the firmware: the books it left, as [`down`] would have them. + fn sleep_settle_commit(&self) { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.stopped_at_commit(); + } + } + + /// Closes publication, waits for quiet, and leaves the stop to the sleep's commit where the + /// sleep driver takes it, or stops the firmware. A refusal changes nothing. + fn sleep_down(self: &Arc) -> Result { + if self.sleeping() { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + lower_domain(firmware); + } + return Ok(()); + } + let deadline = super::now_ns().saturating_add(DRAIN_NS); + self.sleep_gate.store(true, Ordering::Release); + self.preparations.close(); + let result = self.preparations.wait_drained_until(deadline).map_err(|_| EBUSY) + .and_then(|_| self.wait_quiet(deadline)) + .and_then(|_| { + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !firmware.sleep_quiet(super::now_ns())? { + return Err(EBUSY); + } + if self.sleep_leave_for_commit(firmware)? { + return Ok(()); + } + down(firmware) + }); + match result { + Ok(()) => Ok(()), + Err(error) if error == ENODEV || self.crashed.load(Ordering::Acquire) => { + // Nothing to save: let the worker finish detaching what's left, and then the + // domain only has to be off for the sleep. + self.sleeping.store(false, Ordering::Release); + self.queue_events(); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + self.pmp_request(false); + let _ = self.power_auto_off(); + let _ = self.power_lower(); + Ok(()) + } + Err(error) => { + self.sleep_cancel(); + Err(error) + } + } + } + + /// Restarts the firmware and reopens publication. A failure loses the device. + fn sleep_up(self: &Arc) -> Result { + let started = super::now_ns(); + let result = { + let mut state = self.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if !self.crashed.load(Ordering::Acquire) => { + let result = up(firmware); + if result.is_ok() { + // The cores lost power, so the next command on each pair installs it again. + firmware.queues.forget_render_registrations(); + firmware.rings.invalidate_statistics(); + firmware.port.statistics_epoch_bump(); + } + result + } + _ => self + .power_raise() + .and_then(|_| self.power_auto_on()) + .inspect(|_| self.pmp_request(true)), + } + }; + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + match result { + Ok(()) => { + if !self.crashed.load(Ordering::Acquire) { + dev_dbg!( + self.dev.as_ref(), + "GPU firmware back in {} us\n", + super::now_ns().saturating_sub(started) / 1000 + ); + } + self.preparations.reopen(); + self.queue_events(); + Ok(()) + } + Err(error) => { + dev_err!(self.dev.as_ref(), "GPU firmware did not come back: {:?}\n", error); + self.lose_device_quietly(); + Err(error) + } + } + } +} + +impl super::Gpu { + /// Called before any device suspends. Only a sleep that powers the SoC down gets here. + pub(crate) fn sleep_prepare(&self) -> Result { + if !self.cfg.sleep_restart { + dev_warn!( + self.shared.dev.as_ref(), + "no firmware restart on this chip, refusing sleep\n" + ); + return Err(EBUSY); + } + let result = self.shared.sleep_down(); + if let Err(error) = result { + dev_warn!(self.shared.dev.as_ref(), "GPU refused sleep: {:?}\n", error); + } + result + } + + /// Called after every device resumed, whether or not the sleep reached the hardware. + pub(crate) fn sleep_complete(&self) { + if !self.shared.sleeping() { + return; + } + // SAFETY: Reads a global flag word. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if let Some(stopped) = self.shared.sleep_commit_finish() { + if after_commit(stopped, real) == AfterCommit::Running { + dev_info!( + self.shared.dev.as_ref(), + "sleep refused before the commit: GPU firmware never stopped\n" + ); + self.shared.sleep_cancel(); + return; + } + self.shared.sleep_settle_commit(); + } + if !real && self.shared.sleep_hold() { + return; + } + let _ = self.shared.sleep_up(); + } +} + +#[kernel::macros::kunit_tests(asahi_g17_sleep)] +mod tests { + use super::*; + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + enum Call { + Idle(Role), + Wake(Role), + Stop(Role), + Request(bool), + Gating(bool), + Bridge(bool), + AutoOff, + Lower, + Raise, + AutoOn, + Restart(Role), + Done, + } + + struct Fake { + secondary: bool, + fail: Option, + calls: [Option; 14], + count: usize, + } + + impl Fake { + fn new(secondary: bool, fail: Option) -> Self { + Self { secondary, fail, calls: [None; 14], count: 0 } + } + + fn note(&mut self, call: Call) -> Result { + self.calls[self.count] = Some(call); + self.count += 1; + if self.fail == Some(call) { Err(EIO) } else { Ok(()) } + } + + fn called(&self, expected: &[Call]) -> bool { + self.count == expected.len() + && expected.iter().zip(self.calls.iter()).all(|(a, b)| Some(*a) == *b) + } + } + + impl Hardware for Fake { + fn has_secondary(&self) -> bool { self.secondary } + fn idle(&mut self, role: Role) -> Result { self.note(Call::Idle(role)) } + fn wake(&mut self, role: Role) { let _ = self.note(Call::Wake(role)); } + fn stop(&mut self, role: Role) { let _ = self.note(Call::Stop(role)); } + fn request(&mut self, enable: bool) { let _ = self.note(Call::Request(enable)); } + fn gating(&mut self, enable: bool) -> Result { self.note(Call::Gating(enable)) } + fn bridge(&mut self, enable: bool) -> Result { self.note(Call::Bridge(enable)) } + fn auto_off(&mut self) -> Result { self.note(Call::AutoOff) } + fn lower(&mut self) -> Result { self.note(Call::Lower) } + fn raise(&mut self) -> Result { self.note(Call::Raise) } + fn auto_on(&mut self) -> Result { self.note(Call::AutoOn) } + fn restart(&mut self, role: Role) -> Result { self.note(Call::Restart(role)) } + fn power_on_done(&mut self) -> Result { self.note(Call::Done) } + } + + use Role::{Primary as P, Secondary as S}; + + #[test] + fn down_idles_the_primary_first_and_lowers_the_domain_last() { + let mut hw = Fake::new(true, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Idle(S), + Call::Stop(P), + Call::Gating(false), + Call::Stop(S), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + } + + #[test] + fn up_restarts_the_secondary_first_and_ends_the_transition_last() { + let mut hw = Fake::new(true, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_single_instance_never_names_a_secondary() { + let mut hw = Fake::new(false, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Stop(P), + Call::Gating(false), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + let mut hw = Fake::new(false, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_refused_idle_touches_nothing_past_the_firmware() { + for call in [Call::Idle(P), Call::Idle(S)] { + let mut hw = Fake::new(true, Some(call)); + assert_eq!(down(&mut hw), Err(EIO)); + let past = |c: &Call| !matches!(c, Call::Idle(_) | Call::Wake(_)); + assert!(!hw.calls.iter().flatten().any(past)); + } + } + + #[test] + fn a_primary_that_refuses_idle_is_woken_and_nothing_stops() { + let mut hw = Fake::new(true, Some(Call::Idle(P))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Wake(P)])); + } + + #[test] + fn a_secondary_that_refuses_idle_wakes_both_and_stops_nothing() { + let mut hw = Fake::new(true, Some(Call::Idle(S))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Idle(S), Call::Wake(S), Call::Wake(P)])); + } + + #[test] + fn a_refused_sleep_only_brings_the_domain_back() { + let mut hw = Fake::new(true, None); + assert_eq!(hold(&mut hw), Ok(())); + assert!(hw.called(&[Call::Raise, Call::AutoOn])); + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(hold(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + } + + #[test] + fn the_sleep_after_a_refusal_only_lowers_the_domain() { + let mut hw = Fake::new(true, None); + lower_domain(&mut hw); + assert!(hw.called(&[Call::AutoOff, Call::Bridge(false), Call::Lower])); + } + + #[test] + fn a_slow_domain_does_not_fail_down() { + let mut hw = Fake::new(false, Some(Call::Lower)); + assert_eq!(down(&mut hw), Ok(())); + } + + #[test] + fn up_stops_at_the_first_failure() { + let mut hw = Fake::new(true, Some(Call::Restart(S))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + ])); + // No domain, so no bridge and no request either. + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + // No bridge, so the PMP isn't told about the GPU. + let mut hw = Fake::new(true, Some(Call::Bridge(true))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise, Call::Bridge(true)])); + } + + #[test] + fn a_stop_left_to_the_commit_is_undone_only_if_it_never_ran() { + assert_eq!(after_commit(false, false), AfterCommit::Running); + assert_eq!(after_commit(true, false), AfterCommit::Stopped); + assert_eq!(after_commit(true, true), AfterCommit::Stopped); + // No stop, but the power went anyway: the firmware has to restart. + assert_eq!(after_commit(false, true), AfterCommit::Stopped); + } + +} diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 102e8d813a83f..f9321745cecea 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -58,6 +58,7 @@ pub(crate) const CONFIG: Config = Config { m5: None, adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on // the firmware's own blame. recovery_slot_keys: None, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 9d61935a60807..98a36216528e2 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -49,6 +49,7 @@ pub(super) const BASE: Config = Config { m5: None, adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G16, firmware: fw::variant::Firmware::Identified(&[]), diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index ae2b48d9184d1..949bd81d00edd 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -40,6 +40,7 @@ pub(crate) const CONFIG: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J714), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, // The firmware in `fw-text` must be this build. iBoot loads the G17S build of both roles @@ -110,6 +111,7 @@ pub(crate) const CONFIG_MAX: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J716C), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, // As on the M5 Ultra, the firmware in `fw-text` must be this build. The boot loader loads @@ -191,6 +193,7 @@ pub(crate) const CONFIG_ULTRA: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J775D), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, // On bare metal nothing vouches for the firmware the boot loader left in `fw-text`: it diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 891bb757db9e4..1ade3df12c2cb 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -56,6 +56,7 @@ pub(crate) const CONFIG: Config = Config { m5: None, adopt_root_entries: true, fabric_bridge: true, + sleep_restart: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { diff --git a/drivers/gpu/drm/asahi/hw/t8142.rs b/drivers/gpu/drm/asahi/hw/t8142.rs index 044e28f20684e..26fe2e49abcdb 100644 --- a/drivers/gpu/drm/asahi/hw/t8142.rs +++ b/drivers/gpu/drm/asahi/hw/t8142.rs @@ -44,6 +44,7 @@ pub(crate) const PROBE: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J714), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428_M5), diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 84679b3b3609b..8f2cd465ebec7 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -80,6 +80,7 @@ const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) /// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. const G17_GATING_CONTROL: usize = 0xd06030; const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) +const G17_GATING_HOLD: u32 = 0x4; // BIT(2) /// These performance selectors must be sampled before firmware can gate the core registers. const G17_PERF_CONTROL: usize = 0xe0141c; @@ -281,6 +282,37 @@ impl CpuControl { } } } + + /// The provider, for a system sleep's last step to stop the CPU with interrupts off. + pub(crate) fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + + /// Settles the provider's books for a CPU that a system sleep's last step stopped. + pub(crate) fn stopped_elsewhere(&self) { + // SAFETY: The constructor established the live, role-checked provider. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_cpu_stopped(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!( + self.dev.as_ref(), + "Could not settle a stopped GPU coprocessor: {:?}\n", + error + ); + } + } + } + + /// The CPU's control and status words. + pub(crate) fn state(&self) -> Result<(u32, u32)> { + let (mut control, mut status) = (0, 0); + // SAFETY: The constructor established the live, role-checked provider, and both pointers + // are writable. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_cpu_state(self.mbox, &mut control, &mut status) + })?; + Ok((control, status)) + } } /// Device resources for this GPU instance. @@ -895,6 +927,25 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } + + /// Hold part of the gating off, for a domain about to go off. The firmware must be idle. + pub(crate) fn hold_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val | G17_GATING_HOLD, G17_GATING_CONTROL) + } + + /// Both bridge words and the gating word. + pub(crate) fn g17_fabric_words(&self) -> Result<[u32; 3]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + Ok([ + sgx.try_read32(G17_BRIDGE_CONTROL[0])?, + sgx.try_read32(G17_BRIDGE_CONTROL[1])?, + sgx.try_read32(G17_GATING_CONTROL)?, + ]) + } } /// Split the packed per-cluster core masks into one mask per cluster. diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 5f8348eb66329..2688654551442 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include "../hid-ids.h" @@ -86,6 +87,9 @@ struct dchid_init_hdr { #define CMD_REGISTER_RING 0x91 #define CMD_SEND_FIRMWARE 0x95 #define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_HOST_STATE 0xc1 +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 #define CMD_ACK_GPIO_CMD 0xa1 struct dchid_init_block_hdr { @@ -202,6 +206,8 @@ struct dockchannel_hid { bool inherited_mtp; /* Between the system sleep suspend and resume callbacks */ bool suspended; + /* The firmware was told of a sleep and has not been told of the wake */ + bool host_asleep; /* Held from probe on when the multi-touch interface uses power method 2 */ struct gpio_desc *afe_reset; @@ -467,6 +473,14 @@ static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); } +static int dchid_host_state(struct dockchannel_hid *dchid, u8 state) +{ + u8 msg[] = { CMD_HOST_STATE, state }; + int ret = dchid_comm_cmd(dchid, msg, sizeof(msg)); + + return ret < 0 ? ret : 0; +} + static int dchid_enable_interface(struct dchid_iface *iface) { u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; @@ -533,7 +547,7 @@ static int dchid_register_ring(struct dockchannel_hid *dchid) .size = DCHID_RING_SIZE, }; /* Observed in both macOS and standalone T8132 bringup before 0x91. */ - u8 prepare[] = { 0xc1, 0x02 }; + u8 prepare[] = { CMD_HOST_STATE, HOST_STATE_AWAKE }; int ret = 0; if (!dchid->use_ring) @@ -1789,10 +1803,13 @@ static void dockchannel_hid_remove(struct platform_device *pdev) } /* - * The coprocessor keeps running across suspend-to-idle: there is no known - * sleep command for it, and its RTKit session and the trackpad AFE power - * sequence are not restartable. Only mark the sleep window, in which input - * may wake the system and requests from the HID core fail fast. + * The coprocessor keeps running across suspend-to-idle, and its RTKit session + * and the trackpad AFE power sequence are not restartable. Mark the sleep + * window, in which input may wake the system and requests from the HID core + * fail fast. A firmware-driven suspend also sets the host state, after the idle + * notice the helper sends in its .prepare, and the wake sets it back after the + * helper's .resume has sent the on notice, the device link ordering the two. That + * is macOS's order. */ static int dockchannel_hid_suspend(struct device *dev) { @@ -1809,6 +1826,13 @@ static int dockchannel_hid_suspend(struct device *dev) if (ret) goto unwind; } + if (pm_suspend_via_firmware()) { + /* A firmware that doesn't answer isn't a reason to stay awake. */ + ret = dchid_host_state(dchid, HOST_STATE_SLEEP); + if (ret) + dev_warn(dev, "host state not set: %d\n", ret); + dchid->host_asleep = !ret; + } WRITE_ONCE(dchid->suspended, true); return 0; @@ -1828,6 +1852,14 @@ static int dockchannel_hid_resume(struct device *dev) struct dockchannel_hid *dchid = dev_get_drvdata(dev); int i, ret = 0; + if (dchid->host_asleep) { + dchid->host_asleep = false; + ret = dchid_host_state(dchid, HOST_STATE_AWAKE); + if (ret) + dev_warn(dev, "host state not restored: %d\n", ret); + ret = 0; + } + /* Allow feature requests again before notifying any HID driver. */ WRITE_ONCE(dchid->suspended, false); for (i = 0; i < MAX_INTERFACES; i++) { diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index adacda51cc2e5..0971ac3085c29 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -17,6 +17,7 @@ #include #include #include +#include #define LID_WAKE_SETTLE_MS_J716S 2000 @@ -31,22 +32,26 @@ MODULE_PARM_DESC(lid_wake_settle_ms, * @smc: Pointer to apple_smc struct of the mfd parent * @input: Allocated input_dev; devres managed * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) - * @lid_lock: Serialises lid reports with the system sleep callbacks - * @lid_settle_ms: Window after system sleep starts in which a lid-open report is held back - * @wakeup_mode: Set to true when system is suspended and power button events should wake it - * @lid_held: A lid-open report was not passed on during this system sleep - * @sleep_start: CLOCK_BOOTTIME at which this system sleep started + * @pm_nb: Notifier for the whole suspend window, including early aborts + * @lid_lock: Serialises lid reports, the notifier and device sleep callbacks + * @lid_settle_ms: Early lid-open confirmation window for affected boards + * @wakeup_mode: Power/lid events are wake events during system sleep + * @lid_held: A lid-open report was held until a full resume + * @sleep_start: CLOCK_BOOTTIME at the start of the current sleep attempt + * @dark: A dark wake must recheck the lid before sleeping again */ struct macsmc_input { struct device *dev; struct apple_smc *smc; struct input_dev *input; struct notifier_block nb; + struct notifier_block pm_nb; unsigned int lid_settle_ms; - struct mutex lid_lock; /* protects the three fields below */ + struct mutex lid_lock; /* protects wakeup_mode, lid_held, sleep_start and dark */ bool wakeup_mode; bool lid_held; ktime_t sleep_start; + bool dark; }; #define SMC_EV_BTN 0x7201 @@ -59,18 +64,19 @@ struct macsmc_input { static void macsmc_input_event_button(struct macsmc_input *smcin, unsigned long event) { + bool wakeup_mode = READ_ONCE(smcin->wakeup_mode); u8 button = (event >> 8) & 0xff; u8 state = !!(event & 0xff); switch (button) { case BTN_POWER: case BTN_TOUCHID: - pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && state)); + pm_wakeup_dev_event(smcin->dev, 0, (wakeup_mode && state)); /* * Suppress KEY_POWER reports when suspended to avoid powering down * immediately after waking from s2idle. * */ - if (smcin->wakeup_mode) + if (wakeup_mode) return; input_report_key(smcin->input, KEY_POWER, state); @@ -151,6 +157,10 @@ static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long eve } pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + /* Through a sleep, .complete reports where the lid ended up. */ + if (smcin->wakeup_mode) + return; + input_report_switch(smcin->input, SW_LID, lid_state); input_sync(smcin->input); } @@ -173,6 +183,44 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +/* + * A suspend's first steps run before .prepare, and the SMC reports the lid and the button as + * plain events until its AP idle notice. Treat them as wakes from the suspend's start, so a lid + * opened or a button pressed while the machine goes down ends the suspend, as on macOS. + */ +static int macsmc_input_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, pm_nb); + + guard(mutex)(&smcin->lid_lock); + if (action == PM_SUSPEND_PREPARE) { + smcin->sleep_start = ktime_get_boottime(); + smcin->lid_held = false; + WRITE_ONCE(smcin->wakeup_mode, true); + } else if (action == PM_POST_SUSPEND) { + u8 val; + + WRITE_ONCE(smcin->wakeup_mode, false); + smcin->dark = false; + smcin->lid_held = false; + /* A suspend that ended before .prepare held its lid events back too. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { + input_report_switch(smcin->input, SW_LID, val); + input_sync(smcin->input); + } + } + + return NOTIFY_DONE; +} + +static void macsmc_input_unregister_pm_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + unregister_pm_notifier(&smcin->pm_nb); +} + static void macsmc_input_unregister_notifier(void *data) { struct macsmc_input *smcin = data; @@ -275,41 +323,64 @@ static int macsmc_input_probe(struct platform_device *pdev) if (error) return error; - return 0; + smcin->pm_nb.notifier_call = macsmc_input_pm_notify; + error = register_pm_notifier(&smcin->pm_nb); + if (error) + return error; + + return devm_add_action_or_reset(&pdev->dev, macsmc_input_unregister_pm_notifier, smcin); } static int macsmc_input_pm_prepare(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); + u8 val; guard(mutex)(&smcin->lid_lock); smcin->sleep_start = ktime_get_boottime(); smcin->lid_held = false; - smcin->wakeup_mode = true; + WRITE_ONCE(smcin->wakeup_mode, true); + /* A lid opened as the dark wake ended may have sent its event already. */ + if (smcin->dark) { + smcin->dark = false; + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0 && !val) { + dev_info(dev, "the lid opened during a dark wake: not sleeping again\n"); + return -EBUSY; + } + } return 0; } static void macsmc_input_pm_complete(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); - u8 closed; + u8 lid; + bool held; int error; guard(mutex)(&smcin->lid_lock); - smcin->wakeup_mode = false; - if (!smcin->lid_held) + /* Keep the wake window and held lid event through an automatic dark re-sleep. */ + if (pm_dark_wake()) { + smcin->dark = apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &lid) < 0 || lid; return; + } + WRITE_ONCE(smcin->wakeup_mode, false); + held = smcin->lid_held; smcin->lid_held = false; - /* An open report was held back: report what the SMC says now. */ - error = apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &closed); - if (error < 0) { - dev_warn(dev, "Failed to read the lid state after sleep: %d\n", error); - return; + /* Full resume reports the actual state, including an open held for confirmation. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD))) { + error = apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &lid); + if (error < 0) { + dev_warn(dev, "Failed to read the lid state after sleep: %d\n", error); + return; + } + if (held) + dev_info(dev, "lid after sleep: MSLD=%u\n", lid); + input_report_switch(smcin->input, SW_LID, !!lid); + input_sync(smcin->input); } - dev_info(dev, "lid after sleep: MSLD=%u\n", closed); - input_report_switch(smcin->input, SW_LID, !!closed); - input_sync(smcin->input); } static const struct dev_pm_ops macsmc_input_pm_ops = { diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index e443755c12342..2c4350a0f29fe 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -307,6 +307,11 @@ struct apple_dart_fw_root { u64 *accepted; }; +struct apple_dart_sleep_list { + const u32 *pairs; + unsigned int count; +}; + /* * The subtrees behind the slots firmware holds in a four-level locked root * (fw_mirror), recorded when the root is mapped: per slot, the root entry and @@ -338,11 +343,29 @@ struct apple_dart_fw_tree { * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ +#define APPLE_DART_GAPF_PROTECT BIT(4) + struct apple_dart { struct device *dev; const struct apple_dart_hw *hw; void __iomem *regs; + void __iomem *gapf; + /* The GAPF slice as it read at probe; a deep sleep resets it to 0 */ + u32 gapf_boot; + bool sleep_flushed; + /* + * A display DART whose streams are off for a power-down: its registers may be + * unpowered, so maps use TCR as it was, and invalidations wait for the streams. + */ + bool display_dark; + bool display_tlb_stale; + DECLARE_BITMAP(display_tcr_four, DART_MAX_STREAMS); + /* The address filter the loader programs, from the device tree, replayed after a sleep. */ + void __iomem *dapf; + u64 *dapf_ranges; + u32 *dapf_r20; + unsigned int dapf_count; int irq; struct clk_bulk_data *clks; @@ -364,6 +387,10 @@ struct apple_dart { u32 per_stream : 1; u32 isp_host : 1; u32 power_retained : 1; + /* State a consumer saved for a firmware-driven suspend and will restore. */ + u32 sleep_saved : 1; + /* Its prepare list stopped the streams, and TLB commands wait for the restore. */ + u32 sleep_gated : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; /* @@ -378,10 +405,16 @@ struct apple_dart { u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; + /* Clocked by its host's port: the host resumes it after a real sleep. */ + u32 host_resumes : 1; + /* Reset and restored, its address filter written, at every power-up. */ + u32 restore_on_power_up : 1; bool commands_gated; /* Resume restored the translations, but the last steps of its reset failed. */ bool reset_incomplete; struct apple_tunable *tunables; + /* The bootloader sets these; written back where a power-off lost them. */ + struct apple_tunable *resume_tunables; u16 version; /* @@ -436,6 +469,14 @@ struct apple_dart { unsigned int pm_users; wait_queue_head_t pm_wq; u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ + /* The register writes around a firmware-driven suspend, from the device tree. */ + struct apple_dart_sleep_list sleep_prepare; + struct apple_dart_sleep_list sleep_restore; + struct apple_dart_sleep_list sleep_restore_late; + u32 sleep_enable[BITS_TO_U32(DART_MAX_STREAMS)]; + /* Its own config words at the sleep, which the restore keeps. */ + u32 sleep_cfg[20]; + /* Retained radio/display state left in place across suspend-to-idle. */ bool s2idle_kept; /* Resources reachable by a retained domain outlive device removal. */ @@ -669,21 +710,27 @@ static inline u32 apple_dart_readl(struct apple_dart *dart, u32 offset) return value; } -static void apple_dart_apply_tunables(struct apple_dart *dart) +/* Returns how many words it had to change. */ +static unsigned int apple_dart_apply_tunables(struct apple_dart *dart, + const struct apple_tunable *tunables) { + unsigned int changed = 0; size_t i; - for (i = 0; i < dart->tunables->sz; ++i) { + for (i = 0; i < tunables->sz; ++i) { u32 old_value, value; - old_value = apple_dart_readl(dart, - dart->tunables->values[i].offset); - value = old_value & ~dart->tunables->values[i].mask; - value |= dart->tunables->values[i].value; - if (value != old_value) + old_value = apple_dart_readl(dart, tunables->values[i].offset); + value = old_value & ~tunables->values[i].mask; + value |= tunables->values[i].value; + if (value != old_value) { apple_dart_writel(dart, value, - dart->tunables->values[i].offset); + tunables->values[i].offset); + changed++; + } } + + return changed; } /* @@ -2464,6 +2511,17 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) continue; + /* + * Its display is powered off: publish to memory only. With its streams off + * nothing walks the tables, and the TLB is invalidated before they're on. + */ + if (READ_ONCE(stream.dart->display_dark)) { + ret = stream.dart->locked ? + apple_dart_hw_sync_locked(cfg, &stream, true) : 0; + WRITE_ONCE(stream.dart->display_tlb_stale, true); + err = err ?: ret; + continue; + } ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { dev_err_ratelimited(stream.dart->dev, @@ -2594,7 +2652,10 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, } continue; } - four = apple_dart_readl(dart, DART_TCR(dart, sid)) & + if (READ_ONCE(dart->display_dark)) + four = test_bit(sid, dart->display_tcr_four); + else + four = apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level; idx = four ? 0 : addr >> (pgshift + 2 * bits); slot = (addr >> (pgshift + (four ? 2 : 1) * bits)) & @@ -4597,6 +4658,146 @@ static bool apple_dart_has_live_rtkit_client(struct device_node *np) return false; } +static void apple_dart_parse_sleep_list(struct apple_dart *dart, const struct resource *res, + const char *name, struct apple_dart_sleep_list *list) +{ + struct device_node *np = dart->dev->of_node; + unsigned int i; + u32 *pairs; + int count; + + count = of_property_count_u32_elems(np, name); + if (count <= 0) + return; + if (count % 2) { + dev_err(dart->dev, "%s: not a list of offset and value\n", name); + return; + } + pairs = devm_kcalloc(dart->dev, count, sizeof(*pairs), GFP_KERNEL); + if (!pairs || of_property_read_u32_array(np, name, pairs, count)) + return; + for (i = 0; i < count; i += 2) { + if (pairs[i] % 4 || pairs[i] + 4 > resource_size(res)) { + dev_err(dart->dev, "%s: bad offset %#x\n", name, pairs[i]); + return; + } + } + list->pairs = pairs; + list->count = count / 2; +} + +/* The config words a sleep list writes: 0x200 to 0x22c and 0x300 to 0x31c. */ +#define DART_SLEEP_CFG_WORDS 20 + +static unsigned int apple_dart_sleep_cfg_off(unsigned int i) +{ + return i < 12 ? 0x200 + 4 * i : 0x300 + 4 * (i - 12); +} + +static void apple_dart_sleep_cfg_read(struct apple_dart *dart, u32 *cfg) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + cfg[i] = readl(dart->regs + apple_dart_sleep_cfg_off(i)); +} + +/* + * The DART's address filter, which the loader programs and a real sleep clears: one entry + * per range, 0x40 apart, written in the order the loader writes them. + * apple,dapf-ranges gives start and end pairs from the ADT's dapf-instance-0, and the + * optional apple,dapf-r20 each entry's last word, 1 if absent. + */ +#define DART_DAPF_LOCK 0x208 +#define DART_DAPF_LOCKED BIT(2) +#define DART_DAPF_STRIDE 0x40 +#define DART_DAPF_CONFIG 0x31 + +static int apple_dart_dapf_parse(struct platform_device *pdev, struct apple_dart *dart) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf"); + int n = of_property_count_u64_elems(np, "apple,dapf-ranges"); + unsigned int i; + + /* A named DAPF window also exists on inherited cold-boot paths that + * provide no replay description. Keep their loader setup untouched. + * An explicit replay request or orphan metadata still requires data. + */ + if (!of_property_present(np, "apple,dapf-ranges")) { + if (of_property_present(np, "apple,dapf-r20") || + of_property_read_bool(np, "apple,restore-on-power-up")) + return dev_err_probe(dev, -EINVAL, + "DAPF replay requested without apple,dapf-ranges\n"); + return 0; + } + if (n <= 0 || n % 2) + return dev_err_probe(dev, -EINVAL, "apple,dapf-ranges needs start and end pairs\n"); + dart->dapf_count = n / 2; + if (resource_size(res) < dart->dapf_count * DART_DAPF_STRIDE) + return dev_err_probe(dev, -EINVAL, "DAPF region too small for %u ranges\n", + dart->dapf_count); + + dart->dapf = devm_platform_ioremap_resource_byname(pdev, "dapf"); + if (IS_ERR(dart->dapf)) + return PTR_ERR(dart->dapf); + dart->dapf_ranges = devm_kcalloc(dev, n, sizeof(*dart->dapf_ranges), GFP_KERNEL); + dart->dapf_r20 = devm_kcalloc(dev, dart->dapf_count, sizeof(*dart->dapf_r20), GFP_KERNEL); + if (!dart->dapf_ranges || !dart->dapf_r20) + return -ENOMEM; + of_property_read_u64_array(np, "apple,dapf-ranges", dart->dapf_ranges, n); + for (i = 0; i < dart->dapf_count; i++) + dart->dapf_r20[i] = 1; + if (of_property_count_u32_elems(np, "apple,dapf-r20") == dart->dapf_count) + of_property_read_u32_array(np, "apple,dapf-r20", dart->dapf_r20, dart->dapf_count); + + return 0; +} + +/* Entry 0 as written: the end reads back with its low two bits set, the last word's bytes. */ +static bool apple_dart_dapf_holds(struct apple_dart *dart) +{ + return readq(dart->dapf + 0x08) == dart->dapf_ranges[0] && + (readq(dart->dapf + 0x10) | 3) == (dart->dapf_ranges[1] | 3); +} + +/* + * After a power-off, before any consumer's firmware starts. Not while it's kept, unless + * @always, and not while it's locked. Each entry's config word goes last, after its + * streams, as macOS writes it. + */ +static void apple_dart_dapf_replay(struct apple_dart *dart, bool always) +{ + unsigned int i; + + if (!dart->dapf_count) + return; + if (!always && apple_dart_dapf_holds(dart)) + return; + if (readl(dart->regs + DART_DAPF_LOCK) & DART_DAPF_LOCKED) { + dev_warn(dart->dev, "DAPF locked after the sleep: not replayed\n"); + return; + } + + for (i = 0; i < dart->dapf_count; i++) { + void __iomem *entry = dart->dapf + i * DART_DAPF_STRIDE; + + writel(0, entry + 0x04); + writeq(dart->dapf_ranges[2 * i], entry + 0x08); + writeq(dart->dapf_ranges[2 * i + 1], entry + 0x10); + writel(dart->dapf_r20[i], entry + 0x20); + writel(DART_DAPF_CONFIG, entry + 0x00); + } + /* Its consumer's firmware faults on its first access without it: say so plainly. */ + if (!apple_dart_dapf_holds(dart)) { + dev_err(dart->dev, "DAPF write-back didn't take: entry 0 reads %#llx..%#llx\n", + readq(dart->dapf + 0x08), readq(dart->dapf + 0x10)); + return; + } + dev_dbg(dart->dev, "DAPF: %u ranges written back after the sleep\n", dart->dapf_count); +} + /* * The setup a T8110 DART needs when nothing initialised it before Linux: * apple,tunable (applied at every reset), apple,bypass-sids (streams kept in @@ -4881,6 +5082,19 @@ static int apple_dart_probe(struct platform_device *pdev) dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "gapf")) { + dart->gapf = devm_platform_ioremap_resource_byname(pdev, "gapf"); + if (IS_ERR(dart->gapf)) + return PTR_ERR(dart->gapf); + dart->gapf_boot = readl(dart->gapf); + } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf")) { + ret = apple_dart_dapf_parse(pdev, dart); + if (ret) + return ret; + } + dart->restore_on_power_up = of_property_read_bool(dev->of_node, + "apple,restore-on-power-up"); if (tunneled) { dart->tunables = devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable", res); @@ -4901,6 +5115,14 @@ static int apple_dart_probe(struct platform_device *pdev) } } } + dart->resume_tunables = devm_apple_tunable_parse(dev, dev->of_node, + "apple,resume-tunable", res); + if (IS_ERR(dart->resume_tunables)) { + if (PTR_ERR(dart->resume_tunables) != -ENOENT) + return dev_err_probe(dev, PTR_ERR(dart->resume_tunables), + "failed to parse resume tunables\n"); + dart->resume_tunables = NULL; + } if (j713_dcpext || j713_dispext) { /* @@ -4947,7 +5169,7 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled && dart->tunables) { dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", dart->tunables->sz); - apple_dart_apply_tunables(dart); + apple_dart_apply_tunables(dart, dart->tunables); } if (tunneled) { @@ -5144,6 +5366,10 @@ static int apple_dart_probe(struct platform_device *pdev) ret = -EBUSY; goto err_clk_disable; } + apple_dart_parse_sleep_list(dart, res, "apple,sleep-prepare", &dart->sleep_prepare); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore", &dart->sleep_restore); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore-late", + &dart->sleep_restore_late); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); /* @@ -5336,9 +5562,12 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_sysfs_remove; - /* Retain supplier references; generic suspend/resume is not applicable. */ - if (!dart->retained_mtp && !dart->managed_sids && !dart->cold_ane && - !apple_dart_has_borrowed(dart)) + /* + * Retain supplier references; generic suspend/resume is not applicable. + * Not for a cold ANE DART: ANE_SYS goes off while the ANE has no client, + * as on macOS, instead of staying on until the next system sleep. + */ + if (!dart->retained_mtp && !dart->managed_sids && !apple_dart_has_borrowed(dart)) pm_runtime_put(dev); dev_dbg( @@ -5656,6 +5885,183 @@ static void apple_dart_save_state(struct apple_dart *dart) } } +static void apple_dart_sleep_write(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, list->pairs[2 * i + 1], list->pairs[2 * i]); +} + +/* + * A sleep list's config words go back as this DART had them before the sleep, not as + * the list has them: Linux's own setup of those words can differ from macOS's. + */ +static u32 apple_dart_sleep_value(struct apple_dart *dart, u32 off, u32 val) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + if (apple_dart_sleep_cfg_off(i) == off) + return dart->sleep_cfg[i]; + return val; +} + +/* A sleep list, with its config words as the DART had them before the sleep. */ +static void apple_dart_sleep_write_kept(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, + apple_dart_sleep_value(dart, list->pairs[2 * i], + list->pairs[2 * i + 1]), + list->pairs[2 * i]); +} + +static void apple_dart_sleep_wait_tlb(struct apple_dart *dart) +{ + u32 status; + + if (readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT)) + dev_err(dart->dev, "TLB command still busy around a sleep: %#x\n", status); +} + +/* Whether a sleep list stops streams, which the restore then has to start again. */ +static bool apple_dart_sleep_list_stops_streams(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + if (dart->hw->type != DART_T8110) + return false; + for (i = 0; i < list->count; i++) + if (list->pairs[2 * i] >= DART_T8110_DISABLE_STREAMS && + list->pairs[2 * i] < DART_T8110_DISABLE_STREAMS + 0x20) + return true; + return false; +} + +/** + * apple_dart_sleep_listed() - Whether a DART's consumer saves and restores it for a sleep + * @dev: the DART + * + * Return: true if its node has apple,sleep-restore and the DART driver is bound. + */ +bool apple_dart_sleep_listed(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + return dart && dart->sleep_restore.count; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_listed); + +/** + * apple_dart_sleep_prepare() - Save a DART whose consumer is about to gate itself + * @dev: the DART + * + * For a DART with apple,sleep-restore in its node. A consumer calls this before + * telling its firmware to sleep, while the DART's state can be read, and + * apple_dart_sleep_restore() before starting that firmware again. The DART's own + * callbacks then leave it alone. Without the property nothing happens here, and + * they do what they always do. The state is saved before apple,sleep-prepare is + * written, so that list may stop the streams. + * + * Return: 0, or -ENODEV if the DART driver isn't bound. + */ +int apple_dart_sleep_prepare(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + if (!dart) + return -ENODEV; + if (!dart->sleep_restore.count) + return 0; + + apple_dart_sleep_cfg_read(dart, dart->sleep_cfg); + spin_lock_irqsave(&dart->lock, flags); + apple_dart_save_state(dart); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + dart->sleep_enable[i] = apple_dart_readl(dart, dart->hw->enable_streams + 4 * i); + apple_dart_sleep_write(dart, &dart->sleep_prepare); + apple_dart_sleep_wait_tlb(dart); + /* Flushed with no stream left to fill it: nothing may reach a DART whose power goes. */ + if (apple_dart_sleep_list_stops_streams(dart, &dart->sleep_prepare) && + !dart->commands_gated) { + WRITE_ONCE(dart->commands_gated, true); + dart->sleep_gated = true; + } + dart->sleep_saved = true; + spin_unlock_irqrestore(&dart->lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_prepare); + +/** + * apple_dart_sleep_restore_lost() - Put back a DART saved by apple_dart_sleep_prepare() + * @dev: the DART + * @lost: its power went off since, so its registers are at their reset values + * + * A DART that kept its registers only gets back what its prepare list stopped: the late + * list and the streams. + */ +void apple_dart_sleep_restore_lost(struct device *dev, bool lost) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx, i; + unsigned long flags; + + if (!dart || !dart->sleep_saved) + return; + + spin_lock_irqsave(&dart->lock, flags); + dart->sleep_saved = false; + if (lost) { + apple_dart_sleep_write_kept(dart, &dart->sleep_restore); + for (sid = 0; sid < dart->num_streams; sid++) { + if (!dart->save_tcr[sid] && !dart->save_ttbr[sid][0]) + continue; + apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + } + } + if (lost || dart->sleep_gated) { + apple_dart_sleep_write_kept(dart, &dart->sleep_restore_late); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, dart->sleep_enable[i], + dart->hw->enable_streams + 4 * i); + apple_dart_sleep_wait_tlb(dart); + } + if (dart->sleep_gated) { + dart->sleep_gated = false; + WRITE_ONCE(dart->commands_gated, false); + } + spin_unlock_irqrestore(&dart->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_restore_lost); + +/** + * apple_dart_sleep_restore() - The same, for a consumer whose DART loses its registers + * only in a real sleep + * @dev: the DART + * + * A suspend that was refused lost nothing, so nothing is written then. + */ +void apple_dart_sleep_restore(struct device *dev) +{ + apple_dart_sleep_restore_lost(dev, pm_resume_via_firmware()); +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_restore); + static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); @@ -5665,12 +6071,15 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) return -EBUSY; dart->tunnel_state_restored = false; + /* Its consumer saved this already and restores it: see apple_dart_sleep_prepare(). */ + if (dart->sleep_saved) + return 0; if (READ_ONCE(dart->commands_gated)) return 0; /* - * The cold ANE DARTs belong to the ANE backend, which refuses system - * sleep up front while it runs the firmware. Otherwise they hold - * nothing to keep: neither save nor reset them. + * The cold ANE DARTs belong to the ANE backend: it deactivates them + * before ANE_SYS goes off and attaches SID0 again once it is back on. + * Neither save nor reset them. */ if (dart->cold_ane) return 0; @@ -5800,7 +6209,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) int ret; /* No reset or restore: see apple_dart_suspend(). */ - if (dart->cold_ane || dart->live_mtp) + if (dart->cold_ane || dart->live_mtp || dart->sleep_saved) + return 0; + /* Its block can't answer yet: apple_dart_host_resume() restores it. */ + if (dart->host_resumes && READ_ONCE(dart->commands_gated)) return 0; /* Powered down by its DCP, which restores it at power-up. */ if (READ_ONCE(dart->host_off)) @@ -5840,7 +6252,13 @@ static __maybe_unused int apple_dart_resume(struct device *dev) if (dart->locked) return 0; - if (apple_dart_state_retained(dart)) { + /* + * Not for a DART restored at every power-up, as macOS's SPTM does (its + * t8110dart powerup): T8140's ISP DART keeps Linux's values through a wake + * that kept DRAM up, but its address filter takes effect only once written + * again, and the firmware's first access through it faults. + */ + if (!dart->restore_on_power_up && apple_dart_state_retained(dart)) { /* Nothing was lost, only drop what the TLBs may still hold. */ if (!dart->reset_incomplete) return apple_dart_invalidate_each_stream(dart); @@ -5877,18 +6295,128 @@ static __maybe_unused int apple_dart_resume(struct device *dev) ret = apple_dart_hw_finish_reset(dart); } + /* The address filter went with the same power-off, and so may these. */ + apple_dart_dapf_replay(dart, dart->restore_on_power_up); + if (dart->resume_tunables) { + unsigned int changed = apple_dart_apply_tunables(dart, dart->resume_tunables); + + if (changed) + dev_dbg(dev, "%u of %zu tunables written back\n", changed, + dart->resume_tunables->sz); + } + /* Should they fail again, the next resume repeats them. */ dart->reset_incomplete = ret; return ret; } -/* The adopted MTP stays live: system s2idle must retain, never reset it. */ +static __maybe_unused int apple_dart_resume_noirq(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + /* + * After a real sleep a DART clocked by its host's port reads nothing until the host + * sets the port up again. Gate it, so that nothing touches it, until then. + */ + if (dart->host_resumes && pm_resume_via_firmware()) { + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, true); + return pm_runtime_force_resume(dev); + } + + ret = pm_runtime_force_resume(dev); + + /* + * A real sleep took the registers even of a DART runtime PM had suspended, which + * force_resume leaves alone. Its consumer's firmware may DMA through it before + * anything resumes it at runtime, so write the saved state back now. Not one with its + * own sleep list, as the SEP's: its consumer restores that with macOS's writes before + * starting its firmware, and this would land on top. + */ + if (!ret && pm_resume_via_firmware() && pm_runtime_status_suspended(dev) && + !dart->sleep_restore.count) { + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after sleep failed: %d\n", ret); + else + dev_dbg(dev, "restored after sleep\n"); + } + if (pm_resume_via_firmware()) + apple_dart_dapf_replay(dart, false); + + return ret; +} + +/* The host will set this DART's port clock up again after a real sleep, and resume it then. */ +int apple_dart_set_host_resumes(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart) + return -EPROBE_DEFER; + if (dart->tunneled || dart->power_retained || dart->sleep_saved || dart->cold_ane) + return -EINVAL; + dart->host_resumes = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_set_host_resumes); + +/* The host's port is up again: ungate the DART and restore what it saved. */ +int apple_dart_host_resume(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + if (!dart || !dart->host_resumes) + return -EINVAL; + if (!READ_ONCE(dart->commands_gated)) + return 0; + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, false); + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after its host came back failed: %d\n", ret); + else + dev_info(dev, "restored after its host came back\n"); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_host_resume); + +/* + * Its consumer's own power cycle, outside a system sleep: the registers saved and put back. + * Only for a device apple-dart is bound to. + */ +int apple_dart_state_save(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_suspend(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_save); + +int apple_dart_state_restore(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_resume(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_restore); + +/* + * A firmware-driven sleep takes the DARTs in the noirq phase, which every device finishes + * before any early resume: coprocessors such as the PMP and SIO restart their firmware in + * their early resume, and its first DMA has to find the DART already restored. Any other + * sleep keeps the normal phase, which the PCIe-C tunnel DARTs need: their host stops and + * restarts the port clock in its own noirq callbacks. The adopted J613 MTP stays live: + * system s2idle must retain, never reset it. + */ static int apple_dart_system_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); if (!dart->live_mtp) - return pm_runtime_force_suspend(dev); + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_suspend(dev); if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; if (apple_dart_readl(dart, DART_TCR(dart, 1)) != dart->hw->tcr_bypass) @@ -5905,7 +6433,7 @@ static int apple_dart_system_resume(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); if (!dart->live_mtp) - return pm_runtime_force_resume(dev); + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_resume(dev); if (!dart->live_mtp_sleep) return 0; dart->live_mtp_sleep = false; @@ -5917,9 +6445,20 @@ static int apple_dart_system_resume(struct device *dev) return 0; } +static __maybe_unused int apple_dart_sys_suspend_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? pm_runtime_force_suspend(dev) : 0; +} + +static __maybe_unused int apple_dart_sys_resume_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? apple_dart_resume_noirq(dev) : 0; +} + static const struct dev_pm_ops apple_dart_pm_ops = { - RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) SYSTEM_SLEEP_PM_OPS(apple_dart_system_suspend, apple_dart_system_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend_noirq, apple_dart_sys_resume_noirq) + RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) }; static const struct of_device_id apple_dart_of_match[] = { @@ -6292,6 +6831,157 @@ int apple_dart_dcp_dispext_state(struct device_node *consumer) EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_state); #endif +/* + * Bit 4 of this DART's GAPF slice makes the fabric reject writes to its coprocessor's + * CPU power state. It's clear from the flush until the streams come back on. + */ +static int apple_dart_gapf_release(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf) + return 0; + + val = readl(dart->gapf); + writel(val & ~APPLE_DART_GAPF_PROTECT, dart->gapf); + if (readl(dart->gapf) & APPLE_DART_GAPF_PROTECT) { + dev_err(dart->dev, "GAPF slice stayed protected: %#x\n", val); + return -EIO; + } + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); + + return 0; +} + +/* Back to the probe value: after a wake the slice reads 0, and protect alone would give 0x10. */ +static void apple_dart_gapf_protect(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf || !dart->sleep_flushed) + return; + + val = readl(dart->gapf); + writel(val | dart->gapf_boot | APPLE_DART_GAPF_PROTECT, dart->gapf); + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); +} + +/* Before its coprocessor's CPU domain is written: a protected slice would fault the write. */ +int apple_dart_gapf_ready(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart->gapf || !(readl(dart->gapf) & APPLE_DART_GAPF_PROTECT)) + return 0; + + return apple_dart_gapf_release(dart); +} +EXPORT_SYMBOL_GPL(apple_dart_gapf_ready); + +/* + * What macOS writes to T8140's display DARTs around a display power cycle: + * measured under aura, with the offsets the driver has no name for. It never + * writes their TCR or TTBR, and reads the same values back after each power-on, + * deep sleep included, so neither does this. + * + * Stopping again after a sleep that never reached the commit does nothing: the + * domains may be off by then. + */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long bits = streams; + unsigned int sid; + u32 status; + int ret; + + if (on) { + apple_dart_gapf_protect(dart); + dart->sleep_flushed = false; + /* Maps or unmaps while it was dark left their invalidations here. */ + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); + writel(0, dart->regs + 0x210); + writel(0, dart->regs + 0x4060); + writel(streams, dart->regs + DART_T8110_ENABLE_STREAMS); + return 0; + } + + if (flush && dart->sleep_flushed) + return 0; + + if (!READ_ONCE(dart->display_dark)) { + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + } + + writel(U32_MAX, dart->regs + 0x4060); + writel(0, dart->regs + 0x4000); + writel(U32_MAX, dart->regs + 0x704); + for_each_set_bit(sid, &bits, BITS_PER_TYPE(streams)) + writel(BIT(sid), dart->regs + DART_T8110_DISABLE_STREAMS); + if (!flush) + return 0; + + writel(0xa000, dart->regs + DART_T8110_TLB_CMD); + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT); + if (ret) { + dev_err(dev, "flush for a power-down didn't finish: %#x\n", status); + return ret; + } + ret = apple_dart_gapf_release(dart); + if (ret) + return ret; + dart->sleep_flushed = true; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_display_streams); + +/* + * A DART whose domain goes off with its consumer's, at runtime, while other users still map + * into it: dark, maps reach memory only and their invalidations wait; lit, those run. Its + * streams are left as they are. Call it dark while it's still powered. + */ +void apple_dart_power_dark(struct device *dev, bool dark) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid; + + if (dark) { + if (READ_ONCE(dart->display_dark)) + return; + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + return; + } + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); +} +EXPORT_SYMBOL_GPL(apple_dart_power_dark); + static struct platform_driver apple_dart_driver = { .driver = { .name = "apple-dart", diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 1125008130ef4..393596ff6231b 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -281,6 +284,8 @@ struct aic_info { * comes online. */ bool siq_cfg_routing; + /* A firmware-driven suspend powers the controller down: aic_syscore_ops keep it. */ + bool fw_suspend; }; static const struct aic_info aic1_info __initconst = { @@ -333,6 +338,7 @@ static const struct aic_info aic3_t8140_info __initconst = { .fast_ipi = true, .local_fast_ipi = true, .siq_cfg_routing = true, + .fw_suspend = true, }; static const struct aic_info aic3_t8142_info __initconst = { @@ -388,6 +394,8 @@ struct aic_irq_chip { int nr_die; int max_die; + u32 *sleep_masks; + struct aic_info info; }; @@ -410,6 +418,47 @@ static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) writel_relaxed(val, ic->base + reg); } +/* + * A sleep that a firmware engine finishes waits for the CPUs to stop, and a raised + * line ends WFI even with interrupts off. Every line is masked for that last step. + */ +void aic_sleep_mask_all(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, reg, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++) { + reg = ic->info.mask_set + off + i * 4; + *saved++ = aic_ic_read(ic, reg); + aic_ic_write(ic, reg, U32_MAX); + } + } +} + +void aic_sleep_restore(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++, saved++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, *saved); + aic_ic_write(ic, ic->info.mask_clr + off + i * 4, ~*saved); + } + } +} + +/* Takes the next event, which also masks its line. */ +u32 aic_sleep_ack(void) +{ + return readl(aic_irqc->event + aic_irqc->info.event); +} + /* * IRQ irqchip */ @@ -1083,6 +1132,133 @@ static void build_fiq_affinity(struct aic_irq_chip *ic, struct device_node *aff) } } +#ifdef CONFIG_PM_SLEEP +/* + * A firmware-driven suspend powers the controller down and back up. How + * much it keeps isn't measured, and macOS saves and restores it in software. So + * mask everything before the commit, so that no line can end the final WFI, and + * put it all back after. Linux never writes IRQ_CFG or these global registers + * outside of config's enable bit, so they hold what the bootloader set. + */ +static struct { + u32 config; + u32 timeout; + u32 cluster_prio; + u32 *irq_cfg; + bool saved; +} aic_sleep; + +static int aic_syscore_suspend(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!pm_suspend_via_firmware()) + return 0; + + aic_sleep.config = aic_ic_read(ic, AIC2_CONFIG); + aic_sleep.timeout = aic_ic_read(ic, AIC2_TIMEOUT); + aic_sleep.cluster_prio = aic_ic_read(ic, AIC2_CLUSTER_PRIO); + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < ic->nr_irq; i++) + aic_sleep.irq_cfg[die * ic->nr_irq + i] = + aic_ic_read(ic, ic->info.irq_cfg + off + i * 4); + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + off += ic->info.die_stride; + } + aic_sleep.saved = true; + + return 0; +} + +/* The boot CPU doesn't pass through hotplug, so aic_init_cpu() doesn't rerun. */ +static void aic_resume_boot_cpu(void) +{ + u32 fiq = __this_cpu_read(aic_fiq_unmasked); + + aic_init_cpu(smp_processor_id()); + /* + * aic_init_cpu() is for a CPU with no FIQ users yet. This one's kept theirs: + * a timer left marked masked here would never be unmasked at its next EOI. + */ + __this_cpu_write(aic_fiq_unmasked, fiq); + if (is_kernel_in_hyp_mode()) { + if (fiq & BIT(AIC_TMR_EL02_PHYS)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_P, false); + if (fiq & BIT(AIC_TMR_EL02_VIRT)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V, false); + isb(); + } +} + +static void aic_syscore_resume(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!aic_sleep.saved) + return; + aic_sleep.saved = false; + + /* Otherwise the suspend was refused, and the controller stayed powered. */ + if (pm_resume_via_firmware()) { + aic_ic_write(ic, AIC2_TIMEOUT, aic_sleep.timeout); + aic_ic_write(ic, AIC2_CLUSTER_PRIO, aic_sleep.cluster_prio); + aic_ic_write(ic, AIC2_CONFIG, aic_sleep.config); + aic_resume_boot_cpu(); + } + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + aic_ic_write(ic, ic->info.sw_clr + off + i * 4, U32_MAX); + } + for (i = 0; i < ic->nr_irq; i++) { + u32 saved = aic_sleep.irq_cfg[die * ic->nr_irq + i]; + u32 reg = ic->info.irq_cfg + off + i * 4; + unsigned int virq; + + if (aic_ic_read(ic, reg) != saved) + aic_ic_write(ic, reg, saved); + + /* Restore from the IRQ core's books, not from the hardware's. */ + virq = irq_find_mapping(ic->hw_domain, AIC_IRQ_HWIRQ(die, i)); + if (virq && !irqd_irq_masked(irq_get_irq_data(virq))) + aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(i), + MASK_BIT(i)); + } + off += ic->info.die_stride; + } +} + +static const struct syscore_ops aic_syscore_ops = { + .suspend = aic_syscore_suspend, + .resume = aic_syscore_resume, +}; + +static struct syscore aic_syscore = { + .ops = &aic_syscore_ops, +}; + +static void __init aic_sleep_init(struct aic_irq_chip *irqc) +{ + if (!irqc->info.fw_suspend) + return; + + aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); + if (WARN_ON(!aic_sleep.irq_cfg)) + return; + + register_syscore(&aic_syscore); +} +#else +static void __init aic_sleep_init(struct aic_irq_chip *irqc) +{ +} +#endif + static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { int i, die; @@ -1168,6 +1344,11 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->info.die_stride = off - start_off; + irqc->sleep_masks = kcalloc(irqc->nr_die * (irqc->max_irq >> 5), + sizeof(*irqc->sleep_masks), GFP_KERNEL); + if (WARN_ON(!irqc->sleep_masks)) + goto err_unmap; + irqc->hw_domain = irq_domain_create_tree(of_fwnode_handle(node), &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) @@ -1219,6 +1400,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); + aic_sleep_init(irqc); + if (is_kernel_in_hyp_mode()) { struct irq_fwspec mi = { .fwnode = of_fwnode_handle(node), diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 72bcefdae4373..4cb1ae2800182 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -542,6 +542,61 @@ static void isp_test_command_ack_failures(struct kunit *test) KUNIT_EXPECT_EQ(test, state.calls, 0); } +/* The firmware's answer to the stop key request, as macOS reads it */ +static int isp_test_stop_key_reply(struct apple_isp *isp, + struct isp_channel *chan, + unsigned long timeout) +{ + struct kunit *test = kunit_get_current_test(); + unsigned int *calls = test->priv; + u8 *cmd = isp->cmd_virt; + + (*calls)++; + KUNIT_EXPECT_PTR_EQ(test, chan, isp->chan_dg); + KUNIT_EXPECT_EQ(test, chan->req.arg1, 0x20); + KUNIT_EXPECT_EQ(test, chan->req.arg2, 0x20); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(cmd + 4), 0x1d); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(cmd + 8), 1); + KUNIT_EXPECT_TRUE(test, timeout); + put_unaligned_le64(0x10001e00010ULL, cmd + 0x10); + put_unaligned_le64(0x8042006, cmd + 0x18); + chan->rsp = (struct isp_message){ .arg0 = chan->req.arg0 | ISP_IPC_FLAG_ACK, + .arg1 = chan->req.arg2 }; + return 0; +} + +static void isp_test_stop_key_on_debug(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct cmd_stop_key key; + unsigned int calls = 0; + + isp->cmd_virt = kunit_kzalloc(test, 0x400, GFP_KERNEL); + isp->chan_io = kunit_kzalloc(test, sizeof(*isp->chan_io), GFP_KERNEL); + isp->chan_dg = kunit_kzalloc(test, sizeof(*isp->chan_dg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_io); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_dg); + isp->chan_io->name = "IO"; + isp->chan_dg->name = "DEBUG"; + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_COMMAND; + test->priv = &calls; + kunit_deactivate_static_stub(test, isp_cmd_send); + kunit_activate_static_stub(test, ipc_chan_send, isp_test_stop_key_reply); + + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), 0); + KUNIT_EXPECT_EQ(test, calls, 1); + KUNIT_EXPECT_EQ(test, key.addr, 0x10001e00010ULL); + KUNIT_EXPECT_EQ(test, key.key, 0x8042006ULL); + + /* No DEBUG command ring: nothing is sent, on IO either. */ + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_REPLY; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + isp->chan_dg = NULL; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + KUNIT_EXPECT_EQ(test, calls, 1); +} + static void isp_test_call_crop(struct kunit *test) { struct isp_preset source = { @@ -658,6 +713,7 @@ static struct kunit_case isp_cam_test_cases[] = { KUNIT_CASE(isp_test_setfile_failure_stops_setup), KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failures), + KUNIT_CASE(isp_test_stop_key_on_debug), KUNIT_CASE(isp_test_sensor_geometry), KUNIT_CASE(isp_test_sensor_config_list), KUNIT_CASE(isp_test_sensor_config_failure), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index a653d31b3fe35..bbbb58d5fb954 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -565,11 +565,14 @@ int apple_isp_detect_camera(struct apple_isp *isp) err = isp_detect_camera(isp); - /* Resident firmware stays up; the channel is set up per stream. */ + /* + * Resident firmware stays up, the channel set up per stream, unless it starts + * again: then it's off until the first stream, as on macOS. + */ if (isp->hw->resident_fw) { - if (err) + if (err || isp->hw->fw_restart) apple_isp_firmware_halt(isp); - else + if (!err) isp->camera_detected = true; return err; } @@ -1080,7 +1083,7 @@ int apple_isp_start_capture(struct apple_isp *isp) void apple_isp_stop_capture(struct apple_isp *isp) { - if (isp->ipc_failed || isp->fw_state == ISP_FW_DEAD) + if (isp->ipc_failed || isp->fw_state != ISP_FW_RUNNING) return; if (!isp_cmd_ch_stop(isp, 0)) /* TODO channel mask */ isp_cmd_ch_buffer_return(isp, isp->current_ch); diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 24f77995fc3b7..a609fe2c34163 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -16,29 +16,30 @@ #define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) #define CISP_TIMEOUT msecs_to_jiffies(3000) +/* what macOS allows a DEBUG command */ +#define CISP_DEBUG_TIMEOUT msecs_to_jiffies(1000) #define CISP_SEND_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) #define CISP_SEND_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) #define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) #define CISP_POST_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, 0)) #define CISP_POST_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), 0)) -int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +/* Send through the command area on a command channel. */ +static int isp_cmd_send_chan(struct apple_isp *isp, struct isp_channel *chan, + void *args, u32 insize, u32 outsize, + unsigned long timeout) { - struct isp_channel *chan; struct isp_message *req; int err; + /* A failed session takes no more commands, on any channel. */ if (READ_ONCE(isp->ipc_error)) return READ_ONCE(isp->ipc_error); -#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) - KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); -#endif /* Before the command area is overwritten */ err = isp_ipc_resync(isp); if (err) return err; - chan = isp->chan_io; req = &chan->req; req->arg0 = isp->cmd_iova; @@ -72,6 +73,18 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int return err; } +int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ + if (READ_ONCE(isp->ipc_error)) + return READ_ONCE(isp->ipc_error); + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); +#endif + return isp_cmd_send_chan(isp, isp->chan_io, args, insize, outsize, + timeout); +} + static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, u32 outsize) { @@ -102,6 +115,16 @@ int isp_cmd_stop(struct apple_isp *isp, u32 mode) return CISP_SEND_IN(isp, args); } +/* T8140's cold stop, which macOS sends as a command with its reply. */ +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + int isp_cmd_power_down(struct apple_isp *isp) { struct cmd_power_down args = { @@ -136,6 +159,30 @@ int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) return CISP_SEND_INOUT(isp, args); } +/* + * macOS asks for the key on the DEBUG channel. On IO, 0x1d is another + * command, and the firmware answers neither it nor the STOP after it. + */ +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args) +{ + struct isp_channel *chan = isp->chan_dg; + int err; + + if (!chan || chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + return -ENODEV; + + memset(args, 0, sizeof(*args)); + args->opcode = CISP_OPCODE(CISP_DEBUG_CMD_STOP_KEY_GET); + args->enable = 1; + err = isp_cmd_send_chan(isp, chan, args, sizeof(*args), sizeof(*args), + CISP_DEBUG_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, sizeof(*args)); + return 0; +} + int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) { args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 9e2678bd1610c..ab98f3447d6f7 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -101,6 +101,9 @@ #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD 0xc11d #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH 0xc120 +/* Opcodes of the DEBUG channel, which has its own numbering */ +#define CISP_DEBUG_CMD_STOP_KEY_GET 0x001d + #define CISP_POOL_TYPE_META 0x0 #define CISP_POOL_TYPE_RENDERED 0x1 #define CISP_POOL_TYPE_FD 0x2 @@ -159,6 +162,19 @@ struct cmd_print_enable { } __packed; static_assert(sizeof(struct cmd_print_enable) == 0xc); +/* + * CISP_DEBUG_CMD_STOP_KEY_GET: the firmware answers with a word of its + * memory and the value it writes there once it has stopped. + */ +struct cmd_stop_key { + u64 opcode; + u32 enable; + u32 pad_c; + u64 addr; + u64 key; +} __packed; +static_assert(sizeof(struct cmd_stop_key) == 0x20); + struct cmd_trace_enable { u64 opcode; u32 enable; @@ -263,10 +279,12 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout); int isp_cmd_start(struct apple_isp *isp, u32 mode); int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode); int isp_cmd_power_down(struct apple_isp *isp); int isp_cmd_suspend(struct apple_isp *isp); int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args); int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, diff --git a/drivers/media/platform/apple/isp/isp-drv-test.c b/drivers/media/platform/apple/isp/isp-drv-test.c index a1e60e5e29d15..2ff43d1c7ae26 100644 --- a/drivers/media/platform/apple/isp/isp-drv-test.c +++ b/drivers/media/platform/apple/isp/isp-drv-test.c @@ -113,8 +113,8 @@ kunit_test_suite(isp_dapf_test_suite); /* * A deferred start detects the camera once: its firmware must keep running - * or start again. Firmware that starts again is not resident, and only it - * is quiesced before the power-down or idles after a stream. + * or start again. Only firmware that starts again, from a restored copy or + * from its own memory, is quiesced before the power-down. */ static void isp_test_firmware_lifetime(struct kunit *test) { @@ -129,11 +129,11 @@ static void isp_test_firmware_lifetime(struct kunit *test) KUNIT_EXPECT_FALSE_MSG(test, hw->resident_fw && hw->fw_restore, "%s", match->compatible); if (hw->coproc_quiesce) - KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore, "%s", + KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore || hw->fw_restart, "%s", match->compatible); /* Only firmware that starts again idles before it stops. */ if (hw->fw_idle_ms) - KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore, "%s", + KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore || hw->fw_restart, "%s", match->compatible); /* The fallback keeps restartable firmware resident. */ if (hw->resident_fallback) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 79d9e3bb7651b..a1a79ad97b4a7 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "isp-cam.h" @@ -623,6 +624,32 @@ isp_check_firmware_version(struct apple_isp *isp) return compat; } +/* After a system sleep, see apple_isp_resume(). */ +static void apple_isp_resume_work(struct work_struct *work) +{ + struct apple_isp *isp = container_of(work, struct apple_isp, resume_work); + + mutex_lock(&isp->video_lock); + isp->fw_sleep_stopped = false; + mutex_unlock(&isp->video_lock); + apple_isp_video_resume(isp); +} + +/* + * Idle firmware that starts again from its own memory is stopped before a + * system sleep begins, as macOS's camera service powers an idle ISP off + * before the kernel sleeps: the sleep then finds it off, as after a stream. + */ +static int apple_isp_pm_notify(struct notifier_block *nb, unsigned long event, + void *unused) +{ + struct apple_isp *isp = container_of(nb, struct apple_isp, pm_nb); + + if (event == PM_SUSPEND_PREPARE || event == PM_HIBERNATION_PREPARE) + apple_isp_firmware_flush(isp); + return NOTIFY_DONE; +} + static int apple_isp_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -747,6 +774,7 @@ static int apple_isp_probe(struct platform_device *pdev) } mutex_init(&isp->video_lock); + INIT_WORK(&isp->resume_work, apple_isp_resume_work); apple_isp_firmware_init(isp); spin_lock_init(&isp->buf_lock); init_waitqueue_head(&isp->wait); @@ -800,6 +828,11 @@ static int apple_isp_probe(struct platform_device *pdev) goto halt_firmware; } + if (isp->hw->fw_idle_ms && isp->hw->resident_fw) { + isp->pm_nb.notifier_call = apple_isp_pm_notify; + register_pm_notifier(&isp->pm_nb); + } + return 0; halt_firmware: @@ -824,6 +857,9 @@ static void apple_isp_remove(struct platform_device *pdev) { struct apple_isp *isp = platform_get_drvdata(pdev); + if (isp->pm_nb.notifier_call) + unregister_pm_notifier(&isp->pm_nb); + cancel_work_sync(&isp->resume_work); apple_isp_remove_video(isp); /* Stop resident firmware before freeing what it may still use. */ apple_isp_firmware_flush(isp); @@ -1024,31 +1060,6 @@ static const struct isp_mmio_window apple_isp_fw_mmio_t8140[] = { { 0x401660000, 0x4000 }, }; -/* - * The windows the boot loader programs into the first ISP DART's address - * filter, in its order, as they read back on the J700. The DART is in the - * ISP system domain, so they are programmed again after each power-up. - */ -static const struct isp_dapf_window apple_isp_dapf_t8140[] = { - { 0x300704000, 0x300704063 }, /* PMGR, ISP power states */ - { 0x300730000, 0x300733fff }, - { 0x300e3c000, 0x300e3c003 }, - { 0x31062c000, 0x310648003 }, - { 0x401660000, 0x401660003 }, - { 0x220004000, 0x2200162ff }, /* DSID broadcast-clear windows */ - { 0x220044000, 0x2200562ff }, - { 0x220084000, 0x2200962ff }, - { 0x2200c4000, 0x2200d62ff }, - { 0x220104000, 0x2201162ff }, - { 0x302824000, 0x302824007 }, /* PMP bandwidth scratch */ - { 0x3003c0000, 0x3003e4003 }, /* PMGR, with the PMP clock scratch */ -}; - -/* - * The firmware starts again from its data segment as loaded once the - * coprocessor was quiesced before the power-down, as T8132's does, so the - * ISP is powered only while the camera is in use. - */ static const struct apple_isp_hw apple_isp_hw_t8140 = { .gen = ISP_GEN_T8140, .fw_abi = ISP_FW_ABI_H17, @@ -1076,18 +1087,23 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .mbox2_offset = 0x410, .mbox_irq_route = true, .coproc_control = ISP_COPROC_CONTROL_T8140, + /* + * macOS quiesces the coprocessor before the core words go off + * (DisableISPClocksAndPower, 25G83); without it they reach off with BUSY + * latched. + */ .coproc_status = ISP_COPROC_STATUS_T8140, .coproc_quiesce = ISP_COPROC_QUIESCE_T8140, .fw_mmio = apple_isp_fw_mmio_t8140, .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8140), - .dapf = apple_isp_dapf_t8140, - .num_dapf = ARRAY_SIZE(apple_isp_dapf_t8140), .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, - .fw_restore = true, + .resident_fw = true, + .fw_restart = true, + /* As on macOS, the firmware first starts when a client opens the camera. */ .deferred_start = true, - .resident_fallback = true, - .fw_idle_ms = 2000, + /* macOS powers it off 5 s after the last frame (twice in its recording). */ + .fw_idle_ms = 5000, .meta_exposure = true, }; @@ -1378,9 +1394,43 @@ static __maybe_unused int apple_isp_runtime_resume(struct device *dev) return 0; } +/* + * Stop resident firmware. For a system sleep it starts again on resume where + * it can; otherwise, or for hibernation, the camera is gone until the next + * boot. + */ +static void apple_isp_stop_resident(struct apple_isp *isp, bool sleep) +{ + bool restart = sleep && isp->hw->fw_restart; + + if (!isp->hw->resident_fw) + return; + + mutex_lock(&isp->video_lock); + if (isp->fw_state == ISP_FW_RUNNING) { + if (restart) + dev_dbg(isp->dev, "stopping the firmware for system sleep\n"); + else + dev_warn(isp->dev, "stopping the firmware for %s, the camera is unavailable until the next boot\n", + sleep ? "system sleep" : "hibernation"); + isp->fw_sleep_stopped = restart; + } + apple_isp_firmware_halt(isp); + /* Hibernation resumes in another boot, where that memory isn't ours. */ + if (!restart) + isp->fw_state = ISP_FW_DEAD; + mutex_unlock(&isp->video_lock); +} + static __maybe_unused int apple_isp_suspend(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + bool power_lost = isp->hw->resident_fw && pm_suspend_via_firmware(); + + /* A stream that couldn't restart on the other side keeps the system up. */ + if (power_lost && !isp->hw->fw_restart && + test_bit(ISP_STATE_STREAMING, &isp->state)) + return -EBUSY; /* We must restore V4L2 context on system resume. If we were streaming * before, we (essentially) stop streaming and start streaming again. @@ -1388,10 +1438,13 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) apple_isp_video_suspend(isp); /* - * Resident firmware stays up, idle, across suspend-to-idle and - * suspend-to-RAM: its domains stay on, see apple_isp_attach_genpd(). - * Firmware that starts again and still idles after a stream stops. + * Resident firmware stays up, idle, across suspend-to-idle: its domains + * stay on, see apple_isp_attach_genpd(). A sleep that powers the SoC down + * takes it anyway, so stop it first. Firmware that starts again and still + * idles after a stream stops. */ + if (power_lost) + apple_isp_stop_resident(isp, true); apple_isp_firmware_flush(isp); return 0; } @@ -1409,13 +1462,7 @@ static __maybe_unused int apple_isp_freeze(struct device *dev) * that started it. Stop it cleanly instead; if it ran, the camera is * then unavailable until the next system boot. */ - if (isp->hw->resident_fw) { - mutex_lock(&isp->video_lock); - if (isp->fw_state == ISP_FW_RUNNING) - dev_warn(dev, "stopping the firmware for hibernation, the camera is unavailable until the next boot\n"); - apple_isp_firmware_halt(isp); - mutex_unlock(&isp->video_lock); - } + apple_isp_stop_resident(isp, false); return 0; } @@ -1424,6 +1471,16 @@ static __maybe_unused int apple_isp_resume(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + /* + * Firmware stopped for the sleep starts again in process context: its + * handshake here held up the whole resume. A stream that ran across the + * sleep restarts from a freezable work item, once tasks thaw; without one + * the firmware starts at the next stream. + */ + if (isp->fw_sleep_stopped) { + queue_work(system_freezable_wq, &isp->resume_work); + return 0; + } apple_isp_video_resume(isp); return 0; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 1a62f0533faa5..fda0e754e7110 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -242,12 +243,15 @@ struct apple_isp_hw { bool fw_restore; /* - * The firmware can be started only once per system boot, so once - * started it is kept running while the driver is bound. It starts at - * probe, or at the first stream with deferred_start. + * The firmware isn't started for each stream: once started it is kept + * running while the driver is bound. It starts at probe, or at the + * first stream with deferred_start. */ bool resident_fw; + /* Resident firmware that starts again after a stop, from its own memory */ + bool fw_restart; + /* * Start the firmware, and detect the camera, at the first stream * instead of at probe, so the ISP stays off until then. The firmware @@ -408,6 +412,10 @@ struct apple_isp { u64 heap_base; u64 heap_top; u64 data_top; /* 0 unless the bootloader loads the data */ + /* the coprocessor's reset vector at the first start */ + u64 entry; + /* the coprocessor has run since probe */ + bool started; } fw; /* changes under video_lock once the video device is registered */ enum isp_fw_state fw_state; @@ -423,6 +431,14 @@ struct apple_isp { /* configurations the firmware has, once a deferred start detected them */ u32 num_fw_presets; + /* stops idle resident firmware before a system sleep */ + struct notifier_block pm_nb; + + /* stopped for system sleep, to start again on resume */ + bool fw_sleep_stopped; + /* restarts a stream that ran across that sleep, see apple_isp_resume() */ + struct work_struct resume_work; + struct isp_surf *ipc_surf; struct isp_surf *extra_surf; struct isp_surf *data_surf; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 2bea18d71dfa7..2527f712839e0 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -7,9 +7,13 @@ #include #include #include +#include +#include #include +#include #include #include +#include #include #include @@ -29,6 +33,12 @@ #define ISP_COPROC_IN_WFI 0x3 +/* the waits for firmware that has been told to power down */ +#define ISP_STOP_KEY_TRIES 20 +#define ISP_STOP_KEY_MS 10 +#define ISP_STOP_WFI_POLL_US (10 * USEC_PER_MSEC) +#define ISP_STOP_WFI_US (190 * USEC_PER_MSEC) + static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) { return readl(isp->coproc + reg); @@ -262,9 +272,12 @@ static void isp_disable_irq(struct apple_isp *isp) if (isp->hw->mbox_irq_route || isp->hw->mbox_extra_enables) { isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, 0x0); isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, 0x0); + if (isp->hw->mbox_irq_route) + isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, 0x0); } free_irq(isp->irq, isp); isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ + fsleep(10); } static int isp_enable_irq(struct apple_isp *isp) @@ -426,6 +439,39 @@ static void isp_restore_fw_memory(struct apple_isp *isp) clear_highpage(tail->pages[i]); } +/* + * The coprocessor starts at its reset vector, which the bootloader sets and + * which bit 0 latches. Note it at the first start; where a power-off cleared + * it, set it again. + */ +static int isp_coproc_entry(struct apple_isp *isp) +{ + struct isp_firmware *fw = &isp->fw; + u64 rvbar = readq(isp->coproc + ISP_COPROC_RVBAR); + + if (!fw->started) { + fw->entry = rvbar & ~1ULL; + dev_dbg(isp->dev, "firmware reset vector %#llx\n", rvbar); + return 0; + } + + if (!fw->entry || rvbar == (fw->entry | 1)) + return 0; + + if (rvbar & 1) { + isp_err(isp, "reset vector %#llx latched, not the firmware's %#llx\n", + rvbar, fw->entry); + return -EIO; + } + + writeq(fw->entry | 1, isp->coproc + ISP_COPROC_RVBAR); + dev_dbg(isp->dev, "reset vector was %#llx, set to %#llx\n", rvbar, + readq(isp->coproc + ISP_COPROC_RVBAR)); + + return (readq(isp->coproc + ISP_COPROC_RVBAR) & ~1ULL) == fw->entry ? + 0 : -EIO; +} + static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { /* Only a coprocessor that was started has to settle and quiesce */ @@ -440,7 +486,13 @@ static void isp_firmware_shutdown_stage1(struct apple_isp *isp) dev_warn(isp->dev, "coproc not in WFI before power-down (status 0x%x)\n", val); - isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); + /* + * macOS powers T8140's ISP off from WFI with the run bit still set. The sleep + * engine's own restore of the ISP in a wake that kept DRAM up depends on it: + * with the bit cleared that wake stalls until the SoC's watchdog resets it. + */ + if (isp->hw->gen != ISP_GEN_T8140) + isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); if (started) { /* The request bit alone asks for it, and reads back once done */ @@ -488,6 +540,12 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) if (err < 0) goto shutdown; + if (isp->hw->fw_restart) { + err = isp_coproc_entry(isp); + if (err < 0) + goto shutdown; + } + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); isp_gpio_write32(isp, ISP_GPIO_1, 0x0); isp_gpio_write32(isp, ISP_GPIO_2, 0x0); @@ -501,6 +559,7 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); isp_coproc_write32(isp, isp_coproc_control(isp), 0x10); + isp->fw.started = true; /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { @@ -898,6 +957,85 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) return 0; } +/* Wait for the word of its own memory the firmware sets once it has stopped. */ +static void isp_wait_stop_key(struct apple_isp *isp, u64 addr, u32 key) +{ + u64 iova = isp_fw_iova(isp, addr); + phys_addr_t phys = 0; + u32 *word = NULL; + + if (iova < isp_fw_iova(isp, isp->fw.heap_top)) + phys = iommu_iova_to_phys(isp->domain, iova); + if (phys) + word = memremap(phys, sizeof(*word), MEMREMAP_WC); + if (!word) { + dev_warn(isp->dev, "stop key at 0x%llx isn't in firmware memory\n", + addr); + return; + } + + for (int i = 0; i < ISP_STOP_KEY_TRIES && READ_ONCE(*word) != key; i++) + msleep(ISP_STOP_KEY_MS); + + if (READ_ONCE(*word) != key) + dev_warn(isp->dev, "stop key at 0x%llx reads 0x%x, not 0x%x\n", + addr, READ_ONCE(*word), key); + + memunmap(word); +} + +/* + * Stop firmware that starts cold next time. CISP_CMD_SUSPEND is for firmware + * that resumes: it isn't followed by a cold start. + */ +static int isp_stop_firmware_cold(struct apple_isp *isp) +{ + struct cmd_stop_key stop_key = {}; + ktime_t start = ktime_get(); + u32 status; + int err; + + /* Without the key the rest of the stop still works. */ + if (isp_cmd_stop_key_get(isp, &stop_key)) + stop_key.addr = 0; + + err = isp_cmd_stop_cold(isp, 0); + if (err) { + isp_err(isp, "isp_cmd_stop_cold() failed\n"); + return err; + } + + /* The firmware answers the power-down in this word, not on the ring. */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + err = isp_cmd_power_down(isp); + if (err) { + isp_err(isp, "isp_cmd_power_down() failed\n"); + return err; + } + + if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + if (stop_key.addr) + isp_wait_stop_key(isp, stop_key.addr, stop_key.key); + + /* Its domains go off next, so its CPU should be idle by now. */ + if (readl_poll_timeout(isp->coproc + ISP_COPROC_STATUS_T8140, status, + status & ISP_COPROC_IN_WFI, ISP_STOP_WFI_POLL_US, + ISP_STOP_WFI_US)) + dev_warn(isp->dev, "coprocessor not in WFI (status: 0x%x)\n", + status); + else + dev_dbg(isp->dev, "firmware stopped in %lld ms (key at 0x%llx, status 0x%x)\n", + ktime_ms_delta(ktime_get(), start), stop_key.addr, + status); + + return 0; +} + static int isp_stop_command_processor(struct apple_isp *isp) { int res, ret; @@ -905,6 +1043,9 @@ static int isp_stop_command_processor(struct apple_isp *isp) if (READ_ONCE(isp->ipc_error)) return READ_ONCE(isp->ipc_error); + if (isp->hw->fw_restart) + return isp_stop_firmware_cold(isp); + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); /* @@ -1106,28 +1247,50 @@ static void isp_firmware_shutdown(struct apple_isp *isp) /* * Stop the coprocessor before releasing any memory the firmware - * uses, also when it did not acknowledge the suspend request. + * uses, also when it did not acknowledge the stop request. */ if (isp_stop_command_processor(isp)) - dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); + dev_warn(isp->dev, "firmware did not stop, stopping the coprocessor\n"); /* Also stop a receive thread whose producer never leaves a gap. */ cmpxchg(&isp->ipc_error, 0, -ESHUTDOWN); isp_disable_irq(isp); isp_firmware_stop(isp); } +/* + * After a restart, the firmware must find the camera found at probe. The + * query also shows that it takes commands. + */ +static int isp_check_restarted(struct apple_isp *isp) +{ + struct cmd_config_get args = {}; + int err; + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + if (min_t(u32, args.num_channels, ISP_MAX_CHANNELS) != isp->num_channels) { + isp_err(isp, "firmware found %u channels after a restart, %d before\n", + args.num_channels, isp->num_channels); + return -ENODEV; + } + + return 0; +} + /* * Resident firmware (hw->resident_fw) is loaded by the bootloader and - * does not start again once it has been stopped: after CISP_CMD_SUSPEND, - * a coprocessor reset and a power cycle of its domains it never completes - * the first handshake. It is booted once, at probe or with - * deferred_start at the first stream, and runs until the - * driver is unbound, probe fails or the system hibernates, also across - * suspend; after it has stopped the device stays unusable until the next - * system boot. + * changes its own memory as it runs. It is booted at probe, or with + * deferred_start at the first stream, and runs until the driver is unbound, + * probe fails, the system hibernates or a sleep powers the SoC down; + * suspend-to-idle keeps it running. With hw->fw_restart it starts again, and + * rebuilds that memory itself; other stopped firmware stays down until the + * next system boot. */ int apple_isp_firmware_boot(struct apple_isp *isp) { + bool restart = isp->hw->resident_fw && isp->fw.started; int err; switch (isp->fw_state) { @@ -1158,10 +1321,16 @@ int apple_isp_firmware_boot(struct apple_isp *isp) } err = isp_firmware_boot(isp); + if (!err && restart) { + err = isp_check_restarted(isp); + if (err) + isp_firmware_shutdown(isp); + } if (err) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); pm_runtime_put_sync(isp->dev); - if (isp->hw->resident_fw) + /* Firmware that starts again can be tried again at the next open. */ + if (isp->hw->resident_fw && !isp->hw->fw_restart) isp->fw_state = ISP_FW_DEAD; return err; } @@ -1202,6 +1371,16 @@ void apple_isp_firmware_shutdown(struct apple_isp *isp) WRITE_ONCE(isp->ipc_quiescing, false); enable_irq(isp->irq); + /* + * Firmware that starts again goes down after the stream, at once or, with + * hw->fw_idle_ms, once it has idled that long, as macOS's camera service + * powers it off. A system sleep finds it off either way: idle firmware is + * stopped before the sleep begins (apple_isp_pm_notify()). The DRAM-kept + * wake runs the ISP itself and needs it left that way. + */ + if (isp->hw->fw_restart && !isp->hw->fw_idle_ms) + apple_isp_firmware_halt(isp); + /* Not on isp->wq: the stop flushes that. */ if (isp->hw->fw_idle_ms && isp->fw_state == ISP_FW_RUNNING) { isp->fw_idle = true; @@ -1239,7 +1418,7 @@ void apple_isp_firmware_flush(struct apple_isp *isp) mutex_unlock(&isp->video_lock); } -/* Stop the firmware; resident firmware cannot be started again. */ +/* Stop the firmware; resident firmware without hw->fw_restart stays down. */ void apple_isp_firmware_halt(struct apple_isp *isp) { /* Only a capture services the watchdog, but be sure. */ @@ -1252,5 +1431,6 @@ void apple_isp_firmware_halt(struct apple_isp *isp) pm_runtime_put_sync(isp->dev); isp->ipc_stale = NULL; isp->fw_idle = false; - isp->fw_state = isp->hw->resident_fw ? ISP_FW_DEAD : ISP_FW_OFF; + isp->fw_state = isp->hw->resident_fw && !isp->hw->fw_restart ? + ISP_FW_DEAD : ISP_FW_OFF; } diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 9bf93346a3722..41e63a487c3b4 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -646,6 +646,8 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) { struct apple_isp *isp = vb2_get_drv_priv(q); + /* A restart still due from a sleep has nothing left to restart. */ + clear_bit(ISP_STATE_SLEEPING, &isp->state); apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); v4l2_ctrl_grab(isp->exposure_priority, false); diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index c4f9329dbc01c..d6ccf8a3970a2 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -9,8 +9,16 @@ struct macsmc_test_context { int result; unsigned int calls; bool require_sram_lock; + /* What the sleep tests saw, in order: keys read, values written, notices. */ + u32 seen[8]; + unsigned int seen_count; + int ap_power_result; + u64 record; }; +#define MACSMC_TEST_AP_IDLE 0 +#define MACSMC_TEST_AP_ON 1 + static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -18,6 +26,9 @@ static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); ctx->calls++; + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = cmd == SMC_MSG_WRITE_KEY ? + get_unaligned((u32 *)ctx->shmem) : arg; if (ret_data && ctx->result >= 0) *ret_data = ctx->data; return ctx->result; @@ -199,6 +210,165 @@ static void macsmc_key_info_lock_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); } +static int macsmc_test_ap_power(struct apple_smc *smc, bool on) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + /* The answer needs a worker that may be waiting for this mutex. */ + KUNIT_EXPECT_FALSE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = on ? MACSMC_TEST_AP_ON : MACSMC_TEST_AP_IDLE; + return ctx->ap_power_result; +} + +static void macsmc_sleep_order_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + apple_smc_sleep_warn(&ctx->smc); + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + + /* As in the trace, values included. */ + KUNIT_ASSERT_EQ(test, ctx->seen_count, 7U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[5], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[6], SMC_KEY(waka)); +} + +static void macsmc_test_outbox_poll(struct apple_smc *smc, const char *after, + unsigned int min_ms, unsigned int quiet_ms) +{ +} + +static void macsmc_sleep_warn_late_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); + ctx->result = 8; + + /* A sleep that cuts the debug link: slpw comes at the noirq suspend, before slpa. */ + ctx->smc.sleep_warn_late = true; + KUNIT_EXPECT_EQ(test, apple_smc_sleep_enter(&ctx->smc), 0); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); +} + +static void macsmc_sleep_refused_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->ap_power_result = -ETIME; + ctx->result = 4; + + /* No answer: refuse, undo the notice and leave commands working. */ + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), -ETIME); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_ON); +} + +static void macsmc_notice_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + /* The test trigger writes no MBSE value: those announce a real sleep. */ + KUNIT_EXPECT_EQ(test, apple_smc_notice_idle(&ctx->smc), 0); + apple_smc_notice_on(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(MBSW)); +} + +/* MBSW reads ctx->record, MSLD reads closed. */ +static int macsmc_test_lid_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = key; + memset(buf, 0, size); + if (key == SMC_KEY(MBSW) && size == sizeof(ctx->record)) + put_unaligned_le64(ctx->record, buf); + else if (key == SMC_KEY(MSLD)) + *(u8 *)buf = 1; + return size; +} + +static void macsmc_lid_wake_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_read, macsmc_test_lid_read); + ctx->result = 4; + + /* The lid alone: MSLD is read last, after macOS's own order. */ + ctx->record = BIT_ULL(10); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 5U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MSLD)); + + /* The lid with anything else is a normal wake: the lid isn't read. */ + ctx->seen_count = 0; + ctx->record = BIT_ULL(10) | BIT_ULL(0); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 4U); +} + +static void macsmc_wake_reason_test(struct kunit *test) +{ + char names[128]; + + KUNIT_EXPECT_EQ(test, apple_smc_wake_reason_names(0, names, sizeof(names)), 0); + KUNIT_EXPECT_STREQ(test, names, ""); + + /* A bit with no name is left out. */ + apple_smc_wake_reason_names(BIT_ULL(23) | BIT_ULL(42) | BIT_ULL(1), names, sizeof(names)); + KUNIT_EXPECT_STREQ(test, names, "wifibt USB-C_plug"); +} + +static void macsmc_port_reasons_test(struct kunit *test) +{ + /* The debug link's cut: USB-C_plug, acdetach, or both. */ + KUNIT_EXPECT_TRUE(test, apple_smc_port_reasons_only(BIT_ULL(42))); + KUNIT_EXPECT_TRUE(test, apple_smc_port_reasons_only(BIT_ULL(41) | BIT_ULL(42))); + /* Anything else, with it or alone, is a real wake. */ + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(0)); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(42) | BIT_ULL(0))); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(42) | BIT_ULL(10))); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(23))); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(43))); +} + static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_key_info_lock_test), KUNIT_CASE(macsmc_key_index_test), @@ -209,6 +379,13 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_request_bounds_test), KUNIT_CASE(macsmc_sram_value_test), KUNIT_CASE(macsmc_key_info_test), + KUNIT_CASE(macsmc_sleep_order_test), + KUNIT_CASE(macsmc_sleep_warn_late_test), + KUNIT_CASE(macsmc_sleep_refused_test), + KUNIT_CASE(macsmc_notice_test), + KUNIT_CASE(macsmc_lid_wake_test), + KUNIT_CASE(macsmc_wake_reason_test), + KUNIT_CASE(macsmc_port_reasons_test), {} }; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 927ddaa00e0df..fce9dc9527e66 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -22,7 +22,10 @@ #include #include #include +#include #include +#include +#include #include #define SMC_ENDPOINT 0x20 @@ -120,6 +123,14 @@ static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) memcpy_fromio(buf, smc->shmem.iomem, size); } +/* In a noirq phase the mailbox's interrupt is off, so its reply has to be fetched. */ +static bool apple_smc_reply_arrived(struct apple_smc *smc) +{ + apple_rtkit_poll(smc->rtk); + + return completion_done(&smc->cmd_done); +} + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -136,6 +147,9 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return -EIO; if (smc->atomic_mode) return -EIO; + /* Between the AP power notices the firmware expects no commands. */ + if (smc->ap_notified) + return -EAGAIN; reinit_completion(&smc->cmd_done); @@ -152,7 +166,17 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return ret; } - if (wait_for_completion_timeout(&smc->cmd_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) <= 0) { + if (smc->polled) { + bool done; + + ret = read_poll_timeout(apple_smc_reply_arrived, done, done, 100, + SMC_TIMEOUT_MS * USEC_PER_MSEC, false, smc); + } else { + ret = wait_for_completion_timeout(&smc->cmd_done, + msecs_to_jiffies(SMC_TIMEOUT_MS)) > 0 ? + 0 : -ETIMEDOUT; + } + if (ret) { dev_err(smc->dev, "Command timed out (%llx)", msg); return -ETIMEDOUT; } @@ -598,6 +622,10 @@ static int apple_smc_probe(struct platform_device *pdev) ARRAY_SIZE(apple_smc_telemetry_devs), NULL, 0, NULL); + ret = devm_device_init_wakeup(dev); + if (ret) + return ret; + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -634,10 +662,335 @@ static const struct of_device_id apple_smc_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_smc_of_match); +/* + * The SMC keeps running through a firmware-driven suspend. It is told first and + * stops last, and on a wake it hears first. In order, measured on T8140 and + * unverified on other SoCs: MBSE = slpw before any other coprocessor is told; + * MBSE = slpa and the AP power notice after all the others; on a wake the notice, + * MBSE and MBSW read, MBSE = waka. The noirq phases give that order whatever the + * device order is, so these commands poll. What the values mean is inferred + * from their letters. + * + * SMC.OutboxNotEmpty wakes the SoC while anything waits in the outbox. An event that arrives + * during the sleep's entry (the lid, a key) is left there, so the sleep returns at once. + * + * The debug link's cut, on a debug cable only, makes the SMC see a cable pulled and plugged + * back. With the cut after slpw, the SMC woke the AP right after the commit, for USB-C_plug + * or acdetach, in every such sleep (J700, 223 of 223), even with its outbox quiet for 500 ms + * before the notice. A sleep that cuts the link warns the SMC only after the cut. + */ + +/* The gaps macOS leaves before the AP idle notice and after it. */ +#define SMC_SLEEP_ARM_MS 6 +#define SMC_SLEEP_IDLE_MS 30 +/* + * Between a late warning (a sleep that cut the debug link) and slpa. macOS's shortest gap + * was 770 ms; the warning at .prepare leaves 150-500 ms, and those sleeps run to the alarm. + */ +#define SMC_SLEEP_WARN_MS 200 + +/* After the debug link's cut. No message came in it in any logged sleep. */ +#define SMC_OUTBOX_QUIET_MS 100 +#define SMC_OUTBOX_SETTLE_MS 5000 + +static int apple_smc_ap_power(struct apple_smc *smc, bool on) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_ap_power, smc, on); + + return smc->polled ? apple_rtkit_ap_power_noirq(smc->rtk, on) : + apple_rtkit_ap_power(smc->rtk, on); +} + +static void apple_smc_set_ap_notified(struct apple_smc *smc, bool notified) +{ + guard(mutex)(&smc->mutex); + + smc->ap_notified = notified; +} + +static void apple_smc_set_polled(struct apple_smc *smc, bool polled) +{ + guard(mutex)(&smc->mutex); + + smc->polled = polled; +} + +static void apple_smc_write_mbse(struct apple_smc *smc, smc_key value) +{ + int ret; + + ret = apple_smc_write_u32(smc, SMC_KEY(MBSE), value); + if (ret) + dev_err(smc->dev, "Failed to issue MBSE = %p4ch: %d\n", &value, ret); +} + +static void apple_smc_sleep_warn(struct apple_smc *smc) +{ + apple_smc_write_mbse(smc, SMC_KEY(slpw)); + smc->sleep_warned = true; +} + +static int apple_smc_notice_idle(struct apple_smc *smc) +{ + int ret; + + /* + * Commands fail from here on. The mutex isn't held across the notice: the + * answer comes through the same worker as the notifications, and those can + * be waiting for this mutex. + */ + apple_smc_set_ap_notified(smc, true); + + ret = apple_smc_ap_power(smc, false); + if (ret) { + /* Taken and not answered, perhaps, so undo it. */ + apple_smc_ap_power(smc, true); + apple_smc_set_ap_notified(smc, false); + dev_err(smc->dev, "SMC did not take the sleep notice: %d\n", ret); + } else { + dev_dbg(smc->dev, "AP idle notice answered\n"); + } + + return ret; +} + +static int apple_smc_sleep_commit(struct apple_smc *smc) +{ + int ret; + + apple_smc_write_mbse(smc, SMC_KEY(slpa)); + /* Device interrupts are off here. */ + mdelay(SMC_SLEEP_ARM_MS); + + ret = apple_smc_notice_idle(smc); + if (!ret) + mdelay(SMC_SLEEP_IDLE_MS); + + return ret; +} + +/* MBSW's bits, as smc-pmu's event_name properties number them on T8140. Other SoCs differ. */ +static const struct { + u8 bit; + const char *name; +} apple_smc_wake_reasons[] = { + { 0, "pwrbtn" }, { 9, "ipd_touch" }, { 10, "lid" }, { 16, "ipd_kbd" }, + { 18, "charger" }, { 23, "wifibt" }, { 40, "acattach" }, { 41, "acdetach" }, + { 42, "USB-C_plug" }, { 43, "USB2_plug" }, { 44, "USB2_wake" }, { 45, "aupo" }, + { 48, "rtc" }, +}; + +static int apple_smc_wake_reason_names(u64 record, char *buf, size_t size) +{ + int i, len = 0; + + buf[0] = 0; + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (record & BIT_ULL(apple_smc_wake_reasons[i].bit)) + len += scnprintf(buf + len, size - len, "%s%s", len ? " " : "", + apple_smc_wake_reasons[i].name); + + return len; +} + +static void apple_smc_notice_on(struct apple_smc *smc) +{ + u8 record[8] = {}; + char names[128]; + u32 event = 0; + int ret; + + ret = apple_smc_ap_power(smc, true); + if (ret) + dev_err(smc->dev, "SMC did not take the wake notice: %d\n", ret); + apple_smc_set_ap_notified(smc, false); + + ret = apple_smc_read_u32(smc, SMC_KEY(MBSE), &event); + if (ret) + dev_warn(smc->dev, "Failed to read MBSE after a wake: %d\n", ret); + ret = apple_smc_read(smc, SMC_KEY(MBSW), record, sizeof(record)); + if (ret < 0) + dev_warn(smc->dev, "Failed to read MBSW after a wake: %d\n", ret); + + smc->wake_record = get_unaligned_le64(record); + apple_smc_wake_reason_names(smc->wake_record, names, sizeof(names)); + dev_info(smc->dev, "wake record: MBSE %#x, MBSW %8ph%s%s\n", event, record, + names[0] ? ": " : "", names); +} + +/* MBSW's bit for a wake reason, by its name, or -1. */ +static int apple_smc_wake_reason_bit(const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (!strcmp(apple_smc_wake_reasons[i].name, name)) + return apple_smc_wake_reasons[i].bit; + + return -1; +} + +/* + * Woken for the lid alone, which the SMC reports for the lid opening and for it closing: + * tell the sleep driver if it's closed now. + */ +static void apple_smc_lid_wake(struct apple_smc *smc) +{ + int bit = apple_smc_wake_reason_bit("lid"); + u8 lid = 0; + int ret; + + if (bit < 0 || smc->wake_record != BIT_ULL(bit)) + return; + + ret = apple_smc_read_u8(smc, SMC_KEY(MSLD), &lid); + if (ret < 0) { + dev_warn(smc->dev, "Failed to read the lid after a wake: %d\n", ret); + return; + } + dev_info(smc->dev, "woken for the lid, which is %s\n", lid ? "closed" : "open"); + if (lid) + apple_sleep_lid_closed_wake(); +} + +/* Only what a cable pulled and plugged back reports: what the debug link's cut makes. */ +static bool apple_smc_port_reasons_only(u64 record) +{ + static const char * const names[] = { "acattach", "acdetach", "USB-C_plug" }; + u64 port = 0; + int i, bit; + + for (i = 0; i < ARRAY_SIZE(names); i++) { + bit = apple_smc_wake_reason_bit(names[i]); + if (bit >= 0) + port |= BIT_ULL(bit); + } + + return record && !(record & ~port); +} + +static void apple_smc_sleep_wake(struct apple_smc *smc) +{ + apple_smc_notice_on(smc); + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + apple_smc_lid_wake(smc); + if (apple_smc_port_reasons_only(smc->wake_record)) + apple_sleep_debug_link_cut_wake(); + /* A dark wake isn't one: with wakeup_count in use, it would abort the sleep after it. */ + if (smc->wake_record && !pm_dark_wake()) + pm_wakeup_event(smc->dev, 0); +} + +static int apple_smc_prepare(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (!pm_suspend_via_firmware()) + return 0; + if (apple_sleep_debug_link_up()) + smc->sleep_warn_late = true; + else + apple_smc_sleep_warn(smc); + + return 0; +} + +/* Warned, and a refusal came before the notice: tell it the sleep is off. */ +static void apple_smc_complete(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + smc->sleep_warn_late = false; + if (smc->sleep_warned) { + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + } +} + +/* + * For at least @min_ms, then until the outbox is quiet for @quiet_ms. The events go to their + * handlers as usual, through the RTKit worker. + */ +static void apple_smc_outbox_poll(struct apple_smc *smc, const char *after, + unsigned int min_ms, unsigned int quiet_ms) +{ + unsigned int ms, quiet = 0, count = 0, last = 0; + int ret; + + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_outbox_poll, smc, after, min_ms, quiet_ms); + + for (ms = 0; ms < SMC_OUTBOX_SETTLE_MS && (ms < min_ms || quiet < quiet_ms); ms++) { + ret = apple_rtkit_poll(smc->rtk); + if (ret > 0) { + count += ret; + last = ms; + quiet = 0; + } else { + quiet++; + } + usleep_range(1000, 1200); + } + /* A debug cable only. */ + dev_info(smc->dev, "%s: %u messages in %u ms, the last at %u ms%s\n", after, count, ms, + last, ms >= SMC_OUTBOX_SETTLE_MS ? ", still busy" : ""); +} + +/* The noirq suspend's part, polled: the cut, a late warning, the commit's notices. */ +static int apple_smc_sleep_enter(struct apple_smc *smc) +{ + if (apple_sleep_debug_link_cut()) + apple_smc_outbox_poll(smc, "debug link cut", 0, SMC_OUTBOX_QUIET_MS); + if (smc->sleep_warn_late) { + smc->sleep_warn_late = false; + apple_smc_sleep_warn(smc); + apple_smc_outbox_poll(smc, "late sleep warning", SMC_SLEEP_WARN_MS, 0); + } + + return apple_smc_sleep_commit(smc); +} + +static int apple_smc_suspend_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + int ret; + + if (!smc->sleep_warned && !smc->sleep_warn_late) + return 0; + + apple_smc_set_polled(smc, true); + ret = apple_smc_sleep_enter(smc); + apple_smc_set_polled(smc, false); + + return ret; +} + +static int apple_smc_resume_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (smc->ap_notified) { + apple_smc_set_polled(smc, true); + apple_smc_sleep_wake(smc); + apple_smc_set_polled(smc, false); + } + + return 0; +} + +static const struct dev_pm_ops apple_smc_pm_ops = { + .prepare = pm_sleep_ptr(apple_smc_prepare), + .complete = pm_sleep_ptr(apple_smc_complete), + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_smc_suspend_noirq, apple_smc_resume_noirq) +}; + static struct platform_driver apple_smc_driver = { .driver = { .name = "macsmc", .of_match_table = apple_smc_of_match, + .pm = pm_sleep_ptr(&apple_smc_pm_ops), }, .probe = apple_smc_probe, }; diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index cadf0152d4e9b..87afa3627a9ce 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o wow.o mt7932-fullmac-$(CONFIG_MT7932_AWDL) += awdl.o mt7932-fullmac-$(CONFIG_MT7932_AWDL_KUNIT_TEST) += awdl-test.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o @@ -10,5 +10,6 @@ mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += move-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += wow-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 2c8313d2f523a..8601085b299d7 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -564,7 +564,8 @@ static void mt_cal_work(struct work_struct *work) int ret; mutex_lock(&m->command_mutex); - if (!READ_ONCE(m->rf_ready)) + /* Stopped for a sleep, the requests wait for the restart. */ + if (!READ_ONCE(m->rf_ready) || READ_ONCE(m->hif_suspended)) goto unlock; ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) @@ -633,6 +634,7 @@ int mt_register_interface(struct mt7932 *m) INIT_WORK(&m->connect_work, mt_connect_work); INIT_WORK(&m->disconnect_work, mt_disconnect_work); INIT_WORK(&m->power_work, mt_power_work); + INIT_WORK(&m->wow_work, mt_wow_work); init_completion(&m->assoc_start); init_completion(&m->assoc_done); init_completion(&m->disconnect_done); diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index ed08deb9a6181..f16fb04259c7a 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -55,6 +55,7 @@ void mt_stop_host(struct mt7932 *m) flush_work(&m->disconnect_work); cancel_work_sync(&m->cal_work); cancel_work_sync(&m->power_work); + cancel_work_sync(&m->wow_work); cancel_delayed_work_sync(&m->scan_timeout_work); cancel_work_sync(&m->scan_finish_work); /* With all producers joined, this is the sole owner of any remainder. */ diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index f64868a7aeb5f..6ee30c04d8e64 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -7,6 +7,7 @@ #include #include "mt7932.h" +#include "wow.h" #include @@ -46,7 +47,8 @@ static void mt_shutdown(struct pci_dev *pdev); bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && - !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed); + !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed) && + !READ_ONCE(m->hif_suspended); } int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name) @@ -132,6 +134,30 @@ static void mt_rx_post(struct mt7932_ring *q, unsigned int index) d->control = cpu_to_le32(MT7932_RX_CAPACITY << 16); } +/* + * The host interface control command and event, the same ID both ways. The command is + * type, direction, stop, suspend in 36 bytes; the event's body is type, TX traffic, RX + * traffic, suspend, with at least as many bytes. + */ +#define MT7932_CID_HIF_CTRL 0xf6 +#define MT7932_HIF_PCIE 2 +#define MT7932_HIF_IDLE 2 +#define MT7932_HIF_LENGTH 36 + +/* response_lock held. */ +static void mt_hif_event(struct mt7932 *m, const struct mt7932_event *event) +{ + const u8 *body = event->packet + 36; + + if (!m->hif_waiting || event->length < 36 + MT7932_HIF_LENGTH || + body[0] != MT7932_HIF_PCIE || !!body[3] != m->hif_suspending) + return; + m->hif_waiting = false; + m->hif_result = body[3] && (body[1] != MT7932_HIF_IDLE || body[2] != MT7932_HIF_IDLE) ? + -EBUSY : 0; + complete(&m->hif_done); +} + static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) { struct mt7932_event event; @@ -155,10 +181,18 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) return; } spin_lock_irqsave(&m->response_lock, flags); + if (event.eid == MT7932_CID_HIF_CTRL) { + mt_hif_event(m, &event); + goto unlock; + } if (event.eid == 0xee) { mt_link_event(m, &event); goto unlock; } + if (event.eid == MT7932_EID_WOW_REASON) { + mt_wow_reason_event(m, &event); + goto unlock; + } if (event.eid == 0x0d) { mt_scan_event(m, &event); goto unlock; @@ -376,6 +410,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) return -ESHUTDOWN; + /* With the host interface stopped only its own control may reach the BAR. */ + if (READ_ONCE(m->hif_gate) && !READ_ONCE(m->hif_waiting)) + return -ESHUTDOWN; if (q->used >= q->slots) { /* Command buffers can wrap only after every published descriptor has * retired. Never reclaim by elapsed time or by a reply alone. @@ -1106,6 +1143,8 @@ static ssize_t removal_retry_store(struct device *dev, } static DEVICE_ATTR_WO(removal_retry); +static struct pci_driver mt_driver; + static int mt_retire_dma(struct mt7932 *m) { int ret; @@ -1128,11 +1167,35 @@ static int mt_retire_dma(struct mt7932 *m) return ret; } +/* + * Removal waits forever for a reset that fails, which a recovery in a worker can't do. So stop + * here first, where a failure keeps the function bound. A radio stopped this way stays down + * until it's removed, and remove then has nothing left to wait for. + */ +static int mt_radio_stop(struct pci_dev *pdev) +{ + struct mt7932 *m; + int ret = 0; + + device_lock(&pdev->dev); + m = pdev->dev.driver == &mt_driver.driver ? pci_get_drvdata(pdev) : NULL; + if (m) { + mt_stop_host(m); + ret = mt_retire_dma(m); + if (ret) + dev_err(&pdev->dev, "restart refused: DMA still owned: %d\n", ret); + } + device_unlock(&pdev->dev); + + return ret; +} + static void mt_remove(struct pci_dev *pdev) { struct mt7932 *m = pci_get_drvdata(pdev); int ret; + apple_piodma_radio_set_stop(NULL); mt_stop_host(m); while ((ret = mt_retire_dma(m))) { WRITE_ONCE(m->removal_error, ret); @@ -1216,6 +1279,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) init_completion(&m->response); init_completion(&m->cal_response); init_completion(&m->reset_retry); + init_completion(&m->hif_done); mutex_init(&m->command_mutex); spin_lock_init(&m->response_lock); ret = mt_ownership(m); @@ -1251,6 +1315,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = device_create_file(&pdev->dev, &dev_attr_removal_retry); if (ret) dev_warn(&pdev->dev, "removal retry attribute unavailable: %d\n", ret); + apple_piodma_radio_set_stop(mt_radio_stop); return 0; } @@ -1264,21 +1329,192 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* command_mutex held; the firmware answers once its host interface has changed state. */ +static int mt_hif_ctrl(struct mt7932 *m, bool suspend) +{ + u8 body[MT7932_HIF_LENGTH] = { MT7932_HIF_PCIE, 1, suspend, suspend }; + unsigned long flags; + int ret; + + reinit_completion(&m->hif_done); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_suspending = suspend; + m->hif_result = -ETIMEDOUT; + m->hif_waiting = true; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt_request(m, MT7932_CID_HIF_CTRL, true, true, false, body, sizeof(body)); + if (!ret) + wait_for_completion_timeout(&m->hif_done, msecs_to_jiffies(1000)); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_waiting = false; + if (!ret) + ret = m->hif_result; + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* Every published command and frame retired, ahead of the firmware's own stop. */ +static int mt_hif_tx_idle(struct mt7932 *m) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(500); + + do { + mt_data_clean(m); + mt_tx_clean(&m->tx[17]); + if (!READ_ONCE(m->tx[0].queued) && !m->tx[17].queued) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + return -EBUSY; +} + +/* + * Driver own again, DMA back on, then the firmware's host interface, and WoW + * off if the sleep had it. Rings stay as they were. + */ +static int mt_hif_restart(struct mt7932 *m) +{ + bool wow; + int ret; + + ret = mt_ownership(m); + if (ret) + return ret; + mt_write(m, 0x10188, 0xff); + if (mt_read(m, m->rx[0].reg) != lower_32_bits(m->rx[0].dma) || + mt_read(m, m->tx[17].reg) != lower_32_bits(m->tx[17].dma)) { + dev_err(&m->pdev->dev, "DMA rings lost across the sleep: RX0 %#x, TX17 %#x\n", + mt_read(m, m->rx[0].reg), mt_read(m, m->tx[17].reg)); + return -EIO; + } + mt_write(m, W + 0x208, m->hif_glo); + WRITE_ONCE(m->running, true); + mt_write(m, W + 0x204, m->irq_mask); + mutex_lock(&m->command_mutex); + wow = m->wow_armed; + ret = mt_hif_ctrl(m, false); + if (!ret) { + WRITE_ONCE(m->hif_gate, false); + ret = mt_wow_disarm(m); + if (ret) + WRITE_ONCE(m->hif_gate, true); + } + mutex_unlock(&m->command_mutex); + if (ret) { + /* Not left running into the next sleep. */ + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + mt_dma_stop(m); + return ret; + } + WRITE_ONCE(m->hif_suspended, false); + /* What arrived or was asked for while stopped, now that it can run. */ + if (READ_ONCE(m->cal_request_count)) + schedule_work(&m->cal_work); + if (wow) + schedule_work(&m->power_work); + if (READ_ONCE(m->interface_registered) && READ_ONCE(m->reg_pending)) + schedule_work(&m->startup_work); + if (m->netdev) + netif_device_attach(m->netdev); + return 0; +} + /* - * Stopping the firmware for sleep would take the whole remove and probe - * sequence, so leave it running across suspend-to-idle. A saved state keeps - * the PCI core from changing the function's power state or command register, - * and the awake path keeps the PCIe controller powered until resume. + * What macOS does at sleep. Without a WoWLAN configuration cfg80211 has already left + * the network; with one the firmware is given WoW first and stays associated. The firmware + * stops its host interface and is given ownership, and keeps running. PERST# then resets + * the link, not the firmware. + * + * Trouble here costs Wi-Fi at the wake, never the machine its sleep. + */ +static int mt_hif_suspend(struct mt7932 *m) +{ + int ret; + + if (!READ_ONCE(m->ready) || READ_ONCE(m->stopping) || READ_ONCE(m->hif_suspended)) + return 0; + /* New work waits from here, and what already runs finishes before the gate. */ + WRITE_ONCE(m->hif_suspended, true); + if (m->interface_registered) { + if (m->netdev) + netif_device_detach(m->netdev); + flush_work(&m->startup_work); + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + flush_work(&m->cal_work); + flush_work(&m->power_work); + mt_abort_scan(m->wiphy, &m->wdev); + ret = mt_scan_quiesce(m); + if (ret) + dev_warn(&m->pdev->dev, "scan still retiring at the sleep: %d\n", ret); + } + mutex_lock(&m->command_mutex); + mt_wow_arm(m); + WRITE_ONCE(m->hif_gate, true); + ret = mt_hif_tx_idle(m); + if (!ret) + ret = mt_hif_ctrl(m, true); + mutex_unlock(&m->command_mutex); + if (ret) + dev_err(&m->pdev->dev, "firmware did not stop its host interface: %d\n", ret); + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + m->hif_glo = mt_read(m, W + 0x208) & ~(BIT(1) | BIT(3)); + ret = mt_dma_stop(m); + if (ret) + dev_err(&m->pdev->dev, "DMA did not stop for the sleep: %d\n", ret); + mt_write(m, 0xe0010, 1); + ret = mt_poll(m, 0xe0010, BIT(2), BIT(2), 50000); + if (ret) + dev_err(&m->pdev->dev, "firmware did not take ownership for the sleep: %d\n", ret); + else + dev_dbg(&m->pdev->dev, "host interface stopped for the sleep, firmware kept\n"); + return 0; +} + +/* + * Stopping the firmware for sleep takes the whole remove and probe sequence, so + * leave it running across suspend-to-idle. A saved state keeps the PCI core from + * changing the function's power state or command register, and the awake path + * keeps the PCIe controller powered until resume. */ static int mt_suspend(struct device *dev) { if (!pm_suspend_no_platform()) - return -EOPNOTSUPP; + return mt_hif_suspend(pci_get_drvdata(to_pci_dev(dev))); pci_save_state(to_pci_dev(dev)); device_set_awake_path(dev); return 0; } +static int mt_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mt7932 *m = pci_get_drvdata(pdev); + unsigned long flags; + int ret; + + if (!READ_ONCE(m->hif_suspended)) + return 0; + /* As macOS does: a chip that no longer answers isn't touched further. */ + ret = pci_device_is_present(pdev) ? mt_hif_restart(m) : -ENODEV; + if (!ret) { + dev_dbg(dev, "back after the sleep, firmware kept\n"); + return 0; + } + dev_err(dev, "Wi-Fi didn't keep its firmware across the sleep: %d; restarting it\n", ret); + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + apple_piodma_radio_recover(); + return ret; +} + /* * Hibernation is refused. Its freeze phase does not call .suspend, so without * this the firmware would keep writing to memory while the image is saved, @@ -1295,6 +1531,7 @@ static int mt_freeze(struct device *dev) static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .resume = mt_resume, .freeze = mt_freeze, .poweroff = mt_freeze, }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 040206b8815c2..9cbb592320a6a 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -93,6 +93,12 @@ struct mt7932 { bool dma_owned, module_pinned; struct completion reset_retry; int removal_error; + /* The firmware keeps running across a deep sleep with its host interface stopped. */ + struct completion hif_done; + int hif_result; + u32 hif_glo; + /* hif_suspended stops new work; hif_gate then closes the BAR to all but the control. */ + bool hif_waiting, hif_suspending, hif_suspended, hif_gate; bool ram_config; bool runtime_epoch; bool runtime_cal_valid; @@ -158,6 +164,14 @@ struct mt7932 { /* Desired/submitted PS configuration uses command_mutex; TIM uses response_lock. */ u8 power_mode, power_applied; bool power_applied_valid, power_tim; + /* WoWLAN detectors cfg80211 asked for at this sleep; 0 leaves the network. */ + u8 wow_detect; + /* The firmware was given WoW at the sleep; command_mutex. */ + bool wow_armed; + /* A wake reason is awaited, and the one received; response_lock. */ + bool wow_report; + u8 wow_reason; + struct work_struct wow_work; struct completion assoc_start, assoc_done, disconnect_done; struct completion discovery_done; struct completion scan_done; @@ -347,5 +361,15 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); +int mt_power_submit(struct mt7932 *m, u8 mode); +#ifdef CONFIG_PM +extern const struct wiphy_wowlan_support mt_wowlan_support; +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow); +int mt_wow_resume(struct wiphy *wiphy); +#endif +void mt_wow_arm(struct mt7932 *m); +int mt_wow_disarm(struct mt7932 *m); +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event); +void mt_wow_work(struct work_struct *work); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index f3c1c42863c9c..03653ba97eede 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -5,7 +5,7 @@ * submission is accepted configuration, not proof that the radio is asleep. * cfg80211 implements GET_POWER_SAVE from the accepted requested setting. */ -static int mt_power_submit(struct mt7932 *m, u8 mode) +int mt_power_submit(struct mt7932 *m, u8 mode) { u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 04524891d36d0..f60651470b216 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -561,7 +561,8 @@ static void mt_startup_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); generation = m->reg_generation; *reg = m->reg_desired; - if (m->stopping || generation == m->reg_attempted) { + /* Stopped for a sleep, the restart runs this generation again. */ + if (m->stopping || READ_ONCE(m->hif_suspended) || generation == m->reg_attempted) { spin_unlock_irqrestore(&m->response_lock, flags); kfree(reg); return; @@ -809,6 +810,10 @@ static const struct cfg80211_ops mt_cfg_ops = { .del_station = mt_awdl_del_station, .change_station = mt_awdl_change_station, #endif +#ifdef CONFIG_PM + .suspend = mt_wow_suspend, + .resume = mt_wow_resume, +#endif }; int mt_register_regulatory_gate(struct mt7932 *m) @@ -846,6 +851,9 @@ int mt_register_regulatory_gate(struct mt7932 *m) wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_MFP_OPTIONAL); m->wiphy->max_scan_ssids = MT7932_SCAN_SSIDS; m->wiphy->max_scan_ie_len = 600; +#ifdef CONFIG_PM + m->wiphy->wowlan = &mt_wowlan_support; +#endif m->wiphy->reg_notifier = mt_regulatory_notify; /* World roaming may relax NO_IR after observing an AP beacon without * changing alpha2. Request the callback so firmware's CID0f domain is diff --git a/drivers/net/wireless/mediatek/mt7932/wow-test.c b/drivers/net/wireless/mediatek/mt7932/wow-test.c new file mode 100644 index 0000000000000..1301fa93b7bbb --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow-test.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "mt7932.h" +#include "wow.h" + +static bool mt_wow_test_zero(const u8 *p, size_t from, size_t to) +{ + size_t i; + + for (i = from; i < to; i++) + if (p[i]) + return false; + return true; +} + +static void mt_wow_detect_test(struct kunit *test) +{ + struct cfg80211_wowlan wow = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(NULL), 0); + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), MT7932_WOW_DETECT_MAGIC); + wow.magic_pkt = false; + wow.disconnect = true; + /* Losing the AP's beacons ends the association as surely as a deauth. */ + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x14); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x15); +} + +static void mt_wow_ctrl_test(struct kunit *test) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)228); + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, true, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_START); + KUNIT_EXPECT_EQ(test, body[1], 0x15); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), 0x63); + KUNIT_EXPECT_EQ(test, body[4], 0); /* scenario */ + KUNIT_EXPECT_EQ(test, body[5], 1); /* pins */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 6, 8)); + KUNIT_EXPECT_EQ(test, body[8], 3); /* GPIO */ + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + /* Interval, second wake signal, port counts and lists. */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); + + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, false, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_STOP); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 1, 5)); + KUNIT_EXPECT_EQ(test, body[5], 1); + KUNIT_EXPECT_EQ(test, body[8], 3); + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); +} + +static void mt_wow_suspend_mode_test(struct kunit *test) +{ + u8 body[MT7932_SUSPEND_MODE_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)68); + memset(body, 0xa5, sizeof(body)); + mt7932_suspend_mode(body, true); + KUNIT_EXPECT_EQ(test, body[0], 0); /* BSS */ + KUNIT_EXPECT_EQ(test, body[1], 1); + KUNIT_EXPECT_EQ(test, body[2], 1); /* DTIM multiple */ + KUNIT_EXPECT_EQ(test, body[3], 1); /* WoW suspend */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 4, sizeof(body))); + mt7932_suspend_mode(body, false); + KUNIT_EXPECT_EQ(test, body[1], 0); + KUNIT_EXPECT_EQ(test, body[2], 1); + KUNIT_EXPECT_EQ(test, body[3], 1); +} + +static void mt_wow_config_test(struct kunit *test) +{ + static const u8 addr[ETH_ALEN] = { 0x02, 0x11, 0x22, 0x33, 0x44, 0x55 }; + u8 magic[MT7932_WOW_MAGIC_LENGTH], clear[MT7932_WOW_CLEAR_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(magic), (size_t)52); + KUNIT_EXPECT_EQ(test, sizeof(clear), (size_t)44); + memset(magic, 0xa5, sizeof(magic)); + mt7932_wow_magic(magic, addr); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 32), 0x06); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 34), 20); + KUNIT_EXPECT_EQ(test, magic[36], 0); /* entry */ + KUNIT_EXPECT_EQ(test, magic[37], 1); /* enabled */ + KUNIT_EXPECT_EQ(test, magic[38], 1); /* forward */ + KUNIT_EXPECT_EQ(test, magic[39], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 40), 0); + KUNIT_EXPECT_MEMEQ(test, magic + 42, addr, ETH_ALEN); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 48, sizeof(magic))); + + memset(clear, 0xa5, sizeof(clear)); + mt7932_wow_clear(clear); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 32), 0x18); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 34), 12); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 36, sizeof(clear))); +} + +static void mt_wow_reason_test(struct kunit *test) +{ + struct cfg80211_wowlan_wakeup wakeup; + u8 body[MT7932_WOW_REASON_LENGTH] = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body) - 1), -EINVAL); + body[0] = 9; + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body)), 9); + + KUNIT_EXPECT_FALSE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_NONE, &wakeup)); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_MAGIC, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_DISCONNECT, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.disconnect); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + /* A firmware reason with no cfg80211 trigger is still a wireless wake. */ + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(9, &wakeup)); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); +} + +#ifdef CONFIG_PM +static void mt_wow_test_work(struct work_struct *work) +{ +} + +static void mt_wow_test_device_release(struct device *dev) +{ +} + +static void mt_wow_test_put(void *data) +{ + put_device(data); +} + +static void mt_wow_reason_event_test(struct kunit *test) +{ + struct mt7932 *m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + u8 *packet = kunit_kzalloc(test, 36 + MT7932_WOW_REASON_LENGTH, GFP_KERNEL); + struct mt7932_event event = { + .eid = MT7932_EID_WOW_REASON, + .packet = packet, + .length = 36 + MT7932_WOW_REASON_LENGTH, + }; + unsigned long flags; + + KUNIT_ASSERT_NOT_NULL(test, m); + KUNIT_ASSERT_NOT_NULL(test, packet); + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, m->pdev); + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_wow_test_device_release; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mt_wow_test_put, &m->pdev->dev), 0); + KUNIT_ASSERT_EQ(test, dev_set_name(&m->pdev->dev, "mt7932-wow-test"), 0); + spin_lock_init(&m->response_lock); + INIT_WORK(&m->wow_work, mt_wow_test_work); + m->wow_reason = MT7932_WOW_REASON_NONE; + packet[36] = MT7932_WOW_REASON_MAGIC; + + /* Not a sleep WoW was armed for: ignored. */ + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* A short event leaves the wait open. */ + m->wow_report = true; + event.length--; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_TRUE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* One report per sleep. */ + event.length++; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_FALSE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); + packet[36] = MT7932_WOW_REASON_DISCONNECT; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); +} +#endif + +static struct kunit_case mt_wow_test_cases[] = { + KUNIT_CASE(mt_wow_detect_test), + KUNIT_CASE(mt_wow_ctrl_test), + KUNIT_CASE(mt_wow_suspend_mode_test), + KUNIT_CASE(mt_wow_config_test), + KUNIT_CASE(mt_wow_reason_test), +#ifdef CONFIG_PM + KUNIT_CASE(mt_wow_reason_event_test), +#endif + {} +}; + +static struct kunit_suite mt_wow_test_suite = { + .name = "mt7932-wow", + .test_cases = mt_wow_test_cases, +}; + +kunit_test_suite(mt_wow_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/wow.c b/drivers/net/wireless/mediatek/mt7932/wow.c new file mode 100644 index 0000000000000..279a0c01a7b1d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Wake-on-WLAN for a deep sleep. The firmware keeps the association while the + * host sleeps and wakes it through the radio's GPIO, which the SMC turns into + * an AP wake. Nothing in-band can wake the SoC from S2R. + */ +#include + +#include "mt7932.h" +#include "wow.h" + +#ifdef CONFIG_PM +const struct wiphy_wowlan_support mt_wowlan_support = { + .flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT, +}; + +/* response_lock held. */ +static bool mt_wow_associated_locked(struct mt7932 *m) +{ + return m->connected && m->peer_valid && !m->connecting && !m->disconnecting && + !m->link_failed && !m->stopping && m->rf_ready; +} + +/* + * cfg80211 calls this before the device's own suspend. Returning 1 makes it + * leave the network and call again without a configuration. + */ +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + u8 detect = mt7932_wow_detect(wow); + unsigned long flags; + bool associated; + + WRITE_ONCE(m->wow_detect, 0); + if (!wow) + return 0; + /* Only S2R hands the radio to the firmware; s2idle and hibernation leave. */ + if (!detect || pm_suspend_target_state != PM_SUSPEND_MEM || pm_suspend_no_platform()) + return 1; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) + return 1; + WRITE_ONCE(m->wow_detect, detect); + return 0; +} + +int mt_wow_resume(struct wiphy *wiphy) +{ + WRITE_ONCE(mt_from_wiphy(wiphy)->wow_detect, 0); + return 0; +} + +/* + * command_mutex held, before the host interface stops. The firmware takes the + * commands from the ring before it answers the stop. + */ +void mt_wow_arm(struct mt7932 *m) +{ + u8 detect = READ_ONCE(m->wow_detect); + u8 body[MT7932_WOW_CTRL_LENGTH]; + unsigned long flags; + bool associated; + int ret = 0; + + lockdep_assert_held(&m->command_mutex); + WRITE_ONCE(m->wow_detect, 0); + if (!detect) + return; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + m->wow_report = false; + m->wow_reason = MT7932_WOW_REASON_NONE; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) { + dev_dbg(&m->pdev->dev, "not associated at the sleep: Wi-Fi wake not armed\n"); + return; + } + /* Whatever reached the firmware is undone at the wake. */ + m->wow_armed = true; + if (detect & MT7932_WOW_DETECT_MAGIC) { + mt7932_wow_magic(body, m->mac); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_MAGIC_LENGTH); + } + if (!ret) + ret = mt_power_submit(m, MT7932_WOW_POWER_MODE); + if (!ret) { + mt7932_suspend_mode(body, true); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + } + if (!ret) { + mt7932_wow_ctrl(body, true, detect); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (ret) { + dev_err(&m->pdev->dev, "Wi-Fi wake not armed: %d\n", ret); + return; + } + spin_lock_irqsave(&m->response_lock, flags); + m->wow_report = true; + spin_unlock_irqrestore(&m->response_lock, flags); + dev_dbg(&m->pdev->dev, "Wi-Fi wake armed:%s%s, staying associated\n", + detect & MT7932_WOW_DETECT_MAGIC ? " magic packet" : "", + detect & MT7932_WOW_DETECT_DISCONNECT ? " disconnect" : ""); +} + +/* + * command_mutex held, once the host interface runs again. A command that + * cannot be queued leaves the firmware filtering, so the caller resets it. + */ +int mt_wow_disarm(struct mt7932 *m) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + int ret; + + lockdep_assert_held(&m->command_mutex); + if (!m->wow_armed) + return 0; + m->wow_armed = false; + mt7932_suspend_mode(body, false); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + if (!ret) { + mt7932_wow_ctrl(body, false, 0); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (!ret) { + mt7932_wow_clear(body); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_CLEAR_LENGTH); + } + /* The sleep's profile stays until power_work submits the requested one. */ + m->power_applied_valid = false; + if (ret) + dev_err(&m->pdev->dev, "Wi-Fi wake not stopped: %d\n", ret); + return ret; +} + +/* response_lock held. The firmware reports why it woke the host. */ +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ + int reason = event->length < 36 ? -EINVAL : + mt7932_wow_reason(event->packet + 36, event->length - 36); + + if (reason < 0) { + dev_dbg(&m->pdev->dev, "short wake reason event: %zu bytes\n", event->length); + return; + } + if (!m->wow_report) + return; + m->wow_report = false; + m->wow_reason = reason; + schedule_work(&m->wow_work); +} + +void mt_wow_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, wow_work); + struct cfg80211_wowlan_wakeup wakeup; + unsigned long flags; + u8 reason; + + spin_lock_irqsave(&m->response_lock, flags); + reason = m->wow_reason; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!mt7932_wow_wakeup(reason, &wakeup)) { + dev_dbg(&m->pdev->dev, "Wi-Fi did not wake the host\n"); + cfg80211_report_wowlan_wakeup(&m->wdev, NULL, GFP_KERNEL); + return; + } + dev_info(&m->pdev->dev, "woken by Wi-Fi: firmware reason %u%s\n", reason, + wakeup.magic_pkt ? " (magic packet)" : + wakeup.disconnect ? " (disconnect)" : ""); + cfg80211_report_wowlan_wakeup(&m->wdev, &wakeup, GFP_KERNEL); +} +#else +void mt_wow_arm(struct mt7932 *m) +{ +} + +int mt_wow_disarm(struct mt7932 *m) +{ + return 0; +} + +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ +} + +void mt_wow_work(struct work_struct *work) +{ +} +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/wow.h b/drivers/net/wireless/mediatek/mt7932/wow.h new file mode 100644 index 0000000000000..2bbe7d4529a1b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_WOW_H +#define MT7932_WOW_H + +#include +#include +#include +#include +#include +#include + +/* + * Wake-on-WLAN commands, in the order the stock driver sends them for an + * associated sleep: the magic packet entry on the WoW configuration command, + * the power-save profile, suspend mode, then WoW start. Waking reverses it. + */ +#define MT7932_CID_WOW_CTRL 0x4a +#define MT7932_CID_WOW_CONFIG 0x4f +#define MT7932_CID_SUSPEND_MODE 0x58 +#define MT7932_EID_WOW_REASON 0x62 + +#define MT7932_WOW_CTRL_LENGTH 228 +#define MT7932_SUSPEND_MODE_LENGTH 68 +/* The configuration command is a zeroed 32-byte header and one TLV. */ +#define MT7932_WOW_CONFIG_HEADER 32 +#define MT7932_WOW_MAGIC_LENGTH (MT7932_WOW_CONFIG_HEADER + 20) +#define MT7932_WOW_CLEAR_LENGTH (MT7932_WOW_CONFIG_HEADER + 12) +#define MT7932_WOW_TLV_MAGIC 0x06 +#define MT7932_WOW_TLV_CLEAR 0x18 + +#define MT7932_WOW_START 1 +#define MT7932_WOW_STOP 2 + +#define MT7932_WOW_DETECT_MAGIC BIT(0) +#define MT7932_WOW_DETECT_DISCONNECT BIT(2) +#define MT7932_WOW_DETECT_BEACON_LOST BIT(4) + +/* + * Drop everything but magic packets, EAPOL and ARP requests to this station, + * as the stock driver filters for a sleep. + */ +#define MT7932_WOW_FILTER 0x63 + +/* The stock configuration wakes the host on the radio's GPIO 13, level 5. */ +#define MT7932_WOW_HIF_GPIO 3 +#define MT7932_WOW_GPIO_PIN 13 +#define MT7932_WOW_GPIO_LEVEL 5 +#define MT7932_WOW_PIN_COUNT 1 + +/* Power-save profile for the sleep, and the suspend mode's DTIM multiple. */ +#define MT7932_WOW_POWER_MODE 2 +#define MT7932_WOW_MDTIM 1 + +#define MT7932_WOW_REASON_MAGIC 0 +#define MT7932_WOW_REASON_DISCONNECT 8 +#define MT7932_WOW_REASON_NONE 0xff +/* The stock driver ignores a shorter reason event. */ +#define MT7932_WOW_REASON_LENGTH 132 + +/* The firmware detectors for the triggers cfg80211 asked for; 0 if none applies. */ +static inline u8 mt7932_wow_detect(const struct cfg80211_wowlan *wow) +{ + u8 detect = 0; + + if (!wow) + return 0; + if (wow->magic_pkt) + detect |= MT7932_WOW_DETECT_MAGIC; + if (wow->disconnect) + detect |= MT7932_WOW_DETECT_DISCONNECT | MT7932_WOW_DETECT_BEACON_LOST; + return detect; +} + +/* Start, with these detectors, or stop. */ +static inline void mt7932_wow_ctrl(u8 out[MT7932_WOW_CTRL_LENGTH], bool start, u8 detect) +{ + memset(out, 0, MT7932_WOW_CTRL_LENGTH); + out[0] = start ? MT7932_WOW_START : MT7932_WOW_STOP; + if (start) { + out[1] = detect; + put_unaligned_le16(MT7932_WOW_FILTER, out + 2); + } + /* Scenario 0, then the wake signal: one pin, GPIO, which pin, level. */ + out[5] = MT7932_WOW_PIN_COUNT; + out[8] = MT7932_WOW_HIF_GPIO; + out[9] = MT7932_WOW_GPIO_PIN; + out[10] = MT7932_WOW_GPIO_LEVEL; + /* No UDP or TCP wake ports. */ +} + +/* BSS 0 into or out of suspend mode; the DTIM multiple and WoW flag stay set. */ +static inline void mt7932_suspend_mode(u8 out[MT7932_SUSPEND_MODE_LENGTH], bool enable) +{ + memset(out, 0, MT7932_SUSPEND_MODE_LENGTH); + out[1] = enable; + out[2] = MT7932_WOW_MDTIM; + out[3] = 1; +} + +/* Magic packet entry 0 for this address, matched from the start of the frame. */ +static inline void mt7932_wow_magic(u8 out[MT7932_WOW_MAGIC_LENGTH], const u8 *addr) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_MAGIC_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_MAGIC, tlv); + put_unaligned_le16(MT7932_WOW_MAGIC_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); + tlv[4] = 0; /* entry */ + tlv[5] = 1; /* enabled */ + tlv[6] = 1; /* offset counts forward */ + put_unaligned_le16(0, tlv + 8); + ether_addr_copy(tlv + 10, addr); +} + +/* Remove every WoW configuration entry. */ +static inline void mt7932_wow_clear(u8 out[MT7932_WOW_CLEAR_LENGTH]) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_CLEAR_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_CLEAR, tlv); + put_unaligned_le16(MT7932_WOW_CLEAR_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); +} + +/* + * The reason from a wake reason event body, or -EINVAL if it is short. + * MT7932_WOW_REASON_NONE means the radio did not wake the host. + */ +static inline int mt7932_wow_reason(const u8 *body, size_t length) +{ + if (length < MT7932_WOW_REASON_LENGTH) + return -EINVAL; + return body[0]; +} + +/* + * What to tell cfg80211 for a reason. False if the radio did not wake the + * host. A reason without a cfg80211 trigger reports a wake with none set. + */ +static inline bool mt7932_wow_wakeup(u8 reason, struct cfg80211_wowlan_wakeup *wakeup) +{ + memset(wakeup, 0, sizeof(*wakeup)); + wakeup->pattern_idx = -1; + switch (reason) { + case MT7932_WOW_REASON_NONE: + return false; + case MT7932_WOW_REASON_MAGIC: + wakeup->magic_pkt = true; + break; + case MT7932_WOW_REASON_DISCONNECT: + wakeup->disconnect = true; + break; + } + return true; +} + +#endif diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f6b2b8cf939ac..f0892f4986b94 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -40,17 +40,21 @@ #include #include #include +#include +#include #include #include #include "nvme.h" #define APPLE_ANS_BOOT_TIMEOUT USEC_PER_SEC +#define APPLE_ANS_SLEEP_SHN_TIMEOUT USEC_PER_SEC /* RTKit handshake timeout for the T6050 ANS, see apple_nvme_alloc() */ #define APPLE_NVME_T6050_HANDSHAKE_TIMEOUT_MS 10000 #define APPLE_ANS_COPROC_CPU_CONTROL 0x44 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_ANS_COPROC_CPU_STATUS 0x48 #define APPLE_ANS_ACQ_DB 0x1004 #define APPLE_ANS_IOCQ_DB 0x100c @@ -211,6 +215,14 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; + /* Firmware-driven suspend has been traced on this controller. */ + bool sleep_traced; + /* The last endpoint map reply from that trace */ + u32 sleep_epmap_last_mask; + /* Higher namespaces are the firmware's, and a Flush to one faults the controller. */ + u32 sleep_flush_max_nsid; + /* CC.SHN for a sleep, as read back from the OS the sleep was traced on. */ + u32 sleep_shn; /* The ANS is never reset through PMGR here, so no reset is named. */ bool reset_optional; u32 max_queue_depth; @@ -237,6 +249,17 @@ struct apple_nvme { bool recover_to_live; bool suspended; bool final_shutdown; + /* This suspend is firmware-driven: ANS will lose power at its end. */ + bool sleep_stop; + bool sleep_idled; + u32 sleep_csts; + /* ANS came back across a sleep, or failed to. reset_work picks these up. */ + bool sleep_restarted; + bool sleep_failed; + /* Run bit set after a sleep, and not yet reported on. */ + bool sleep_restarting; + ktime_t sleep_run_time; + bool sleep_registered; unsigned long recovery_flags; void __iomem *mmio_coproc; @@ -1397,6 +1420,31 @@ static int apple_nvme_disable(struct apple_nvme *anv, bool shutdown) return apple_nvme_disable_locked(anv, shutdown); } +/* + * All of a sleep's stop that can be taken back. Nothing is in flight or can start, and + * the controller and its firmware keep running for a refused sleep to come back to. + */ +static int apple_nvme_sleep_quiesce(struct apple_nvme *anv) +{ + guard(mutex)(&anv->disable_lock); + if (READ_ONCE(anv->quarantined)) + return -EIO; + + if (!test_bit(NVME_CTRL_FROZEN, &anv->ctrl.flags)) + nvme_start_freeze(&anv->ctrl); + if (!nvme_wait_freeze_timeout(&anv->ctrl)) { + /* A reset cancels what's still in flight. Don't sleep on that. */ + dev_warn(anv->dev, "requests still in flight after the timeout\n"); + return -EBUSY; + } + nvme_quiesce_io_queues(&anv->ctrl); + nvme_quiesce_admin_queue(&anv->ctrl); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); + + return 0; +} + static void apple_nvme_recovery_work(struct work_struct *work) { struct apple_nvme *anv = container_of(work, struct apple_nvme, recovery_work); @@ -1647,6 +1695,8 @@ static void apple_nvme_stop_crashed_session(struct apple_nvme *anv) anv->ctrl.ctrl_config = 0; } +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv); + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1668,6 +1718,17 @@ static void apple_nvme_reset_work(struct work_struct *work) return; } WRITE_ONCE(anv->abort_reset, false); + if (READ_ONCE(anv->sleep_restarting) && !apple_nvme_sleep_wait_on(anv)) + WRITE_ONCE(anv->sleep_failed, true); + if (READ_ONCE(anv->sleep_failed)) { + /* + * Starting it some other way can only reset it, which faults, and failing the + * controller fails every held write. Leave the queues quiesced and frozen. + */ + if (phase_locked) + mutex_unlock(&anv->disable_lock); + return; + } if (state != NVME_CTRL_RESETTING) { dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state); ret = -ENODEV; @@ -1735,6 +1796,12 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = -EIO; goto out; } + /* A real sleep took the controller's state with its power. */ + if (READ_ONCE(anv->sleep_restarted)) { + WRITE_ONCE(anv->sleep_restarted, false); + if (!(readl(anv->mmio_nvme + NVME_REG_CC) & NVME_CC_ENABLE)) + goto rtkit_ready; + } ret = apple_nvme_disable_locked(anv, false); if (ret) goto out; @@ -2595,6 +2662,49 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) return ERR_PTR(ret); } +/* + * The sleep's last check, with interrupts off: the shutdown notification sent at suspend + * (apple_nvme_sleep_send_shn()) has completed. Nothing is written here. + */ +static int apple_nvme_sleep_shutdown(void *data) +{ + struct apple_nvme *anv = data; + u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + + anv->sleep_csts = csts; + if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) + return 0; + dev_err(anv->dev, "shutdown notification not complete at the commit: CSTS %#x\n", csts); + return -EIO; +} + +/* + * The rest of a sleep's stop, last of all. From the first idle message only a real sleep + * brings it back, so a failure goes on to the commit: what a flush acknowledged is on + * flash, and the watchdog is the net. + */ +static int apple_nvme_sleep_last_stop(void *data) +{ + struct apple_nvme *anv = data; + unsigned int us; + u32 cc, csts = anv->sleep_csts; + int ret; + + cc = readl(anv->mmio_nvme + NVME_REG_CC); + + anv->sleep_idled = true; + ret = apple_rtkit_sleep_idle_polled(anv->rtk, + anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL, &us); + if (ret) + dev_err(anv->dev, "ANS did not idle: %d\n", ret); + else + dev_dbg(anv->dev, "ANS idle, acknowledged in about %u us, CC %#x CSTS %#x\n", + us, cc, csts); + apple_rtkit_mailbox_recv_off(anv->rtk); + + return ret ? 1 : 0; +} + static int apple_nvme_probe(struct platform_device *pdev) { struct apple_nvme *anv; @@ -2623,6 +2733,19 @@ static int apple_nvme_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + if (anv->hw->sleep_traced) { + anv->sleep_registered = + !apple_sleep_coproc_register(anv->dev, anv->mmio_coproc, true, + apple_nvme_sleep_last_stop, anv); + /* Never idled without its shutdown notification. */ + if (anv->sleep_registered && + apple_sleep_coproc_set_last_check(anv->dev, apple_nvme_sleep_shutdown)) { + apple_sleep_coproc_unregister(anv->dev); + anv->sleep_registered = false; + } + apple_rtkit_set_warm_epmap_mask(anv->rtk, anv->hw->sleep_epmap_last_mask); + } + apple_nvme_set_reset_ready(anv, true); nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -2641,6 +2764,8 @@ static void apple_nvme_remove(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + if (anv->sleep_registered) + apple_sleep_coproc_unregister(anv->dev); WRITE_ONCE(anv->removing, true); apple_nvme_set_reset_ready(anv, false); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); @@ -2706,6 +2831,83 @@ static void apple_nvme_shutdown(struct platform_device *pdev) } } +/* macOS sees the firmware report on 68 ms after HELLO. Each try waits RTKit's timeout again. */ +#define APPLE_NVME_SLEEP_ON_TRIES 15 + +/* + * After a real sleep ANS has lost power, and its resident firmware restarts once the run bit + * is set. Measured on macOS under aura, the same in five runs: HELLO first and unprompted, + * then the endpoint map, the last part answered with endpoint 0x20 only. No endpoint is + * started and no buffer asked for. About 68 ms later the firmware reports on, and the AP state + * follows, with the controller's registers reading 0. A refused suspend never told the + * firmware anything, and takes a reset. + * + * The run bit goes in the early phase, so HELLO comes within milliseconds of the wake as on + * macOS. The wait for on doesn't: macOS rings the SEP a few ms after ANS's HELLO, before ANS + * reports on, so the reset work waits, after the SEP's own resume has had its turn. + */ +static int apple_nvme_resume_early(struct device *dev) +{ + struct apple_nvme *anv = dev_get_drvdata(dev); + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret; + + + if (!anv->hw->needs_ioq_registers || !anv->suspended || !anv->sleep_idled) + return 0; + anv->sleep_idled = false; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + /* Its link doesn't come up with the pins as the sleep left them. */ + apple_sleep_restore_pins(); + /* macOS writes the SART's entries back again right before the run bit. */ + if (anv->sart) + apple_sart_replay(anv->sart); + + dev_dbg(dev, "ANS: restarting after the sleep, CPU control %#x\n", readl(control)); + anv->sleep_run_time = ktime_get(); + ret = apple_rtkit_sleep_return_start(anv->rtk, control); + if (ret) { + dev_crit(dev, "ANS restart after sleep failed: %d\n", ret); + anv->sleep_failed = true; + return 0; + } + WRITE_ONCE(anv->sleep_restarting, true); + + return 0; +} + +/* From the reset work: wait for ANS to report on. False means hold every request. */ +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv) +{ + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret, tries = 0; + + do { + ret = apple_rtkit_boot(anv->rtk); + if (ret != -ETIME) + break; + dev_warn(anv->dev, "ANS: not on %lld ms after the run bit, waiting again\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + } while (++tries < APPLE_NVME_SLEEP_ON_TRIES); + + WRITE_ONCE(anv->sleep_restarting, false); + if (ret) { + dev_crit(anv->dev, "ANS did not come back after sleep: %d after %lld ms, CPU control %#x status %#x, crashed %d\n", + ret, ktime_ms_delta(ktime_get(), anv->sleep_run_time), readl(control), + readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_STATUS), + apple_rtkit_is_crashed(anv->rtk)); + dev_crit(anv->dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return false; + } + WRITE_ONCE(anv->sleep_restarted, true); + /* The one line per wake that says ANS is back. */ + dev_info(anv->dev, "ANS: on %lld ms after the run bit, resetting the controller before any I/O\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + + return true; +} + static int apple_nvme_resume(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2715,6 +2917,20 @@ static int apple_nvme_resume(struct device *dev) return -EIO; if (!anv->suspended) return 0; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + anv->sleep_idled = false; + anv->sleep_stop = false; + /* + * Hold every request rather than fail it: the queues stay frozen and quiesced + * from the suspend, so nothing reaches a controller that isn't there and no + * filesystem sees a write error. Only a reboot brings ANS back from this. + */ + if (READ_ONCE(anv->sleep_failed)) { + dev_crit(dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return 0; + } + /* The reset work waits for ANS first, if it's still coming up. */ WRITE_ONCE(anv->suspended, false); return nvme_try_sched_reset(&anv->ctrl); } @@ -2743,19 +2959,106 @@ static void apple_nvme_keep_link_powered(struct apple_nvme *anv) device_set_awake_path(anv->pd_dev[i]); } +/* + * What is written has to be on flash before power goes: an fsynced overwrite is + * known to revert after an abrupt reset on these controllers unless flushed. + */ +static int apple_nvme_sleep_flush(struct apple_nvme *anv) +{ + struct nvme_ctrl *ctrl = &anv->ctrl; + struct nvme_ns *ns; + int srcu_idx, ret = 0; + + srcu_idx = srcu_read_lock(&ctrl->srcu); + list_for_each_entry_srcu(ns, &ctrl->namespaces, list, + srcu_read_lock_held(&ctrl->srcu)) { + struct nvme_command c = { }; + + if (ns->head->ns_id > anv->hw->sleep_flush_max_nsid) + continue; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(ns->head->ns_id); + ret = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (ret) + break; + } + srcu_read_unlock(&ctrl->srcu, srcu_idx); + + return ret; +} + +static bool apple_nvme_sleep_wanted(struct apple_nvme *anv) +{ + return anv->sleep_registered && + pm_suspend_target_state == PM_SUSPEND_MEM && pm_suspend_via_firmware() && + nvme_ctrl_state(&anv->ctrl) == NVME_CTRL_LIVE && anv->owns_rtkit && + apple_rtkit_is_running(anv->rtk); +} + +/* + * As macOS sends it, from its power-state change during the device suspend: the shutdown + * notification, waited for with interrupts on for as long as macOS allows (35 s). The + * controller can take seconds to save its context (1.7 s in one macOS sleep). A sleep + * refused later finds it shut down, and resume's controller reset takes it back. + */ +#define APPLE_ANS_SLEEP_SHN_MACOS_TIMEOUT_US (35 * USEC_PER_SEC) + +static int apple_nvme_sleep_send_shn(struct apple_nvme *anv) +{ + ktime_t start = ktime_get(); + u32 csts; + s64 us; + int ret; + + anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; + anv->ctrl.ctrl_config |= anv->hw->sleep_shn; + writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); + ret = readl_poll_timeout(anv->mmio_nvme + NVME_REG_CSTS, csts, + (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, + 1000, APPLE_ANS_SLEEP_SHN_MACOS_TIMEOUT_US); + us = ktime_us_delta(ktime_get(), start); + if (ret) + dev_err(anv->dev, "shutdown notification failed after %lld ms: CSTS %#x\n", + div_s64(us, USEC_PER_MSEC), csts); + else if (us > USEC_PER_SEC) + dev_info(anv->dev, "shutdown notification took %lld ms\n", div_s64(us, USEC_PER_MSEC)); + return ret; +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); int ret; if (anv->hw->needs_ioq_registers) { - if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) + /* A sleep right after a wake, as from a dark wake: let that reset finish first. */ + flush_work(&anv->ctrl.reset_work); + anv->sleep_stop = apple_nvme_sleep_wanted(anv); + anv->sleep_idled = false; + anv->sleep_failed = false; + anv->sleep_restarted = false; + if (anv->sleep_stop) { + ret = apple_nvme_sleep_flush(anv); + if (ret) { + dev_err(dev, "refusing to sleep: flush failed: %d\n", ret); + anv->sleep_stop = false; + return -EIO; + } + } + if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) { + anv->sleep_stop = false; return -EBUSY; + } WRITE_ONCE(anv->suspended, true); } - ret = apple_nvme_disable(anv, true); - if (ret) - return ret; + if (anv->sleep_stop) { + anv->sleep_failed = apple_nvme_sleep_quiesce(anv); + } else { + ret = apple_nvme_disable(anv, true); + if (ret) + return ret; + } /* * Only the NVMe controller was disabled. Its owned RTKit session stays * live, so resume rebuilds queues without clearing CPU_CONTROL.RUN. @@ -2763,6 +3066,26 @@ static int apple_nvme_suspend(struct device *dev) if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); apple_nvme_keep_link_powered(anv); + if (anv->sleep_failed) { + /* Requests are stuck, so bring it back and refuse. */ + dev_err(dev, "refusing to sleep: NVMe didn't quiesce\n"); + anv->sleep_failed = false; + anv->sleep_stop = false; + WRITE_ONCE(anv->suspended, false); + nvme_try_sched_reset(&anv->ctrl); + return -EIO; + } + if (anv->sleep_stop && apple_nvme_sleep_send_shn(anv)) { + dev_err(dev, "refusing to sleep: NVMe shutdown notification didn't complete\n"); + anv->sleep_stop = false; + WRITE_ONCE(anv->suspended, false); + nvme_try_sched_reset(&anv->ctrl); + return -EIO; + } + if (anv->sleep_stop) { + apple_sleep_coproc_set_ready(dev, true); + dev_dbg(dev, "NVMe suspended for a sleep\n"); + } return 0; } @@ -2775,8 +3098,10 @@ static int apple_nvme_suspend(struct device *dev) return ret; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, - apple_nvme_resume); +static const struct dev_pm_ops apple_nvme_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_nvme_suspend, apple_nvme_resume) + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_nvme_resume_early) +}; static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, @@ -2809,6 +3134,10 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .has_separate_nvmmu = true, .needs_ioq_registers = true, .max_queue_depth = 64, + .sleep_traced = true, + .sleep_epmap_last_mask = BIT(0), + .sleep_flush_max_nsid = 1, + .sleep_shn = NVME_CC_SHN_MASK, /* * An fsynced overwrite completed and later reverted after an abrupt * reset; explicit flushes persisted the preceding data. Let the block diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h index c5e7f1df4da5b..bf97e6cf081bb 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h @@ -17,7 +17,10 @@ #define PIODMA_IRQ_COMPLETE APPLE_PIODMA_IRQ_COMPLETION #define PIODMA_IRQ_LIMIT 16 -/* Only RELEASE before publication can end guard ownership. No reset event. */ +/* + * Only RELEASE before publication can end guard ownership, or POWER_LOST: the + * engine's whole register file reading as reset after the SoC powered it off. + */ enum apple_piodma_state { PIODMA_IDLE, PIODMA_GUARDED, @@ -39,6 +42,7 @@ enum apple_piodma_event { PIODMA_COMPLETE, PIODMA_ADMIT, PIODMA_FAIL, + PIODMA_POWER_LOST, PIODMA_EVENT_COUNT, }; @@ -107,6 +111,10 @@ static inline int apple_piodma_next(enum apple_piodma_state state, if (state == PIODMA_RETAINED) return PIODMA_FAILED_RETAINED; break; + case PIODMA_POWER_LOST: + if (state == PIODMA_ADMITTED) + return PIODMA_GUARDED; + break; default: break; } diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c index 941a1786c0483..9cccb3ca4bcd0 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c @@ -61,12 +61,12 @@ static void apple_piodma_transition_guards_test(struct kunit *test) { int state, event, next; - /* Exhaust all transitions: no retained state can release or retry. */ + /* Exhaust all transitions: only a power loss can end a retained state. */ for (state = PIODMA_IDLE; state < PIODMA_STATE_COUNT; state++) { for (event = PIODMA_GUARD; event < PIODMA_EVENT_COUNT; event++) { next = apple_piodma_next(state, event); if (apple_piodma_retained(state)) { - if (next >= 0) + if (next >= 0 && event != PIODMA_POWER_LOST) KUNIT_EXPECT_TRUE(test, apple_piodma_retained(next)); if (event == PIODMA_RELEASE || event == PIODMA_PRIME || event == PIODMA_GUARD) @@ -128,8 +128,24 @@ static void apple_piodma_irq_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 1, 2)); } +/* Losing power reopens the guard from an admitted radio, and from nothing else. */ +static void apple_piodma_power_lost_test(struct kunit *test) +{ + static const enum apple_piodma_state refused[] = { + PIODMA_IDLE, PIODMA_GUARDED, PIODMA_ATTEMPTED, PIODMA_FAILED, + PIODMA_REJECTED, PIODMA_RETAINED, PIODMA_READY, PIODMA_FAILED_RETAINED, + }; + int i; + + KUNIT_EXPECT_EQ(test, apple_piodma_next(PIODMA_ADMITTED, PIODMA_POWER_LOST), + PIODMA_GUARDED); + for (i = 0; i < ARRAY_SIZE(refused); i++) + KUNIT_EXPECT_LT(test, apple_piodma_next(refused[i], PIODMA_POWER_LOST), 0); +} + static struct kunit_case apple_piodma_cases[] = { KUNIT_CASE(apple_piodma_lifecycle_test), + KUNIT_CASE(apple_piodma_power_lost_test), KUNIT_CASE(apple_piodma_failure_test), KUNIT_CASE(apple_piodma_transition_guards_test), KUNIT_CASE(apple_piodma_arena_test), diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c index a95dd5112f201..9261d23abee16 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c @@ -92,6 +92,16 @@ struct apple_t8140_bootstrap { struct kexec_blocker kexec_blocker; struct delayed_work admission_retry; struct device *retry_host; + /* The host stops the radios for a sleep that takes this engine's power. */ + bool sleep_armed; + bool power_lost; + /* After a real sleep: read the engine once the host's port is up again. */ + bool power_check; + bool irq_held; + int (*radio_stop)(struct pci_dev *pdev); + struct module *radio_stop_owner; + void (*radio_recover)(void *data); + void *radio_recover_data; }; /* A successful supplier probe owns its unpublished arena until full reset. */ @@ -193,6 +203,8 @@ static int apple_piodma_pm_prepare(struct device *dev) return 0; if (apple_piodma_sleep_allowed(apple_piodma_state(bootstrap), pm_suspend_no_platform())) return 0; + if (READ_ONCE(bootstrap->sleep_armed)) + return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this * supplier probed. Its later image/restore/suspend ioctls go straight @@ -372,6 +384,10 @@ static irqreturn_t apple_t8140_bootstrap_irq(int irq, void *data) u32 fabric, status, tag, active; int count; + /* After a real sleep its block can't answer until the host's port is back. */ + if (READ_ONCE(bootstrap->irq_held)) + return IRQ_NONE; + /* Acknowledge order follows the attributed prior controller contract. */ fabric = readl(bootstrap->fabric); writel(fabric, bootstrap->fabric); @@ -744,6 +760,60 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + WRITE_ONCE(bootstrap->sleep_armed, armed); + /* A check still pending means the block never came back, and stays unread. */ + if (armed) + bootstrap->power_lost = false; +} + +/* The next bootstrap starts as the first did, on the same retained memory. */ +int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + void *arena = bootstrap->arena; + int i; + + /* + * Everything reading as it did before the first command is what shows the sleep took + * the engine's pointers with it. Read only now: the block answers once the host's + * port is set up, after resume_noirq. + */ + if (bootstrap->power_check) { + bootstrap->power_check = false; + bootstrap->power_lost = !apple_t8140_bootstrap_idle(bootstrap); + if (!bootstrap->power_lost) + dev_err(supplier, "engine kept state across sleep; radios stay down\n"); + WRITE_ONCE(bootstrap->irq_held, false); + } + if (!bootstrap->power_lost || READ_ONCE(bootstrap->irq_fault)) + return -EIO; + bootstrap->power_lost = false; + + memset(arena, 0, PIODMA_ARENA_SIZE); + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { + void *slot = arena + i * PIODMA_SLOT_SIZE; + + memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); + *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); + memset(bootstrap->slots[i].packet, 0, sizeof(bootstrap->slots[i].packet)); + memset(bootstrap->slots[i].completion, 0, sizeof(bootstrap->slots[i].completion)); + bootstrap->slots[i].result = 0; + WRITE_ONCE(bootstrap->slots[i].matched_status, 0); + bootstrap->slots[i].submitted = false; + bootstrap->slots[i].observed = false; + reinit_completion(&bootstrap->slots[i].event); + } + dma_wmb(); + WRITE_ONCE(bootstrap->active_tag, 0); + atomic_set(&bootstrap->irq_count, 0); + + return apple_piodma_transition(bootstrap, PIODMA_POWER_LOST); +} + bool apple_piodma_bootstrap_retained(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -753,6 +823,43 @@ bool apple_piodma_bootstrap_retained(struct device *supplier) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); +/* The stop and the module it's in, set and read together. */ +static DEFINE_SPINLOCK(radio_stop_lock); + +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + int (*stop)(struct pci_dev *pdev); + struct module *owner; + bool held; + int ret; + + /* Its module stays while it runs. One that is unloading: its remove stops the radio. */ + spin_lock(&radio_stop_lock); + stop = bootstrap->radio_stop; + owner = bootstrap->radio_stop_owner; + held = !stop || try_module_get(owner); + spin_unlock(&radio_stop_lock); + if (!held) + return -EBUSY; + if (!stop) + return 0; + + ret = stop(pdev); + module_put(owner); + + return ret; +} + +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + bootstrap->radio_recover_data = data; + WRITE_ONCE(bootstrap->radio_recover, recover); +} + void apple_piodma_bootstrap_admitted(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -790,6 +897,31 @@ int apple_piodma_radio_check(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(apple_piodma_radio_check); +void apple_piodma_radio_recover(void) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + void (*recover)(void *data) = bootstrap ? READ_ONCE(bootstrap->radio_recover) : NULL; + + if (recover) + recover(bootstrap->radio_recover_data); +} +EXPORT_SYMBOL_GPL(apple_piodma_radio_recover); + +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + + if (!bootstrap) + return; + spin_lock(&radio_stop_lock); + bootstrap->radio_stop = stop; + bootstrap->radio_stop_owner = owner; + spin_unlock(&radio_stop_lock); +} +EXPORT_SYMBOL_GPL(__apple_piodma_radio_set_stop); + static int apple_t8140_bootstrap_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -822,7 +954,8 @@ static int apple_t8140_bootstrap_probe(struct platform_device *pdev) ret = apple_t8140_bootstrap_resources(pdev, bootstrap); if (ret) goto err_arena; - ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, 0, + /* The host's resume_noirq waits for one completion, with device interrupts still off. */ + ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, IRQF_EARLY_RESUME, dev_name(dev), bootstrap); if (ret) goto err_arena; @@ -873,12 +1006,20 @@ static int apple_t8140_bootstrap_sleep_check(struct apple_t8140_bootstrap *boots static int apple_t8140_bootstrap_suspend_noirq(struct device *dev) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); + int ret; if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !READ_ONCE(bootstrap->sleep_armed)) return -EOPNOTSUPP; - return apple_t8140_bootstrap_sleep_check(bootstrap); + /* Radios that failed to come back don't cost the machine a deep sleep. */ + if (READ_ONCE(bootstrap->sleep_armed) && + (!apple_piodma_ready(apple_piodma_state(bootstrap)) || bootstrap->power_check)) + return 0; + ret = apple_t8140_bootstrap_sleep_check(bootstrap); + if (!ret && READ_ONCE(bootstrap->sleep_armed)) + WRITE_ONCE(bootstrap->irq_held, true); + return ret; } static int apple_t8140_bootstrap_resume_noirq(struct device *dev) @@ -890,6 +1031,15 @@ static int apple_t8140_bootstrap_resume_noirq(struct device *dev) return 0; if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; + /* + * Its block reads nothing until the host sets its port up again, so neither read it + * nor take its interrupt until then: apple_piodma_bootstrap_power_lost() checks it. + */ + if (READ_ONCE(bootstrap->sleep_armed) && pm_resume_via_firmware()) { + bootstrap->power_check = true; + return 0; + } + WRITE_ONCE(bootstrap->irq_held, false); /* Genpd may gate this island; restore configuration, never a command. */ ret = apple_t8140_bootstrap_initialize(bootstrap); if (ret) diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h index 8509283b46068..5536059e4e1db 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h @@ -116,6 +116,11 @@ void apple_piodma_bootstrap_guard_end(struct device *supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed); +int apple_piodma_bootstrap_power_lost(struct device *supplier); +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev); +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data); #else static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { @@ -145,5 +150,24 @@ static inline bool apple_piodma_bootstrap_retained(struct device *supplier) { return false; } + +static inline void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ +} + +static inline int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + return -ENODEV; +} + +static inline void apple_piodma_bootstrap_set_recover(struct device *supplier, + void (*recover)(void *data), void *data) +{ +} #endif #endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 9fe2ca34bd014..da7142251f64d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -516,6 +517,19 @@ struct apple_pcie { bool neo_config_ready; u8 neo_secondary_bus; struct device *piodma_supplier; + /* What it takes to come back from a deep sleep, if the loader gave it. */ + struct apple_pcie_neo_cold *neo_cold; + struct notifier_block neo_pm_nb; + bool neo_sleep_armed; + struct work_struct neo_recover_work; + bool neo_recover; + /* The functions' state from the last sleep that found them answering */ + struct pci_saved_state *neo_fn_state[2]; + /* Their MSI-X, set aside while they're out of reach */ + bool neo_fn_msix[2]; + bool neo_rc_down; + /* Down already when the sleep began: a failed wake left it unprogrammed */ + bool neo_rc_was_down; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -544,6 +558,8 @@ struct apple_pcie_port { unsigned int link_irqs[2]; u32 saved_intmask; int sid_map_sz; + struct gpio_desc *perst_gpio; + struct gpio_desc *pwren_gpio; int idx; bool started; bool needs_stop; @@ -2696,6 +2712,13 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, u32 stat; int ret; + /* Bringing the link up again, after a deep sleep. */ + if (port->perst_gpio) { + reset = port->perst_gpio; + pwren = port->pwren_gpio; + goto have_gpios; + } + /* * Assert PERST# and configure the pin as output. * The Aquantia AQC113 10GB nic used desktop macs is sensitive to @@ -2733,7 +2756,12 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, else return PTR_ERR(pwren); } + if (!num_aux_resets) { + port->perst_gpio = reset; + port->pwren_gpio = pwren; + } +have_gpios: rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ @@ -3717,6 +3745,10 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); + /* Nothing answers on a host that has lost power and isn't set up again. */ + if (READ_ONCE(pcie->neo_rc_down)) + return NULL; + if (bus->number != cfg->busr.start) { /* Pair with ready publication before consuming the bus number. */ if (!smp_load_acquire(&pcie->neo_config_ready) || @@ -4286,6 +4318,233 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) return ret; } +/* + * What the bootloader did to this host at boot, on the same registers, in its + * order, with its tunables. On this SoC a deep sleep powers all of it off. + */ +struct apple_pcie_neo_cold { + void __iomem *axi2af; + void __iomem *cio3pll; + void __iomem *clkgen; + void __iomem *phy_block; + struct apple_tunable *axi2af_tunable; + struct apple_tunable *cio3pll_tunable; + struct apple_tunable *clkgen_tunable; + struct apple_tunable *phy_tunable; + struct apple_tunable *port_tunable; + struct apple_tunable *rc_tunable; + struct apple_tunable *rc_gen3_tunable; + struct apple_tunable *rc_gen4_tunable; + /* The port's DART, clocked by the port: restored after the port is set up. */ + struct device *dart; +}; + +#define NEO_PHY_CTRL 0x0 +#define NEO_PHY_CTRL_CLK0REQ BIT(0) +#define NEO_PHY_CTRL_CLK1REQ BIT(1) +#define NEO_PHY_CTRL_CLK0ACK BIT(2) +#define NEO_PHY_CTRL_CLK1ACK BIT(3) +#define NEO_PHY_CTRL_RESET BIT(4) +#define NEO_PHY_RUN 0x4 +#define NEO_PHY_READY 0x8 +#define NEO_PHY_CTRL_UP BIT(27) +#define NEO_PHY_SETTLE_US 300 +#define NEO_AXI_EN0 0x104 +#define NEO_AXI_EN1 0x108 +#define NEO_CLK_CTRL 0x0 +#define NEO_CLK_BUSY BIT(31) +#define NEO_CLK_OFF BIT(2) +#define NEO_CLK_START 0x4 +#define NEO_CLK_START_GO BIT(31) +#define NEO_PHY_BLOCK_COMMON 0x4000 +#define NEO_PHY_BLOCK_PHY 0x8000 +#define NEO_PHY_POLL_US 50000 +#define NEO_RC_WRITE_ZERO 0x04 +#define NEO_RC_READY_CFG 0x54 +#define NEO_RC_READY_START 0x50 +#define NEO_RC_READY_STAT 0x58 +#define NEO_RC_POLL_US 250000 +#define NEO_DBI_CAP_LNKCAP 0x7c +#define NEO_DBI_CAP_LNKCAP2 0x9c +#define NEO_DBI_LNKCAP2_SLS GENMASK(6, 1) +#define NEO_DBI_CAP_LNKCTL2 0xa0 +#define NEO_DBI_PORT_LINK_CONTROL 0x710 +#define NEO_DBI_PORT_LINK_MODE GENMASK(21, 16) +#define NEO_DBI_WIDTH_SPEED_CONTROL 0x80c +#define NEO_DBI_LINK_WIDTH GENMASK(12, 8) +#define NEO_DBI_SPEED_CHANGE BIT(17) +#define NEO_DBI_RO_WR 0x8bc +#define NEO_DBI_RO_WR_EN BIT(0) + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001000ff }, + { 0x82c, 0x00000000 }, +}; + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs2[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static int apple_pcie_neo_phy_clocks(void __iomem *phy) +{ + u32 stat; + int ret; + + rmw_set(NEO_PHY_CTRL_CLK0REQ, phy + NEO_PHY_CTRL); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK0ACK, + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(NEO_PHY_CTRL_CLK1REQ, phy + NEO_PHY_CTRL); + return readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK1ACK, + 10, NEO_PHY_POLL_US); +} + +/* + * The part of the block that ANS shares, from power-off in macOS's order, measured on + * one sleep. The sleep driver does it at the wake when ANS is bound, and it can't be + * redone under a running PHY. + */ +static int apple_pcie_neo_common_init(struct apple_pcie *pcie) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *phy = cold->phy_block + NEO_PHY_BLOCK_PHY; + u32 stat, val; + int ret; + + if (readl_relaxed(pcie->base + NEO_RC_READY_STAT) & BIT(0)) + return 0; + + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN0); + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN1); + apple_tunable_apply(cold->axi2af, cold->axi2af_tunable); + apple_tunable_apply(cold->cio3pll, cold->cio3pll_tunable); + apple_tunable_apply(cold->clkgen, cold->clkgen_tunable); + rmw_set(NEO_CLK_START_GO, cold->clkgen + NEO_CLK_START); + rmw_clear(NEO_CLK_OFF, cold->clkgen + NEO_CLK_CTRL); + ret = readl_relaxed_poll_timeout(cold->clkgen + NEO_CLK_CTRL, stat, !(stat & NEO_CLK_BUSY), + 1, NEO_PHY_POLL_US); + if (ret) + return ret; + apple_tunable_apply(cold->phy_block, cold->phy_tunable); + + ret = apple_pcie_neo_phy_clocks(phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, phy + NEO_PHY_CTRL); + udelay(1); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, !(stat & NEO_PHY_CTRL_RESET), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(BIT(0), phy + NEO_PHY_RUN); + udelay(NEO_PHY_SETTLE_US); + rmw_set(BIT(4), phy + NEO_PHY_RUN); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_READY, stat, stat & BIT(0), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + + val = readl_relaxed(cold->phy_block + NEO_PHY_BLOCK_COMMON); + writel_relaxed((val & ~GENMASK(1, 0)) | 1, cold->phy_block + NEO_PHY_BLOCK_COMMON); + rmw_set(NEO_PHY_CTRL_UP, phy + NEO_PHY_CTRL); + + writel_relaxed(0, pcie->base + NEO_RC_WRITE_ZERO); + writel_relaxed(0x140, pcie->base + NEO_RC_READY_CFG); + writel_relaxed(1, pcie->base + NEO_RC_READY_START); + + return readl_relaxed_poll_timeout(pcie->base + NEO_RC_READY_STAT, stat, stat & BIT(0), + 100, NEO_RC_POLL_US); +} + +static int apple_pcie_neo_cold_init(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + int speed = of_pci_get_max_link_speed(port->np); + u32 stat, val; + int i, ret; + + ret = apple_pcie_neo_common_init(pcie); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs[i].value, + apple_pcie_neo_port_regs[i].offset); + for (i = 0; i < 16; i++) + apple_pcie_port_writel(port, 0, 0x3000 + 4 * i); + for (i = 0; i < 512; i++) + apple_pcie_port_writel(port, 0, pcie->hw->port_msimap + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs2); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs2[i].value, + apple_pcie_neo_port_regs2[i].offset); + apple_tunable_apply(port->base, cold->port_tunable); + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* PERST# off before the PHY's clocks, as macOS has it coming from power-off. */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, NEO_RC_POLL_US); + if (ret) + return ret; + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, + !(stat & PORT_LINKSTS_BUSY), 100, NEO_RC_POLL_US); + if (ret) + return ret; + + rmw_clear(NEO_PHY_CTRL_CLK0REQ | NEO_PHY_CTRL_CLK1REQ, port->phy + NEO_PHY_CTRL); + ret = apple_pcie_neo_phy_clocks(port->phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, port->phy + NEO_PHY_CTRL); + rmw_set(BIT(9), port->phy + NEO_PHY_CTRL); + rmw_set(BIT(10), port->phy + NEO_PHY_CTRL); + + rmw_set(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + apple_tunable_apply(dbi, cold->rc_tunable); + apple_tunable_apply(dbi, cold->rc_gen3_tunable); + apple_tunable_apply(dbi, cold->rc_gen4_tunable); + if (speed > 0) { + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP); + writel_relaxed((val & ~PCI_EXP_LNKCAP_SLS) | speed, dbi + NEO_DBI_CAP_LNKCAP); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP2); + writel_relaxed((val & ~NEO_DBI_LNKCAP2_SLS) | + FIELD_PREP(NEO_DBI_LNKCAP2_SLS, (1 << speed) - 1), + dbi + NEO_DBI_CAP_LNKCAP2); + writew_relaxed((readw_relaxed(dbi + NEO_DBI_CAP_LNKCTL2) & ~PCI_EXP_LNKCTL2_TLS) | + speed, dbi + NEO_DBI_CAP_LNKCTL2); + rmw_set(NEO_DBI_SPEED_CHANGE, dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + } + val = readl_relaxed(dbi + NEO_DBI_PORT_LINK_CONTROL) & ~NEO_DBI_PORT_LINK_MODE; + writel_relaxed(val | FIELD_PREP(NEO_DBI_PORT_LINK_MODE, 1), + dbi + NEO_DBI_PORT_LINK_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_WIDTH_SPEED_CONTROL) & ~NEO_DBI_LINK_WIDTH; + writel_relaxed(val | FIELD_PREP(NEO_DBI_LINK_WIDTH, 1), dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP) & ~PCI_EXP_LNKCAP_MLW; + writel_relaxed(val | FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1), dbi + NEO_DBI_CAP_LNKCAP); + rmw_clear(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + + return 0; +} + static int apple_pcie_attach_domains(struct apple_pcie *pcie) { struct device *dev = pcie->dev; @@ -4324,6 +4583,169 @@ static int apple_pcie_attach_domains(struct apple_pcie *pcie) return 0; } +static void apple_pcie_neo_recover_work(struct work_struct *work); + +/* A radio left unusable by the wake, or its link. POST_SUSPEND runs this once all is thawed. */ +static void apple_pcie_neo_recover(void *data) +{ + struct apple_pcie *pcie = data; + + WRITE_ONCE(pcie->neo_recover, true); + /* Pairs with the barrier at PM_POST_SUSPEND: one side sees the other's write. */ + smp_mb(); + if (!READ_ONCE(pcie->neo_sleep_armed)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); +} + +/* Runs before the freezer. The radios stay bound, and their drivers do their own suspend. */ +static int apple_pcie_neo_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct apple_pcie *pcie = container_of(nb, struct apple_pcie, neo_pm_nb); + + switch (action) { + case PM_SUSPEND_PREPARE: + if (pm_suspend_default_s2idle()) + break; + flush_work(&pcie->neo_recover_work); + pcie->neo_sleep_armed = true; + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, true); + break; + case PM_POST_SUSPEND: + if (!pcie->neo_sleep_armed) + break; + WRITE_ONCE(pcie->neo_sleep_armed, false); + /* Pairs with the barrier in the recovery request. */ + smp_mb(); + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, false); + if (READ_ONCE(pcie->neo_recover)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +static void __iomem *apple_pcie_neo_map(struct platform_device *pdev, const char *name, + struct resource **res) +{ + *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + if (!*res) + return IOMEM_ERR_PTR(-ENOENT); + + /* These overlap each other and the port PHYs, so they can't be claimed. */ + return devm_ioremap(&pdev->dev, (*res)->start, resource_size(*res)) ?: + IOMEM_ERR_PTR(-ENOMEM); +} + +/* A shared block and the tunable the loader publishes for it. */ +static int apple_pcie_neo_get(struct platform_device *pdev, const char *name, const char *prop, + void __iomem **base, struct apple_tunable **tunable) +{ + struct resource *res; + + *base = apple_pcie_neo_map(pdev, name, &res); + if (IS_ERR(*base)) + return PTR_ERR(*base); + *tunable = devm_apple_tunable_parse(&pdev->dev, pdev->dev.of_node, prop, res); + + return PTR_ERR_OR_ZERO(*tunable); +} + +/* The DART in the host's iommu-map, which reads nothing until the port is set up. */ +static struct device *apple_pcie_neo_dart(struct device *dev) +{ + struct device_node *np = of_parse_phandle(dev->of_node, "iommu-map", 1); + struct platform_device *pdev = np ? of_find_device_by_node(np) : NULL; + int ret; + + of_node_put(np); + if (!pdev) + return NULL; + /* The reference is kept: this host can't be unbound. */ + ret = apple_dart_set_host_resumes(&pdev->dev); + /* The DART resumes, and gates itself, before this host restores it. */ + if (!ret && !device_link_add(dev, &pdev->dev, DL_FLAG_STATELESS)) + ret = -EINVAL; + if (ret) { + put_device(&pdev->dev); + dev_warn(dev, "no deep sleep: its DART can't wait for the port: %d\n", ret); + return NULL; + } + return &pdev->dev; +} + +/* Without all of this the host can't come back, and a deep sleep stays refused. */ +static void apple_pcie_neo_sleep_probe(struct platform_device *pdev, struct apple_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pcie_neo_cold *cold; + struct apple_pcie_port *port; + struct resource *res, *cfg; + int ret; + + port = list_first_entry_or_null(&pcie->ports, struct apple_pcie_port, entry); + cfg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!port || !cfg || !port->perst_gpio || !port->phy) { + dev_warn(dev, "no deep sleep: nothing here took the link up, so no pins to restart it\n"); + return; + } + cold = devm_kzalloc(dev, sizeof(*cold), GFP_KERNEL); + if (!cold) + return; + + if (apple_pcie_neo_get(pdev, "axi2af", "apple,tunable-axi2af", &cold->axi2af, + &cold->axi2af_tunable) || + apple_pcie_neo_get(pdev, "cio3pll", "apple,tunable-cio3pll", &cold->cio3pll, + &cold->cio3pll_tunable) || + apple_pcie_neo_get(pdev, "clkgen", "apple,tunable-clkgen", &cold->clkgen, + &cold->clkgen_tunable)) + goto absent; + cold->phy_block = apple_pcie_neo_map(pdev, "phy-block", &res); + if (IS_ERR(cold->phy_block)) + goto absent; + cold->phy_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-phy", res); + if (IS_ERR(cold->phy_tunable)) + goto absent; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "port0"); + if (!res) + goto absent; + cold->port_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-port", res); + if (IS_ERR(cold->port_tunable)) + goto absent; + cold->rc_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc", cfg); + if (IS_ERR(cold->rc_tunable)) + goto absent; + cold->rc_gen3_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen3", + cfg); + if (IS_ERR(cold->rc_gen3_tunable)) + goto absent; + cold->rc_gen4_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen4", + cfg); + if (IS_ERR(cold->rc_gen4_tunable)) + goto absent; + cold->dart = apple_pcie_neo_dart(dev); + if (!cold->dart) + goto absent; + + pcie->neo_cold = cold; + INIT_WORK(&pcie->neo_recover_work, apple_pcie_neo_recover_work); + apple_piodma_bootstrap_set_recover(pcie->piodma_supplier, apple_pcie_neo_recover, pcie); + pcie->neo_pm_nb.notifier_call = apple_pcie_neo_pm_notify; + ret = register_pm_notifier(&pcie->neo_pm_nb); + if (ret) + pcie->neo_cold = NULL; + else + dev_dbg(dev, "deep sleep: radios stay powered, link stopped and retrained\n"); + return; + +absent: + dev_warn(dev, "no deep sleep: the loader didn't publish what this host needs\n"); +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -4538,6 +4960,8 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) dev_err(dev, "radio admission failed=%d; host/supplier retained, no retry\n", ret); + else + apple_pcie_neo_sleep_probe(pdev, pcie); return 0; } @@ -5247,6 +5671,8 @@ static void apple_pcie_stop_for_sleep(struct device *dev) apple_pcie_tunnel_hierarchy_lost(dev); } +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie); + /* * The J716c radio drivers save PCI state without entering D3. The PCI core * keeps the root port in D0 as well. Its host and DART have no switchable @@ -5304,6 +5730,34 @@ static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); + if (pcie->neo_sleep_armed) { + struct apple_pcie_port *port; + + pcie->neo_rc_was_down = READ_ONCE(pcie->neo_rc_down); + /* A port that didn't come back from the last sleep keeps what it saved then. */ + list_for_each_entry(port, &pcie->ports, entry) { + int i; + + apple_pcie_port_disable_irq(port); + if (pcie->neo_rc_was_down) + continue; + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + } + /* The functions too: the core read them through closed config space. */ + if (!pcie->neo_rc_was_down) + apple_pcie_neo_save_functions(pcie); + WRITE_ONCE(pcie->neo_rc_down, true); + /* + * Genpd would gate the host here, and an abandoned sleep then finds + * it unprogrammed with nothing to bring it up. The commit lowers it. + */ + device_set_awake_path(dev); + return 0; + } + /* * The managed radio has no suspend boundary of its own; suspend-to-idle * leaves it in place like any other internal port. @@ -5369,6 +5823,324 @@ static int apple_pcie_reset_for_resume(struct apple_pcie *pcie) return 0; } +/* + * The radio kept its power and firmware, in reset since the commit. Its rail isn't touched: + * cycling it, or the port's own reset, is what a cold start needs. PERST# released, the link + * trains. Device interrupts are off in this phase, so this polls for it. + */ +static int apple_pcie_neo_link_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + usleep_range(1000, 2000); + ret = gpiod_set_value_cansleep(port->perst_gpio, 0); + if (ret) + return ret; + /* 100 ms after PERST# as at boot, and after the link (PCIe r5.0, 6.6.1). */ + msleep(100); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, 250000); + if (ret) + return ret; + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, stat & PORT_LINKSTS_UP, + 100, link_up_timeout * 1000); + if (ret) + return ret; + apple_pcie_port_writel(port, BIT(PORT_INT_LINK_UP), PORT_INTSTAT); + msleep(100); + + return 0; +} + +/* + * The PCI core saves the functions again at every sleep, and with the radio down it reads them + * through closed config space. Keep the last state they saved while they answered. + */ +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) + return; + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + kfree(pcie->neo_fn_state[i]); + pcie->neo_fn_state[i] = pci_store_saved_state(fn); + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* With the link back: what the core restores next is that state. */ +static void apple_pcie_neo_load_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int i; + + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (pcie->neo_fn_state[i]) + pci_load_saved_state(fn, pcie->neo_fn_state[i]); + pci_dev_put(fn); + } +} + +/* + * Closed config space keeps the PCI core's restore out of the functions, except for one thing: + * it writes an MSI-X table in their BAR, with no check of its own. With the functions out of + * reach, set that aside so that it has nothing to write; their reach back puts it in place. + */ +static void apple_pcie_neo_msix_gate(struct apple_pcie *pcie, bool reachable) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_msix); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (reachable && pcie->neo_fn_msix[i]) { + fn->msix_enabled = 1; + pcie->neo_fn_msix[i] = false; + } else if (!reachable && fn->msix_enabled) { + fn->msix_enabled = 0; + pcie->neo_fn_msix[i] = true; + } + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* + * The link back, and the one engine read that native ECAM needs on it. The root port's + * bus number is restored first, as that read crosses it. Not a firmware load. + */ +static int apple_pcie_neo_radio_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root; + int ret; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return -ENODEV; + pci_restore_state(root); + ret = apple_pcie_neo_link_up(pcie, port); + if (!ret) + ret = apple_piodma_bootstrap_power_lost(pcie->piodma_supplier); + if (!ret) + ret = apple_piodma_bootstrap_prime(pcie->piodma_supplier, root); + if (!ret) { + apple_piodma_bootstrap_admitted(pcie->piodma_supplier); + apple_pcie_neo_load_functions(pcie, root); + /* The functions' configuration before the flag that says it's there. */ + smp_store_release(&pcie->neo_config_ready, true); + apple_pcie_neo_msix_gate(pcie, true); + } + pci_dev_put(root); + + return ret; +} + +/* + * The rail is the one thing that brings back a chip whose link won't train: macOS's last resort + * for a dead chip too. A cold chip then has its firmware loaded by the probe that follows. + */ +static int apple_pcie_neo_rail_cycle(struct apple_pcie *pcie, struct apple_pcie_port *port, + struct pci_dev *root) +{ + struct pci_dev *fn; + int i, ret; + + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = gpiod_set_value_cansleep(port->pwren_gpio, 0); + if (ret) + return ret; + msleep(pcie->hw->pwren_off_ms); + ret = gpiod_set_value_cansleep(port->pwren_gpio, 1); + if (ret) + return ret; + msleep(pcie->hw->pwren_on_ms); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) + return ret; + /* The PCI core's restore found them gone. Wi-Fi's stop needs its BAR back. */ + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + pci_set_power_state(fn, PCI_D0); + pci_restore_state(fn); + pci_dev_put(fn); + } + return 0; +} + +/* The way a hotplug would. Bluetooth first: Wi-Fi's reset needs it gone. */ +static int apple_pcie_neo_stop_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int ret = 0; + + /* Stopping Wi-Fi writes its BAR, which needs the chip to answer. */ + fn = root->subordinate ? pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)) : NULL; + if (fn && !pci_device_is_present(fn)) + ret = -ENODEV; + pci_dev_put(fn); + if (ret) + return ret; + + pci_lock_rescan_remove(); + if (root->subordinate) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 1)); + if (fn) { + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)); + if (fn) { + ret = apple_piodma_bootstrap_radio_stop(pcie->piodma_supplier, fn); + if (!ret) + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + } + pci_unlock_rescan_remove(); + + return ret; +} + +static void apple_pcie_neo_scan_functions(struct pci_dev *root) +{ + pci_lock_rescan_remove(); + if (root->subordinate) { + pci_scan_child_bus(root->subordinate); + pci_assign_unassigned_bridge_resources(root); + pci_bus_add_devices(root->subordinate); + } + pci_unlock_rescan_remove(); +} + +/* + * Keeping the firmware across a sleep is what's wanted, and this is for when it didn't work: the + * radio lost power, its rings or its IPC. Removing the functions resets the chip, and their + * probes load the firmware again from ROM, as at boot. The link only needs its rail cycled if + * it didn't come back. Otherwise the next sleep tries the whole wake again. + */ +static void apple_pcie_neo_recover_work(struct work_struct *work) +{ + struct apple_pcie *pcie = container_of(work, struct apple_pcie, neo_recover_work); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + struct pci_dev *root; + int ret = 0; + + WRITE_ONCE(pcie->neo_recover, false); + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + if (READ_ONCE(pcie->neo_rc_down)) { + ret = -EIO; + goto out; + } + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) { + dev_warn(pcie->dev, "radio link down after sleep: cycling its rail\n"); + ret = apple_pcie_neo_rail_cycle(pcie, port, root); + if (ret) + goto out; + } + ret = apple_pcie_neo_stop_functions(pcie, root); + if (ret != -ENODEV) + apple_pcie_neo_scan_functions(root); +out: + if (ret) + dev_err(pcie->dev, "radios stay down until the next sleep: %d\n", ret); + else + dev_info(pcie->dev, "radios restarted after sleep, firmware loaded again\n"); + pci_dev_put(root); +} + +/* + * Before the PCI core restores the functions, which need the link. macOS's host bridge + * holds them back the same way until its radio port is enabled. + */ +static int apple_pcie_neo_resume_noirq(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i, ret = 0; + + /* Refused: as before the sleep. A host a failed wake left unprogrammed stays down. */ + if (!pm_resume_via_firmware()) { + if (pcie->neo_rc_was_down) + return 0; + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + WRITE_ONCE(pcie->neo_rc_down, false); + return 0; + } + + list_for_each_entry(port, &pcie->ports, entry) { + /* The pin controller just put it back as it was: released, with no clock yet. */ + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = apple_pcie_neo_cold_init(pcie, port); + if (ret) { + dev_err(pcie->dev, "host did not come back after sleep: %d\n", ret); + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + apple_pcie_tunnel_restore_irq_hw(port); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, port->saved_rid2sid[i]); + } + ret = apple_dart_host_resume(pcie->neo_cold->dart); + if (ret) { + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + WRITE_ONCE(pcie->neo_rc_down, false); + apple_sleep_radio_port_up(); + + /* A failed link stays clocked, and its functions read as gone until the next sleep. */ + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) { + /* The functions' restore then finds nothing, and their drivers say so. */ + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + dev_err(pcie->dev, "radios did not come back after sleep: %d\n", ret); + apple_pcie_neo_recover(pcie); + } else { + dev_dbg(pcie->dev, "radio link back after sleep, rail and firmware kept\n"); + } + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + + return 0; +} + static int apple_pcie_resume_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -5377,6 +6149,8 @@ static int apple_pcie_resume_noirq(struct device *dev) /* The domain's hold for the sleep ends first, whatever follows. */ apple_pcie_release_domains(pcie); + if (pcie->neo_sleep_armed) + return apple_pcie_neo_resume_noirq(pcie); if (pcie->hw->adopt_link) { if (!pcie->link_kept) return 0; diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 36e5afed0227f..09eec7f9d6185 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -564,6 +564,33 @@ static void atcphy_test_update_port_taken(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); } +/* The root hub, once another update that held it has brought the port up again */ +static int atcphy_test_pm_get_after_other(struct apple_atcphy *atcphy, + struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('g'); + guard(mutex)(&atcphy->lock); + atcphy->usb3_port_off = false; + return 0; +} + +/* An update that waited for another one doing the same change leaves the port alone */ +static void atcphy_test_update_after_other(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get_after_other); + steps->mode_port_on = true; + + /* The lanes and the PIPE are in place, the port still owed by the other update */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3_DP); +} + /* * Zeroed memory in place of the core, PIPE handler and USB2 PHY registers: * enough to park the PIPE and power the PHY off, with a PIPE lock request @@ -641,18 +668,25 @@ static struct apple_atcphy *atcphy_test_alloc_failed_handoff(struct kunit *test) return atcphy; } +static int atcphy_test_count_refused_lock(struct apple_atcphy *atcphy); + /* Once dwc3 has stopped, its USB3 PHY power-off parks the PIPE without one */ static void atcphy_test_stopped_power_off_common(struct kunit *test, bool dp) { struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + unsigned int requests = 0; KUNIT_ASSERT_NOT_NULL(test, phy); phy_set_drvdata(phy, atcphy); if (!dp) atcphy->mode = APPLE_ATCPHY_MODE_USB3; + /* The stopped controller can't acknowledge a lock, so none is asked for. */ + test->priv = &requests; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, atcphy_test_count_refused_lock); KUNIT_EXPECT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); KUNIT_EXPECT_EQ(test, atcphy->mode, @@ -1368,6 +1402,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_retry), KUNIT_CASE(atcphy_test_update_unsupported), KUNIT_CASE(atcphy_test_update_port_taken), + KUNIT_CASE(atcphy_test_update_after_other), KUNIT_CASE(atcphy_test_set_port_on), KUNIT_CASE(atcphy_test_host_start), KUNIT_CASE(atcphy_test_stopped_power_off), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 95cc2c9365c1f..4f831f4a2bcb7 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -32,13 +32,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -558,6 +561,34 @@ enum atcphy_lane_mode { #define USB2PHY_MISCTUNE_APBCLK_GATE_OFF BIT(29) #define USB2PHY_MISCTUNE_REFCLK_GATE_OFF BIT(30) +/* + * The power control of the USB controller the T8140 PHY serves ("pwrctl"). macOS's + * controller driver takes the core into hibernation with it at a system sleep and back + * at the wake, in the sequences below. + */ +#define AUSBC_PWR_CFG 0x00 +#define AUSBC_PWR_CFG_STATE_REQ GENMASK(5, 4) +#define AUSBC_PWR_STATUS 0x04 +#define AUSBC_PWR_STATUS_SLEEP BIT(20) +#define AUSBC_PWR_EVENTS 0x08 +#define AUSBC_PWR_ENABLE 0x10 +#define AUSBC_PWR_CLEAR 0x14 +#define AUSBC_PWR_EVENT_MASK GENMASK(3, 0) +#define AUSBC_PWR_EVENT_D0_ON BIT(0) +#define AUSBC_PWR_EVENT_VDD_OFF BIT(1) +#define AUSBC_PWR_EVENT_WAKE GENMASK(3, 2) +#define AUSBC_PWR_TIMEOUT_US 200000 +/* + * Held around the PHY's USB3 setup at a sleep's entry and wake, with the controller's + * always-on side up: set, acknowledged at once, cleared (measured). + */ +#define AUSBC_PWR_PHY_LOCK 0x18 +#define AUSBC_PWR_PHY_LOCK_ACK 0x1c +#define AUSBC_PWR_PHY_LOCK_US 10000 + +/* The controller's DARTs the PHY saves and restores around its power for a sleep */ +#define ATCPHY_SLEEP_DARTS 4 + enum atcphy_generation { ATCPHY_GENERATION_T8103, ATCPHY_GENERATION_T8122, @@ -757,6 +788,8 @@ enum atcphy_t8132_cio_tunable { * off for a PIPE or lane change and is not powered on again yet * @usb3_port_work: Retries the power-on of @usb3_port_off after a USB failure * @usb3_port_retries: Retries left for @usb3_port_work + * @sleep_finish_work: Ends a sleep for the controller, outside its device lock + * @sleep_finish_kept: Whether the controller kept its state, for @sleep_finish_work * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -858,9 +891,38 @@ struct apple_atcphy { bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; + /* + * The controller sleeps in place for this system sleep, the SuperSpeed side is checked + * against the mux after it, and the controller is in hibernation now. + */ + bool sleep_in_place; + bool sleep_ss_down; + bool hibernated; + int sleep_ret; + /* Going through hibernation failed on this boot: the controller stops from now on. */ + bool sleep_no_keep; + /* + * What macOS's controller driver takes down after the hibernation and brings up at + * the wake: the controller's DARTs, the PHY's power domain and that domain's parent, + * and the fabric bridge bits around the parent. Found at probe; this sleep took them + * down. + */ + bool sleep_power; + bool sleep_powered_down; + /* They didn't come back on: nothing in them may be touched until a reboot. */ + bool sleep_dead; + struct device *sleep_darts[ATCPHY_SLEEP_DARTS]; + unsigned int sleep_ndarts; + struct device_node *pd_usb, *pd_common, *pd_aon; + u32 *bridge_gate; + u32 *bridge_saved; + unsigned int bridge_count; + bool bridge_gated; bool usb3_port_off; struct delayed_work usb3_port_work; unsigned int usb3_port_retries; + struct work_struct sleep_finish_work; + bool sleep_finish_kept; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -898,6 +960,7 @@ struct apple_atcphy { void __iomem *usb2phy_reg; void __iomem *pipehandler; void __iomem *lpdptx; + void __iomem *pwrctl; } regs; struct { @@ -2684,9 +2747,11 @@ static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) /* * The T8140 keeps per-device defaults for its secondary eUSB2 register * bank in the ADT. The power-off path asserts the PHY resets, so apply - * them after every power-on. + * them after every power-on, but not with the controller's core in + * hibernation: see atcphy_resume_noirq(). */ - apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + if (!atcphy->hibernated) + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); } static int atcphy_power_on(struct apple_atcphy *atcphy) @@ -4124,6 +4189,8 @@ static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy); */ static int atcphy_rpm_get(struct apple_atcphy *atcphy) { + if (READ_ONCE(atcphy->sleep_dead)) + return -ENODEV; if (!atcphy->idle_rpm) return 0; return pm_runtime_resume_and_get(atcphy->dev); @@ -4180,6 +4247,26 @@ static void atcphy_rpm_release(void *data) atcphy->rpm_held = false; } +/* What a power-down of the common block takes: the DisplayPort side and a tunnel clock. */ +static void atcphy_forget_common_state(struct apple_atcphy *atcphy) +{ + atcphy->dp_t8122.aux = false; + atcphy->dp_t8122.pll = false; + atcphy->dp_t8122.pairs = 0; + atcphy->dp_t8122.rate = 0; + atcphy->tunnel_clock_on = false; + atcphy->tunnel_rate = 0; + atcphy->tunnel_users = 0; + if (atcphy->hw->has_t8140_dp) { + atcphy->dp_owned_pmas = 0; + atcphy->dp_idle_pmas = 0; + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->aux_power_requested = false; + atcphy->mode_dirty = false; + } +} + /* * A T8122 PHY loses its register state whenever its power domain is switched * off. In system sleep that happens to a domain that no running device holds: @@ -4215,13 +4302,7 @@ static void atcphy_restore_after_pd_off(struct apple_atcphy *atcphy) return; WRITE_ONCE(atcphy->pd_was_off, false); - atcphy->dp_t8122.aux = false; - atcphy->dp_t8122.pll = false; - atcphy->dp_t8122.pairs = 0; - atcphy->dp_t8122.rate = 0; - atcphy->tunnel_clock_on = false; - atcphy->tunnel_rate = 0; - atcphy->tunnel_users = 0; + atcphy_forget_common_state(atcphy); if (!atcphy->dp_only) { _atcphy_dwc3_reset_assert(atcphy); @@ -5699,16 +5780,24 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return ret; } -static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +/* Whether an update to @target_mode has nothing left to do, recording the request if so */ +static bool atcphy_mux_done(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + guard(mutex)(&atcphy->lock); + if (!atcphy_mux_is_noop(atcphy, target_mode)) + return false; + atcphy->typec_mode = target_mode; + return true; +} + +static int atcphy_mux_update_to(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { struct atcphy_usb3_pm pm = {}; - enum atcphy_mode target_mode; bool in_mode, port_on, usb_err = true; int ret; scoped_guard(mutex, &atcphy->lock) { atcphy_restore_after_pd_off(atcphy); - target_mode = atcphy_typec_target_mode(atcphy, state); /* * Only a lane or PIPE change, or a SuperSpeed port still owed * its power, needs the USB3 root hub below. Taking it means @@ -5733,6 +5822,13 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux if (ret) goto out; + /* + * Another update holds the root hub for the whole of its change. One that finished + * while this one waited may have left nothing to do: don't cycle the port again. + */ + if (atcphy_mux_done(atcphy, target_mode)) + goto put; + /* * The order in which dwc3 is taken down and brought up around a PHY * change (see dwc3-apple.c), for the SuperSpeed side only: the @@ -5784,6 +5880,11 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux return ret; } +static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +{ + return atcphy_mux_update_to(atcphy, atcphy_typec_target_mode(atcphy, state)); +} + static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); @@ -6190,6 +6291,16 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph atcphy->res.usb2phy_reg = res; } + /* Optional: without it the controller stops for a system sleep. */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwrctl"); + if (res) { + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map pwrctl regs"); + atcphy->regs.pwrctl = addr; + } + return 0; } @@ -6212,6 +6323,91 @@ static bool atcphy_usb2_behind_fixed_hub(struct apple_atcphy *atcphy) return hub; } +static void atcphy_put_node(void *np) +{ + of_node_put(np); +} + +static void atcphy_put_device(void *dev) +{ + put_device(dev); +} + +/* + * The pieces of a sleep that keeps the controller's power, from the device tree: the + * bridge bits (apple,axi2af-gate), the PHY's power domain and its parent, and the + * controller's DARTs (its iommus). Without all of them the controller stops for a sleep. + */ +static int atcphy_probe_sleep_power(struct apple_atcphy *atcphy) +{ + struct device *dev = atcphy->dev; + struct device_node *ep __free(device_node) = + of_graph_get_endpoint_by_regs(atcphy->np, 1, -1); + struct device_node *ctrl __free(device_node) = + ep ? of_graph_get_remote_port_parent(ep) : NULL; + int i, n, ret; + + if (!atcphy->regs.pwrctl || !atcphy->res.axi2af || !ctrl) + return 0; + + n = of_property_count_u32_elems(atcphy->np, "apple,axi2af-gate"); + if (n <= 0) + return 0; + if (n % 2) + return dev_err_probe(dev, -EINVAL, "apple,axi2af-gate: not offset and bit pairs\n"); + atcphy->bridge_gate = devm_kcalloc(dev, n, sizeof(u32), GFP_KERNEL); + atcphy->bridge_saved = devm_kcalloc(dev, n / 2, sizeof(u32), GFP_KERNEL); + if (!atcphy->bridge_gate || !atcphy->bridge_saved) + return -ENOMEM; + ret = of_property_read_u32_array(atcphy->np, "apple,axi2af-gate", atcphy->bridge_gate, n); + if (ret) + return ret; + for (i = 0; i < n; i += 2) + if (atcphy->bridge_gate[i] % 4 || + atcphy->bridge_gate[i] + 4 > resource_size(atcphy->res.axi2af)) + return dev_err_probe(dev, -EINVAL, "apple,axi2af-gate: bad offset %#x\n", + atcphy->bridge_gate[i]); + atcphy->bridge_count = n / 2; + + atcphy->pd_usb = of_parse_phandle(atcphy->np, "power-domains", 0); + ret = devm_add_action_or_reset(dev, atcphy_put_node, atcphy->pd_usb); + if (ret) + return ret; + atcphy->pd_common = of_parse_phandle(atcphy->pd_usb, "power-domains", 0); + ret = devm_add_action_or_reset(dev, atcphy_put_node, atcphy->pd_common); + if (ret) + return ret; + if (!atcphy->pd_common) + return 0; + atcphy->pd_aon = of_parse_phandle(atcphy->pd_common, "power-domains", 0); + ret = devm_add_action_or_reset(dev, atcphy_put_node, atcphy->pd_aon); + if (ret) + return ret; + + n = of_count_phandle_with_args(ctrl, "iommus", "#iommu-cells"); + if (n <= 0 || n > ATCPHY_SLEEP_DARTS) + return 0; + for (i = 0; i < n; i++) { + struct of_phandle_args args; + struct platform_device *pdev; + + if (of_parse_phandle_with_args(ctrl, "iommus", "#iommu-cells", i, &args)) + return 0; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!pdev) + return 0; + ret = devm_add_action_or_reset(dev, atcphy_put_device, &pdev->dev); + if (ret) + return ret; + atcphy->sleep_darts[i] = &pdev->dev; + } + atcphy->sleep_ndarts = n; + atcphy->sleep_power = true; + + return 0; +} + /* * A DisplayPort-only instance has no USB controller to reset and no PIPE: put * the PHY in its off state and register the mux, switch and DP PHY. @@ -6377,6 +6573,8 @@ static bool atcphy_has_t8103_usb3_workaround(const struct atcphy_hw *hw, of_device_is_compatible(root, "apple,t8103"); } +static void atcphy_sleep_finish_work(struct work_struct *work); + static int atcphy_probe(struct platform_device *pdev) { struct apple_atcphy *atcphy; @@ -6399,6 +6597,9 @@ static int atcphy_probe(struct platform_device *pdev) mutex_init(&atcphy->lock); platform_set_drvdata(pdev, atcphy); ret = devm_delayed_work_autocancel(dev, &atcphy->usb3_port_work, atcphy_usb3_port_work); + if (ret) + return ret; + ret = devm_work_autocancel(dev, &atcphy->sleep_finish_work, atcphy_sleep_finish_work); if (ret) return ret; @@ -6427,6 +6628,11 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; atcphy->fixed_usb2 = atcphy_usb2_behind_fixed_hub(atcphy); atcphy->typec_mode = APPLE_ATCPHY_MODE_USB2; + if (atcphy->fixed_usb2) { + ret = atcphy_probe_sleep_power(atcphy); + if (ret) + return ret; + } followed = atcphy_probe_pd_notifier(atcphy); /* @@ -6452,6 +6658,604 @@ static int atcphy_probe(struct platform_device *pdev) return 0; } +/* + * A system sleep with the controller kept, as macOS does it on the J700: its host goes to + * Sleep, not Off. xHCI has halted and saved its state by the time this PHY suspends, as + * the controller is its consumer. The PHY then takes the controller's core into + * hibernation through its power control, holds the core's reset without the clamp, and + * puts the eUSB2 PHY in its sleep state: reference clock gated, APB in reset, the link + * left as it is. Then, as macOS's controller driver does next (measured), it saves + * and stops the controller's DARTs, lowers ATC0_USB, gates the fabric bridge and lowers + * ATC0_COMMON. ATC0_USB_AON stays on with the power control, the PIPE handler and the + * USB2 PHY. + * + * At the wake ATC0_COMMON comes back, the bridge, ATC0_USB and the DARTs after it, with + * the common block at its reset values. It's set up in the mode it had again, the + * controller is brought to D0, and xHCI restores its own state. + * + * USB2 and USB3 qualify, without DisplayPort, whose link the common block's power-down + * would end. With USB3 the PIPE stays routed to the USB3 PHY, as macOS leaves it (measured): + * the halted controller doesn't acknowledge a PIPE lock, and the power control acknowledges + * its PHY lock at the wake only with the PIPE on the PHY. The lanes are left as they are: the + * commit takes them down. At the wake the common block is set up again under that lock, with + * the PIPE parked, and the PIPE is routed to the lanes when the controller's + * core is set up, before xHCI starts, with no Type-C mode change and no SuperSpeed port power + * cycle. macOS's raise from Sleep to its saved USB3 mode is also just a handshake and the USB3 + * PHY's power level (static, setUSB3Mode). macOS also stops its PHY's fabric traffic before + * the clock gate; that block isn't known here, and a halted controller has none. + */ +/* + * The DARTs' own callbacks leave them to the PHY once it saved them, and the sleep driver + * knows the domains' words. + */ +static bool atcphy_sleep_power_ready(const struct apple_atcphy *atcphy) +{ + unsigned int i; + + for (i = 0; i < atcphy->sleep_ndarts; i++) + if (!apple_dart_sleep_listed(atcphy->sleep_darts[i])) + return false; + return !apple_sleep_domain_check(atcphy->pd_usb) && + !apple_sleep_domain_check(atcphy->pd_common); +} + +static bool atcphy_can_sleep_in_place(const struct apple_atcphy *atcphy) +{ + enum atcphy_pipehandler_state pipe = atcphy_pipe_state_for(atcphy, atcphy->mode); + + return atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->sleep_power && + atcphy_sleep_power_ready(atcphy) && !atcphy->sleep_no_keep && atcphy->host_active && + (atcphy->mode == APPLE_ATCPHY_MODE_USB2 || atcphy->mode == APPLE_ATCPHY_MODE_USB3) && + !atcphy->mode_dirty && !atcphy->pipe_pending && + (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY || + atcphy->pipe_state == pipe) && !atcphy->usb_off_pending && + !atcphy->dp_owned_pmas && !atcphy->aux_power_requested && !atcphy->tunnel_users; +} + +static void atcphy_usb2_sleep_enter(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); +} + +static void atcphy_usb2_sleep_exit(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); +} + +/* + * macOS's raise from Sleep: D0 asked for, the core's reset released, D0_ON awaited. Returns + * APPLE_ATCPHY_SLEEP_POWER_LOST for a block that lost power, and -ENODEV only if the core + * may be unpowered, when touching its registers would be an SError. + * With @went_off the core's rail is known to have gone off, so only D0_ON will do. Otherwise + * a core no longer reporting SLEEP has left hibernation, and the event alone is missing. + */ +static int atcphy_controller_d0(struct apple_atcphy *atcphy, bool went_off) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 val, status; + int ret; + + /* All zeros: the block lost power in the sleep and was reset, so no D0_ON will come. */ + status = readl(pc + AUSBC_PWR_STATUS); + val = readl(pc + AUSBC_PWR_EVENTS); + if (!went_off && (val & AUSBC_PWR_EVENT_WAKE)) + dev_dbg(atcphy->dev, "USB controller's wake event up at the wake: %#x\n", val); + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, 0); + clear32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + /* As the reset's release does: a block that lost its state may have the clamp on. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + if (!went_off && !status) { + dev_dbg(atcphy->dev, "USB controller lost its power state in the sleep\n"); + return APPLE_ATCPHY_SLEEP_POWER_LOST; + } + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, val, val & AUSBC_PWR_EVENT_D0_ON, 100, + AUSBC_PWR_TIMEOUT_US); + status = readl(pc + AUSBC_PWR_STATUS); + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, + AUSBC_PWR_EVENT_D0_ON | AUSBC_PWR_EVENT_WAKE); + if (!ret) + return 0; + + dev_err(atcphy->dev, "USB controller gave no D0_ON: events %#x, status %#x\n", val, status); + if (!went_off && !(status & AUSBC_PWR_STATUS_SLEEP)) + return 0; + + return -ENODEV; +} + +/* + * A failure refuses this sleep, and the controller stops for the later ones, so that it + * can't hold every other driver's sleep up. sleep_ret says what state the core is in. + */ +static int atcphy_controller_hibernate(struct apple_atcphy *atcphy) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 events, status = 0; + bool went_off; + int ret; + + set32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, events, events & AUSBC_PWR_EVENT_VDD_OFF, + 100, AUSBC_PWR_TIMEOUT_US); + went_off = !ret; + if (!ret) { + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_VDD_OFF); + /* Armed as macOS arms them. That they drive wake source 70 is inferred. */ + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_WAKE); + ret = readl_poll_timeout(pc + AUSBC_PWR_STATUS, status, + status & AUSBC_PWR_STATUS_SLEEP, 100, + AUSBC_PWR_TIMEOUT_US); + } + if (ret) { + dev_err(atcphy->dev, "USB controller did not hibernate: events %#x, status %#x\n", + events, status); + atcphy->sleep_no_keep = true; + atcphy->sleep_ret = atcphy_controller_d0(atcphy, went_off); + return ret; + } + + /* The core's reset, without the clamp that only macOS's Off path sets. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + atcphy_usb2_sleep_enter(atcphy); + dev_dbg(atcphy->dev, "USB controller in hibernation for the sleep\n"); + + return 0; +} + +/* + * The fabric bridge bits, cleared before ATC0_COMMON goes off. The wake sets those that + * were set, as the bridge comes back at its reset values. + */ +static void atcphy_bridge_gate(struct apple_atcphy *atcphy, bool open) +{ + unsigned int i; + + if (open && !atcphy->bridge_gated) + return; + atcphy->bridge_gated = !open; + for (i = 0; i < atcphy->bridge_count; i++) { + void __iomem *reg = atcphy->regs.axi2af + atcphy->bridge_gate[2 * i]; + u32 bits = atcphy->bridge_gate[2 * i + 1]; + + if (open) { + if (atcphy->bridge_saved[i]) + set32(reg, atcphy->bridge_saved[i]); + continue; + } + atcphy->bridge_saved[i] = readl(reg) & bits; + if (atcphy->bridge_saved[i]) + clear32(reg, bits); + } +} + +#define ATCPHY_DOMAIN_TRIES 3 + +static int atcphy_sleep_domain_up(struct device_node *pd) +{ + int i, ret; + + for (i = 0; i < ATCPHY_DOMAIN_TRIES; i++) { + ret = apple_sleep_domain_set(pd, true); + if (!ret) + break; + } + + return ret; +} + +/* + * Back up in macOS's order: ATC0_COMMON, the bridge, ATC0_USB, then the DARTs. -ENODEV if + * a domain didn't come on, when nothing in it may be touched. + */ +static int atcphy_sleep_power_up(struct apple_atcphy *atcphy) +{ + unsigned int i; + int ret; + + atcphy->sleep_powered_down = false; + ret = atcphy_sleep_domain_up(atcphy->pd_common); + if (!ret) { + atcphy_bridge_gate(atcphy, true); + ret = atcphy_sleep_domain_up(atcphy->pd_usb); + } + if (ret) { + dev_err(atcphy->dev, "USB host's power domains not back on: %d, USB stays off\n", + ret); + WRITE_ONCE(atcphy->sleep_dead, true); + atcphy->sleep_no_keep = true; + return -ENODEV; + } + for (i = 0; i < atcphy->sleep_ndarts; i++) + apple_dart_sleep_restore_lost(atcphy->sleep_darts[i], true); + + return 0; +} + +/* + * What macOS's controller driver does right after the hibernation: the DARTs saved and + * stopped, ATC0_USB off, the bridge gated, ATC0_COMMON off, 0.7 to 1.7 s before the CPUs + * stop. A failure leaves everything up again. + */ +static int atcphy_sleep_power_down(struct apple_atcphy *atcphy) +{ + char bridge[48] = ""; + unsigned int i, n; + int ret = 0, len = 0; + + for (n = 0; n < atcphy->sleep_ndarts; n++) { + ret = apple_dart_sleep_prepare(atcphy->sleep_darts[n]); + if (ret) + break; + } + if (ret) { + while (n--) + apple_dart_sleep_restore_lost(atcphy->sleep_darts[n], false); + dev_err(atcphy->dev, "USB host's DARTs not saved for the sleep: %d\n", ret); + return ret; + } + + atcphy->sleep_powered_down = true; + ret = apple_sleep_domain_set(atcphy->pd_usb, false); + if (!ret) { + for (i = 0; i < atcphy->bridge_count; i++) + len += scnprintf(bridge + len, sizeof(bridge) - len, " %#x", + readl(atcphy->regs.axi2af + atcphy->bridge_gate[2 * i])); + atcphy_bridge_gate(atcphy, false); + ret = apple_sleep_domain_set(atcphy->pd_common, false); + } + if (ret) { + dev_err(atcphy->dev, "USB host's power domains not off for the sleep: %d\n", ret); + return ret; + } + dev_info(atcphy->dev, "USB host's power domains off for the sleep, fabric bridge was%s\n", + bridge); + + return 0; +} + +/* + * The common block went off with ATC0_COMMON. Set it up again in the mode it had, USB2 or + * USB3, without the controller's reset or a USB2 PHY power cycle: atcphy_configure() powers + * the USB2 PHY on over the running one, which rewrites only what it holds, so the hub's link + * stays. The host role goes in again as at probe, for a USB2 PHY that lost its registers + * too. The PIPE is parked first, unlocked as the controller's core is in reset, and stays + * parked until the core is set up and routes it. + */ +static int atcphy_common_restore(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode = atcphy->mode; + int ret; + + ret = atcphy_park_pipehandler(atcphy, true); + if (ret) + dev_warn(atcphy->dev, "PIPE not parked after the sleep: %d\n", ret); + atcphy_forget_common_state(atcphy); + ret = atcphy_power_off(atcphy); + if (ret) + dev_warn(atcphy->dev, "PHY did not power down after the sleep: %d\n", ret); + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy_setup_pipehandler(atcphy); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + + ret = atcphy_configure(atcphy, mode); + if (!ret || mode == APPLE_ATCPHY_MODE_USB2) + return ret; + /* USB2 keeps the hub; the mux update after the wake then tries the mode again. */ + dev_warn(atcphy->dev, "mode %d not set up again after the sleep (%d): USB2 for now\n", + mode, ret); + return atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB2); +} + +/* + * The wake's path, and that of an entry that failed after the hibernation: the domains, + * the USB2 PHY, the common block if it went off, then D0. See atcphy_controller_d0(). + */ +static void atcphy_phy_lock(struct apple_atcphy *atcphy, bool lock) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 val; + + writel(lock, pc + AUSBC_PWR_PHY_LOCK); + if (readl_poll_timeout(pc + AUSBC_PWR_PHY_LOCK_ACK, val, !!(val & BIT(0)) == lock, 10, + AUSBC_PWR_PHY_LOCK_US)) + dev_warn(atcphy->dev, "PHY lock %s not acknowledged: %#x\n", + lock ? "request" : "release", val); +} + +static int atcphy_sleep_come_back(struct apple_atcphy *atcphy, bool common_lost, bool went_off) +{ + bool kept; + int ret = 0, d0; + + if (atcphy->sleep_powered_down && atcphy_sleep_power_up(atcphy)) + return -ENODEV; + atcphy_usb2_sleep_exit(atcphy); + /* Not all zeros: the always-on side kept power, and the link with it. */ + kept = readl(atcphy->regs.pwrctl + AUSBC_PWR_STATUS); + if (common_lost && kept && atcphy->mode == APPLE_ATCPHY_MODE_USB2) { + /* As macOS: USB2 needs nothing of the core, which a mode change sets up. */ + atcphy_forget_common_state(atcphy); + } else if (common_lost) { + if (kept) + atcphy_phy_lock(atcphy, true); + ret = atcphy_common_restore(atcphy); + if (kept) + atcphy_phy_lock(atcphy, false); + if (ret) + dev_err(atcphy->dev, "PHY not set up again after the sleep: %d\n", ret); + } + + d0 = atcphy_controller_d0(atcphy, went_off); + return d0 < 0 ? d0 : ret ?: d0; +} + +/* + * macOS takes a USB host that fails its raise Off, then starts it cold. The same here, with + * nothing touching the core: clamped in reset, its domains and the always-on one off, then + * on again. The block comes back reset, as after a sleep that took its power, and the host + * starts over. -ENODEV if the core still may be unpowered. + */ +static int atcphy_sleep_recover(struct apple_atcphy *atcphy) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + if (!atcphy->sleep_power || !atcphy->pd_aon) + return -ENODEV; + dev_warn(atcphy->dev, "USB controller didn't come back: power cycling it, as macOS's Off does\n"); + /* Domains that didn't come on get another try first. */ + if (READ_ONCE(atcphy->sleep_dead)) { + WRITE_ONCE(atcphy->sleep_dead, false); + atcphy->sleep_powered_down = true; + if (atcphy_sleep_power_up(atcphy)) + return -ENODEV; + } + _atcphy_dwc3_reset_assert(atcphy); + atcphy_usb2_sleep_enter(atcphy); + ret = atcphy_sleep_power_down(atcphy); + if (!ret) { + ret = apple_sleep_domain_set(atcphy->pd_aon, false); + ret = atcphy_sleep_domain_up(atcphy->pd_aon) ?: ret; + } + if (ret) + dev_warn(atcphy->dev, "USB power cycle incomplete: %d\n", ret); + ret = atcphy_sleep_come_back(atcphy, true, false); + if (ret < 0) + dev_err(atcphy->dev, "USB controller not back after the power cycle: %d\n", ret); + else + dev_info(atcphy->dev, "USB controller back after a power cycle: the host starts over\n"); + + return ret; +} + +static int atcphy_suspend_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + int ret; + + guard(mutex)(&atcphy->lock); + /* Suspend-to-idle keeps every domain on: nothing to do. */ + if (!atcphy->sleep_in_place || !pm_suspend_via_firmware()) + return 0; + /* A mode change since the prepare: the controller would lose its PIPE. */ + if (!atcphy_can_sleep_in_place(atcphy)) { + dev_err(dev, "USB changed for the sleep (mode %d, PIPE %d, host %d): refusing it\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + atcphy->sleep_ret = -EBUSY; + return -EBUSY; + } + ret = atcphy_controller_hibernate(atcphy); + if (ret) { + if (atcphy->sleep_ret == -ENODEV) + atcphy->sleep_ret = atcphy_sleep_recover(atcphy); + return ret; + } + atcphy->hibernated = true; + ret = atcphy_sleep_power_down(atcphy); + if (ret) { + /* No resume callback follows a failed suspend: back now, and stop from now on. */ + atcphy->sleep_no_keep = true; + atcphy->sleep_ret = atcphy_sleep_come_back(atcphy, atcphy->sleep_powered_down, + true); + if (atcphy->sleep_ret == -ENODEV) + atcphy->sleep_ret = atcphy_sleep_recover(atcphy); + atcphy->hibernated = false; + return ret; + } + apple_sleep_usb_set_asleep(true); + + return 0; +} + +/* + * A PHY in a DisplayPort mode doesn't sleep in place, and a sleep that reached its commit + * lowered ATC0_COMMON under it, behind its power domain's back: the lanes and the AUX + * channel went. Have atcphy_resume() set the tracked mode up again, before the display + * controller connects the link anew. + */ +static void atcphy_sleep_lost_dp(struct apple_atcphy *atcphy) +{ + if (!atcphy->hw->has_t8140_dp || !atcphy->hw->restore_after_pd_off || + !pm_resume_via_firmware() || !atcphy_modes[atcphy->mode].enable_dp_aux) + return; + WRITE_ONCE(atcphy->pd_was_off, true); + dev_dbg(atcphy->dev, "DisplayPort mode %d lost in the sleep: setting it up again\n", + atcphy->mode); +} + +static int atcphy_resume_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + + guard(mutex)(&atcphy->lock); + if (!atcphy->hibernated) { + atcphy_sleep_lost_dp(atcphy); + return 0; + } + apple_sleep_usb_set_asleep(false); + /* The PHY took ATC0_COMMON down itself, so this goes for a refused sleep too. */ + atcphy->sleep_ret = atcphy_sleep_come_back(atcphy, true, false); + if (atcphy->sleep_ret == -ENODEV) + atcphy->sleep_ret = atcphy_sleep_recover(atcphy); + atcphy->hibernated = false; + if (!atcphy->sleep_ret) + dev_dbg(dev, "USB controller back in D0 after the sleep, always-on side kept, PHY in mode %d\n", + atcphy->mode); + if (atcphy->sleep_ret == APPLE_ATCPHY_SLEEP_POWER_LOST) { + dev_dbg(dev, "USB controller back in D0 after the sleep, always-on side lost, PHY in mode %d\n", + atcphy->mode); + /* + * The secondary bank may have lost its defaults. It's next to the core, maybe + * under the core's rail, and an access with the rail off would be an SError: + * only now. macOS writes nothing there at a wake that kept the always-on side. + */ + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + } + + return 0; +} + +static struct apple_atcphy *atcphy_from_usb2_phy(struct phy *phy) +{ + if (!phy || phy->ops != &apple_atc_usb2_phy_ops) + return NULL; + return phy_get_drvdata(phy); +} + +/* + * After a sleep in place the PHY goes back to what the mux last asked for. The wake set the + * block up in its mode again and the controller's core routed the PIPE, so a mux update is + * left only if the mux changed during the sleep or that setup failed. + */ +static void atcphy_sleep_ss_up(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode; + int ret; + + scoped_guard(mutex, &atcphy->lock) { + if (!atcphy->sleep_ss_down || atcphy->sleep_dead) + return; + atcphy->sleep_ss_down = false; + mode = atcphy->typec_mode; + atcphy_restore_after_pd_off(atcphy); + if (atcphy_mux_is_noop(atcphy, mode)) { + dev_dbg(atcphy->dev, "SuperSpeed side already in mode %d after the sleep\n", + mode); + return; + } + } + ret = atcphy_mux_update_to(atcphy, mode); + dev_dbg(atcphy->dev, "SuperSpeed side back after the sleep, mode %d: %d\n", mode, ret); +} + +/* No lane or port change here: anything the sleep needs is done at noirq, in a few ms. */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + + guard(mutex)(&atcphy->lock); + if (atcphy_can_sleep_in_place(atcphy)) { + atcphy->sleep_in_place = true; + atcphy->sleep_ss_down = true; + atcphy->sleep_ret = 0; + return 0; + } + if (atcphy->regs.pwrctl && atcphy->fixed_usb2) + dev_dbg(atcphy->dev, + "USB controller stops for the sleep: mode %d, PIPE %d, host %d\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + + return -EBUSY; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_prepare); + +int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + + return atcphy->sleep_in_place ? atcphy->sleep_ret : -ENODATA; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_result); + +int apple_atcphy_sleep_recover(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + + return atcphy_sleep_recover(atcphy); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_recover); + +/* + * A controller that was restarted after the sleep is set up as at boot and may hibernate + * again. Only the PHY's own failures stop it for the later sleeps; a sleep refused for a + * USB change doesn't. + */ +static void atcphy_sleep_finish(struct apple_atcphy *atcphy, bool kept) +{ + scoped_guard(mutex, &atcphy->lock) { + if (!kept && atcphy->sleep_ret < 0 && atcphy->sleep_ret != -EBUSY) + atcphy->sleep_no_keep = true; + atcphy->sleep_in_place = false; + atcphy->sleep_ret = 0; + } + atcphy_sleep_ss_up(atcphy); +} + +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (atcphy) + atcphy_sleep_finish(atcphy, kept); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish); + +static void atcphy_sleep_finish_work(struct work_struct *work) +{ + struct apple_atcphy *atcphy = container_of(work, struct apple_atcphy, sleep_finish_work); + + atcphy_sleep_finish(atcphy, READ_ONCE(atcphy->sleep_finish_kept)); +} + +/* + * The SuperSpeed side may wait for the controller's device lock, which an unbind of the + * controller holds while it cancels the controller's own work: so this work is the PHY's. + */ +void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return; + WRITE_ONCE(atcphy->sleep_finish_kept, kept); + queue_work(system_unbound_wq, &atcphy->sleep_finish_work); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish_async); + +void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (atcphy) + flush_work(&atcphy->sleep_finish_work); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish_flush); + static int atcphy_suspend(struct device *dev) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); @@ -6528,7 +7332,10 @@ static void atcphy_shutdown(struct platform_device *pdev) dev_err(atcphy->dev, "USB3 common shutdown failed: %d\n", ret); } -static DEFINE_SIMPLE_DEV_PM_OPS(atcphy_pm_ops, atcphy_suspend, atcphy_resume); +static const struct dev_pm_ops atcphy_pm_ops = { + SYSTEM_SLEEP_PM_OPS(atcphy_suspend, atcphy_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(atcphy_suspend_noirq, atcphy_resume_noirq) +}; static const struct atcphy_hw atcphy_hw_t8103 = { .gen = ATCPHY_GENERATION_T8103, @@ -6575,6 +7382,8 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .has_usb2phy_reg = true, .has_t8140_dp = true, .optional_tunables = true, + .park_pipe_unlocked = true, + .restore_after_pd_off = true, }; static const struct atcphy_hw atcphy_hw_t8142 = { diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 7390d28dfafab..3e8ab5a9b933b 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -26,6 +27,7 @@ #include #include +#include #include "pinctrl-utils.h" #include "core.h" @@ -49,6 +51,11 @@ struct apple_gpio_pinctrl { struct pinctrl_desc pinctrl_desc; struct gpio_chip gpio_chip; u8 irq_group_base; + /* The pins' words across a firmware-driven suspend */ + u32 *saved; + unsigned long *saved_valid; + /* Pins this driver has written. Others belong to someone else. */ + unsigned long *touched; struct apple_gpio_irq_group irqgrps[]; }; @@ -91,6 +98,8 @@ static const struct regmap_config regmap_config = { static void apple_gpio_set_reg(struct apple_gpio_pinctrl *pctl, unsigned int pin, u32 mask, u32 value) { + if (pctl->touched) + set_bit(pin, pctl->touched); regmap_update_bits(pctl->map, REG_GPIO(pin), mask, value); } @@ -594,6 +603,15 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); + /* Only with the regmap reaching the hardware: a cache would hide the loss. */ + if (hw && hw->no_regcache) { + pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); + pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->saved || !pctl->saved_valid || !pctl->touched) + return -ENOMEM; + } + for (i = 0; i < npins; i++) { pins[i].number = i; pins[i].name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "PIN%u", i); @@ -645,10 +663,63 @@ static const struct of_device_id apple_gpio_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_gpio_pinctrl_of_match); +/* + * A firmware-driven suspend powers this block off, and neither the pin core nor + * a consumer sets a pin up again. Slots that fault aren't readable in the regmap, + * so they stay out of this. + */ +static int apple_gpio_pinctrl_suspend_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i; + + if (!pctl->saved || pm_suspend_target_state != PM_SUSPEND_MEM || + !pm_suspend_via_firmware()) + return 0; + + /* + * Only the pins this driver configured. The others belong to the firmware or another + * processor, and writing one back after the wake raises an SError. + */ + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) + if (!regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) + set_bit(i, pctl->saved_valid); + + return 0; +} + +static int apple_gpio_pinctrl_resume_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i, val, restored = 0; + + if (!pctl->saved || !pm_resume_via_firmware()) + return 0; + + for_each_set_bit(i, pctl->saved_valid, pctl->pinctrl_desc.npins) { + if (!regmap_read(pctl->map, REG_GPIO(i), &val) && val == pctl->saved[i]) + continue; + regmap_write(pctl->map, REG_GPIO(i), pctl->saved[i]); + restored++; + } + dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + + return 0; +} + +static const struct dev_pm_ops apple_gpio_pinctrl_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_gpio_pinctrl_suspend_noirq, + apple_gpio_pinctrl_resume_noirq) +}; + static struct platform_driver apple_gpio_pinctrl_driver = { .driver = { .name = "apple-gpio-pinctrl", .of_match_table = apple_gpio_pinctrl_of_match, + .pm = pm_sleep_ptr(&apple_gpio_pinctrl_pm_ops), .suppress_bind_attrs = true, }, .probe = apple_gpio_pinctrl_probe, diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 39e0972acd0e8..586e3f675db9e 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -1529,6 +1530,40 @@ static const struct apple_pmc_hw apple_pmc_hw_t6050 = { .has_pending = true, }; +/* + * A system sleep that reached the hardware may have taken the enables and floors + * with it. Put back what the driver's books say: these are no-ops if they stayed. + */ +static int apple_pmc_resume_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + unsigned long flags; + unsigned int rail; + int ret = 0, err; + + if (!pm_resume_via_firmware() || !pmc) + return 0; + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails; rail++) { + err = apple_pmc_enable_write(pmc, rail); + ret = ret ?: err; + if (pmc->floor_rails & BIT(rail)) { + err = apple_pmc_floor_apply(pmc, rail); + ret = ret ?: err; + } + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + if (ret) + dev_err(dev, "state not restored after sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_pmc_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmc_resume_early) +}; + static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t6050-pmc", .data = &apple_pmc_hw_t6050 }, { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, @@ -1545,6 +1580,7 @@ static struct platform_driver apple_pmc_driver = { .driver = { .name = "apple-pmc", .of_match_table = apple_pmc_of_match, + .pm = pm_sleep_ptr(&apple_pmc_pm_ops), /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 85b5a5b22f009..bc7164ca48532 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -15,11 +15,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #define APPLE_PMGR_RESET BIT(31) @@ -41,6 +43,8 @@ #define APPLE_PMGR_PS_PWRGATE 0x0 #define APPLE_PMGR_PS_SET_TIMEOUT 100 +/* How long a lowered domain may report BUSY while its switch finishes */ +#define APPLE_PMGR_BUSY_SETTLE_US 1000 #define APPLE_PMGR_PS_CONSUMER_TIMEOUT 1000 #define APPLE_PMGR_PS_MAX_RETENTION 4 #define APPLE_PMGR_PS_MAX_CHAIN 4 @@ -65,9 +69,13 @@ struct apple_pmgr_ps { bool force_reset; bool externally_clocked; bool system_sleep; + /* Lowered as macOS lowers it, and unlatched even in a system sleep. */ + bool macos_lower; bool firmware_owned; /* apple,sleep-retained: kept on while the system sleeps */ bool sleep_retained; + /* apple,sleep-driver-owned: the driver behind it brings it back after a sleep */ + bool sleep_driver_owned; /* Never enable automatic power management (apple,no-auto-pm). */ bool no_auto_pm; bool inherited_on; @@ -76,6 +84,10 @@ struct apple_pmgr_ps { bool strict_state; bool wait_parent; u32 reset_time_us; + /* Off in genpd but on in hardware as the system slept: someone else raised it. */ + bool sleep_raised_by_other; + /* That domain's target as the system slept. */ + u32 sleep_target; /* * apple,consumer-switched: firmware-owned, but powered off and on by its * consumer (apple_pmgr_ps_consumer_off()). The state word before @@ -127,6 +139,22 @@ static int apple_pmgr_ps_wait_parent(struct apple_pmgr_ps *ps, u32 *cur) * power-on then fails as it did before, and genpd, which ignores errors in * that phase, keeps the domain requested on. */ +/* Pulse the PS reset of a domain that is off with BUSY latched; it stays off. */ +static void apple_pmgr_ps_unlatch_off(struct apple_pmgr_ps *ps, u32 *cur) +{ + u32 off = *cur & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY | + APPLE_PMGR_PS_RESET | APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_TARGET); + + regmap_write(ps->regmap, ps->offset, off | APPLE_PMGR_PS_RESET); + regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + !(*cur & APPLE_PMGR_BUSY), 1, APPLE_PMGR_PS_SET_TIMEOUT); + regmap_write(ps->regmap, ps->offset, off); + regmap_read(ps->regmap, ps->offset, cur); + if (*cur & APPLE_PMGR_BUSY) + dev_warn_ratelimited(ps->dev, "PS %s: off with BUSY latched: %#x\n", + ps->genpd.name, *cur); +} + static int apple_pmgr_ps_clear_latch(struct apple_pmgr_ps *ps, u32 reg, u32 *cur) { u32 off = (reg & ~APPLE_PMGR_PS_TARGET) | APPLE_PMGR_PS_PWRGATE; @@ -197,11 +225,27 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a genpd->name, cur); } + /* + * As macOS lowers a domain: automatic power management off first, back at full + * power, and only then the target. Taking both at once from an auto-gated state + * can leave BUSY latched, and a latch stalls the sleep engine's wake. + */ + if (ps->macos_lower && pstate != APPLE_PMGR_PS_ACTIVE && (reg & APPLE_PMGR_AUTO_ENABLE)) { + regmap_write(ps->regmap, ps->offset, + reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY)); + regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) == + APPLE_PMGR_PS_ACTIVE, 1, timeout); + } + /* Do not replay the hardware-owned BUSY bit in this whole-word write. */ reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY | APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); + /* Writing the history bits clears them. */ + if (ps->macos_lower && pstate != APPLE_PMGR_PS_ACTIVE) + reg |= APPLE_PMGR_FLAGS; dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); @@ -259,6 +303,36 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } } + /* + * A domain can reach off with BUSY latched: on T8140 both ISP CPU cores do, + * every time they go down under Linux. macOS leaves them off with it clear, + * and the sleep engine, which powers them up itself in a wake that keeps DRAM + * up, stalls on a latched one until its watchdog resets the SoC. Only a PS + * reset clears it, and the domain is off, so pulse one as on the power-on + * path (not in a system sleep transition, see apple_pmgr_ps_clear_latch()). + */ + /* + * Off only once BUSY clears too: macOS waits for (word & 0x8ff) == 0. The + * actual state reads 0 a moment before the switch is done. BUSY still set + * after this is a latch. + */ + if (!ret && ps->macos_lower && pstate != APPLE_PMGR_PS_ACTIVE && (cur & APPLE_PMGR_BUSY)) + regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, cur, + !(cur & APPLE_PMGR_BUSY), 1, + APPLE_PMGR_BUSY_SETTLE_US); + + if (!ret && pstate != APPLE_PMGR_PS_ACTIVE && (cur & APPLE_PMGR_BUSY) && + !FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) && (!ps->system_sleep || ps->macos_lower) && + !ps->wait_parent && !ps->strict_state && !ps->externally_clocked) { + /* macOS never needs this: say so when it does. */ + if (ps->macos_lower) + dev_info_ratelimited(ps->dev, + "PS %s: off with BUSY%s: %#x; cleared by a PS reset\n", + genpd->name, ps->system_sleep ? " in a system sleep" : "", + cur); + apple_pmgr_ps_unlatch_off(ps, &cur); + } + report: if (ret < 0) dev_err_ratelimited(ps->dev, @@ -356,6 +430,12 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) /* A live coprocessor keeps this domain through system sleep. */ if (ps->sleep_retained && ps->system_sleep) return -EBUSY; + /* + * Its driver took it down for the sleep and raises it in its own order. Gating it + * here would take its parents too, and genpd would raise them all first at the wake. + */ + if (ps->system_sleep && apple_sleep_domain_is_down(ps->dev->of_node)) + return -EBUSY; ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); if (ret) @@ -445,6 +525,111 @@ static int apple_pmgr_ps_resume_early(struct device *dev) return 0; } +/* T8140's always-on PMGR, the block the AOP shares */ +#define APPLE_PMGR_T8140_AON_BASE 0x308280000ULL + +static bool apple_pmgr_ps_in_aon(struct device_node *node) +{ + struct resource res; + + return node->parent && !of_address_to_resource(node->parent, 0, &res) && + res.start == APPLE_PMGR_T8140_AON_BASE; +} + +/* + * By the noirq phase genpd has powered off what it is going to. A domain it has off + * whose target is still on was raised by a coprocessor's firmware: on T8140 the AOP + * raises audio domains of its own in the always-on PMGR, and the sleep keeps them. + */ +static int apple_pmgr_ps_suspend_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + u32 reg; + + ps->sleep_raised_by_other = false; + if (ps->firmware_owned || ps->inherited_on) + return 0; + + spin_lock_irqsave(&ps->genpd.slock, flags); + if (!regmap_read(ps->regmap, ps->offset, ®)) { + /* + * Off for genpd but still powered, as the loader left it (T8140 + * isp_dprx): finish the power-off now. Otherwise the sleep lowers it + * at the commit, raises it again at the wake, and the resync lowers + * it a second time with BUSY left. macOS has it off at the commit and + * doesn't touch it after the wake. Not in the always-on PMGR, whose + * words the AOP raises for itself. + */ + if (ps->macos_lower && ps->genpd.status == GENPD_STATE_OFF && + FIELD_GET(APPLE_PMGR_PS_TARGET, reg) && FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) && + !apple_pmgr_ps_in_aon(dev->of_node)) { + apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_PWRGATE, false); + regmap_read(ps->regmap, ps->offset, ®); + } + ps->sleep_target = FIELD_GET(APPLE_PMGR_PS_TARGET, reg); + ps->sleep_raised_by_other = ps->genpd.status == GENPD_STATE_OFF && + ps->sleep_target != APPLE_PMGR_PS_PWRGATE; + } + spin_unlock_irqrestore(&ps->genpd.slock, flags); + + return 0; +} + +/* + * A firmware-driven suspend powers the SoC down and its sleep engine leaves the + * domains in states of its own: AUTO_ENABLE cleared, some at clock gating. genpd + * only powers a domain back on if it turned it off itself, so its idea of a + * domain it left on is now wrong. Put each domain back where genpd thinks it is. + * + * On T8140 macOS re-raises 22 domains in the first 5 ms after a wake and lowers + * 30 others (measured under aura). Which of them Linux needs isn't established, + * so this follows genpd's books rather than a list. A domain whose driver took it + * down for the sleep, as the USB host's PHY does, is that driver's to raise, in + * its own order. A domain someone else raised before the sleep, and still at that + * target, stays theirs. + */ +static int apple_pmgr_ps_resume_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + u32 reg; + int ret; + + if (!pm_resume_via_firmware() || ps->firmware_owned || ps->inherited_on || + ps->sleep_driver_owned || apple_sleep_domain_is_down(dev->of_node)) + return 0; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ret = regmap_read(ps->regmap, ps->offset, ®); + if (!ret) { + if (ps->genpd.status == GENPD_STATE_OFF) { + u32 target = FIELD_GET(APPLE_PMGR_PS_TARGET, reg); + + /* + * Lowering an AOP-raised audio domain here leaves the internal + * microphone array's data path dark: the AOP never raises it again. + */ + if (target != APPLE_PMGR_PS_PWRGATE && + !(ps->sleep_raised_by_other && target == ps->sleep_target)) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_PWRGATE, + false); + } else { + bool auto_enable = !ps->no_auto_enable && !ps->no_auto_pm; + + if (FIELD_GET(APPLE_PMGR_PS_TARGET, reg) != APPLE_PMGR_PS_ACTIVE || + (auto_enable && !(reg & APPLE_PMGR_AUTO_ENABLE))) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + auto_enable); + } + } + spin_unlock_irqrestore(&ps->genpd.slock, flags); + if (ret) + dev_err(dev, "PS %s: resync after sleep failed: %d\n", ps->genpd.name, ret); + + return 0; +} + static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); @@ -653,6 +838,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ps); ps->reset_time_us = t8132 ? APPLE_PMGR_T8132_RESET_TIME : APPLE_PMGR_RESET_TIME; ps->needs_reset_idle_wait = t8132; + ps->macos_lower = of_device_is_compatible(node, "apple,t8140-pmgr-pwrstate"); ps->no_auto_enable = t8132 && !strict_state; ps->strict_state = strict_state; @@ -705,6 +891,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) "retention workaround needs a consumer switch\n"); } ps->sleep_retained = of_property_read_bool(node, "apple,sleep-retained"); + ps->sleep_driver_owned = of_property_read_bool(node, "apple,sleep-driver-owned"); /* * Only T8132 hands over inherited-on domains with this contract. Older * M3 device trees carry the property with another meaning; keep them @@ -1099,6 +1286,7 @@ MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_noirq, apple_pmgr_ps_resume_noirq) }; static struct platform_driver apple_pmgr_ps_driver = { diff --git a/drivers/pmdomain/apple/pmp-report-test.c b/drivers/pmdomain/apple/pmp-report-test.c index be4e884e4a852..4356c7f01b83f 100644 --- a/drivers/pmdomain/apple/pmp-report-test.c +++ b/drivers/pmdomain/apple/pmp-report-test.c @@ -12,6 +12,16 @@ static void pmp_range(u8 *table, u32 id, u32 base, u32 count) put_unaligned_le32(count, table + 8); } +static void pmp_request_word_replay(struct kunit *test) +{ + /* A request an entry withdrew stays off; one nobody here owns stays on. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x31, 0x03, 0x02), 0x32ULL); + /* The found word can be empty: only the requests come back. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0, 0x03, 0x01), 0x01ULL); + /* Nothing requested, nothing known. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x10, 0, 0), 0x10ULL); +} + static void pmp_ranges_t6030(struct kunit *test) { u8 table[2 * PMP_PTD_RANGE_SIZE]; @@ -290,6 +300,7 @@ static void pmp_dvfs_words(struct kunit *test) static struct kunit_case apple_pmp_report_cases[] = { KUNIT_CASE(pmp_ranges_t6030), + KUNIT_CASE(pmp_request_word_replay), KUNIT_CASE(pmp_ranges_t8122), KUNIT_CASE(pmp_ranges_t6031), KUNIT_CASE(pmp_ranges_reject_bad_records), diff --git a/drivers/pmdomain/apple/pmp-report-validation.h b/drivers/pmdomain/apple/pmp-report-validation.h index c5b489e6381ef..c20a29412d46e 100644 --- a/drivers/pmdomain/apple/pmp-report-validation.h +++ b/drivers/pmdomain/apple/pmp-report-validation.h @@ -26,6 +26,15 @@ #define PMP_PTD_RANGE_REQUEST 10 #define PMP_PTD_RANGE_ACK 11 +/* + * The request word for a PMP that is starting: requests made through entries, + * and every other bit as it was found. + */ +static inline u64 apple_pmp_request_word(u64 found, u64 known, u64 shadow) +{ + return (found & ~known) | shadow; +} + /* * A complete SOC-DEV-DVFS request word: the valid bit, an optional main level * in bits 4:0, and zero or more pairs. Classes 0x21 diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index e1c97e41bf25d..23f27f7c07b9a 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -115,8 +115,17 @@ struct apple_pmp_report { /* Requests made through entries: bits requested on, all bits. */ u64 shadow; u64 known; - /* Requests are held in memory until the PMP runs (@defer_requests). */ + /* Bits whose drivers withdraw them at the sleep's commit, with the PMP still running. */ + u64 commit_owned; + /* + * Requests are held in memory until the PMP runs (@defer_requests), and + * while it's stopped for a system sleep. + */ bool defer; + /* The request word when a system sleep stopped the PMP. */ + u64 sleep_request; + /* The last SOC-DEV-DVFS word written to each entry, 0 if none. */ + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; /* J613 floor owner, absent on all other report instances. */ @@ -444,7 +453,7 @@ static void apple_pmp_report_ready_work(struct work_struct *work) spin_lock_irqsave(&rep->lock, flags); val = readq(rep->base + offs->tgt_read); - val = (val & ~rep->known) | rep->shadow; + val = apple_pmp_request_word(val, rep->known, rep->shadow); writeq(val, rep->base + offs->tgt_write); want = rep->shadow; rep->defer = false; @@ -458,6 +467,97 @@ static void apple_pmp_report_ready_work(struct work_struct *work) readq(rep->base + offs->status), want); } +/* A report names its PMP where a SoC has more than one kind of link to it. */ +static bool apple_pmp_report_of(const struct apple_pmp_report *rep, const struct device *pmp) +{ + struct device_node *np = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + bool match = !np || np == pmp->of_node; + + of_node_put(np); + return match; +} + +/** + * apple_pmp_report_sleep() - withdraw and hold requests before a system sleep stops the PMP + * @pmp: the PMP device, still running + * @at_commit: the PMP stops at the sleep's commit, so requests whose drivers withdraw them + * there are left set + * + * The devices behind these requests have stopped by now, so the PMP is told before + * it goes. Requests made from here on only update the books, so nothing waits on an + * acknowledgment from a stopped PMP. Bits no entry owns are left alone: nothing here + * knows whether their devices have stopped. + */ +void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 val, withdrawn; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + rep->sleep_request = readq(rep->base + offs->tgt_read); + withdrawn = rep->sleep_request & rep->known & ~(at_commit ? rep->commit_owned : 0); + if (withdrawn) + writeq(rep->sleep_request & ~withdrawn, rep->base + offs->tgt_write); + rep->defer = true; + spin_unlock_irqrestore(&rep->lock, flags); + + if (withdrawn && readq_poll_timeout(rep->base + offs->actual, val, + !(val & withdrawn), 100, 50000)) + dev_warn(rep->dev, "PMP still acknowledges withdrawn requests %#llx (ack %#llx)\n", + withdrawn, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_sleep); + +/** + * apple_pmp_report_wake() - hand the PMP every request after a system sleep + * @pmp: the PMP device, running again + * @lost_power: the sleep reached the hardware, so the report may have lost its + * contents: bits no entry owns come from the word kept at the sleep + * + * The power requests go first and the performance requests after them, the + * order an entry uses when it powers on. + */ +void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ + struct apple_pmp_report *rep; + unsigned long flags; + unsigned int i; + u64 val, want; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + val = lost_power ? rep->sleep_request : readq(rep->base + offs->tgt_read); + writeq(apple_pmp_request_word(val, rep->known, rep->shadow), + rep->base + offs->tgt_write); + for (i = 0; rep->dvfs_shadow && i < offs->dvfs_count; i++) + if (rep->dvfs_shadow[i]) + writeq(rep->dvfs_shadow[i], + rep->base + offs->update + 8 * (offs->dvfs_first + i)); + want = rep->shadow; + rep->defer = false; + spin_unlock_irqrestore(&rep->lock, flags); + + if (want && readq_poll_timeout(rep->base + offs->actual, val, + (val & want) == want, 1000, 1000000)) + dev_warn(rep->dev, "PMP did not acknowledge requests %#llx (ack %#llx)\n", + want, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_wake); + /* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) { @@ -2203,6 +2303,12 @@ static int apple_pmp_report_probe(struct platform_device *pdev) } rep->defer = rep->offsets->defer_requests; + if (rep->offsets->dvfs_count) { + rep->dvfs_shadow = devm_kcalloc(dev, rep->offsets->dvfs_count, + sizeof(*rep->dvfs_shadow), GFP_KERNEL); + if (!rep->dvfs_shadow) + return -ENOMEM; + } INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); @@ -2745,7 +2851,13 @@ EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); */ static void apple_pmp_report_dvfs_write(struct apple_pmp_report *rep, u32 entry, u64 val) { - writeq(val, rep->base + rep->offsets->update + 8 * entry); + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + rep->dvfs_shadow[entry - rep->offsets->dvfs_first] = val; + if (!rep->defer) + writeq(val, rep->base + rep->offsets->update + 8 * entry); + spin_unlock_irqrestore(&rep->lock, flags); } /** @@ -2820,6 +2932,136 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo return ret; } +/** + * apple_pmp_report_entry_request() - set or withdraw one entry's request + * @entry: the report entry's node + * @enable: make the request, or withdraw it + * + * For a driver that switches its device's power itself around a system sleep. The + * PMP may use a device whose request is set, so that driver withdraws the request + * before its power goes and makes it again after its power is back. A withdrawn + * request isn't replayed after the PMP restarts. + * + * Return: 0, also with no report bound or the PMP not running, or the error from + * waiting for its acknowledgment. + */ +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable) +{ + struct device_node *report; + struct apple_pmp_report *rep; + u32 id; + int ret = 0; + + if (of_property_read_u32(entry, "reg", &id)) + return -EINVAL; + + report = of_get_parent(entry); + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != report) + continue; + ret = apple_pmp_report_set_state(rep, id, enable, false); + break; + } + rcu_read_unlock(); + of_node_put(report); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_request); + +static struct apple_pmp_report *apple_pmp_report_of_entry(const struct device_node *entry, + u32 *id) +{ + struct device_node *report; + struct apple_pmp_report *rep, *found = NULL; + + if (of_property_read_u32(entry, "reg", id) || *id >= 64) + return NULL; + report = of_get_parent(entry); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == report) { + found = rep; + break; + } + } + of_node_put(report); + + return found; +} + +/** + * apple_pmp_report_entry_commit_owned() - mark an entry withdrawn by its driver at the commit + * @entry: the report entry's node + * @owned: its driver withdraws it at the sleep's commit, or no longer does + * + * For a device whose stop moves to the sleep's commit: a PMP that stops there too keeps the + * request through apple_pmp_report_sleep(), until apple_pmp_report_entry_withdraw_polled(). + * + * Return: 0, or -ENODEV with no report bound for the entry. + */ +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned) +{ + struct apple_pmp_report *rep; + unsigned long flags; + int ret = -ENODEV; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + spin_lock_irqsave(&rep->lock, flags); + if (owned) + rep->commit_owned |= BIT_ULL(id); + else + rep->commit_owned &= ~BIT_ULL(id); + spin_unlock_irqrestore(&rep->lock, flags); + ret = 0; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_commit_owned); + +/** + * apple_pmp_report_entry_withdraw_polled() - withdraw an entry's request at the sleep's commit + * @entry: the report entry's node + * + * Interrupts off, one CPU: the request goes even with requests held for the sleep, and the + * PMP's acknowledgment is polled for while it runs. One already withdrawn returns at once. + * + * Return: 0, also with no report bound, or -ETIMEDOUT with no acknowledgment. + */ +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 bit, val; + int ret = 0; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + bit = BIT_ULL(id); + spin_lock_irqsave(&rep->lock, flags); + rep->known |= bit; + rep->shadow &= ~bit; + val = readq(rep->base + rep->offsets->tgt_read); + if (val & bit) + writeq(val & ~bit, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + if (apple_pmp_report_running(rep)) + ret = readq_poll_timeout_atomic(rep->base + rep->offsets->actual, val, + !(val & bit), 10, 50000); + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_withdraw_polled); + /* * System sleep. An external display processor's request keeps the PMP's * external display state up while nothing is shown, about 0.25 W in diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index a5d949d545029..5bd714eaf1b27 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -1334,6 +1334,24 @@ static void macsmc_power_remove(struct platform_device *pdev) blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); } +/* + * The charge moved while asleep, and an event's uevent that ran before the SMC + * was back found nothing to report. + */ +static void macsmc_power_complete(struct device *dev) +{ + struct macsmc_power *power = dev_get_drvdata(dev); + + if (power->batt) + power_supply_changed(power->batt); + if (power->ac) + power_supply_changed(power->ac); +} + +static const struct dev_pm_ops macsmc_power_pm_ops = { + .complete = pm_sleep_ptr(macsmc_power_complete), +}; + static const struct platform_device_id macsmc_power_id[] = { { .name = "macsmc-power" }, { /* sentinel */ } @@ -1343,6 +1361,7 @@ MODULE_DEVICE_TABLE(platform, macsmc_power_id); static struct platform_driver macsmc_power_driver = { .driver = { .name = "macsmc-power", + .pm = pm_sleep_ptr(&macsmc_power_pm_ops), }, .id_table = macsmc_power_id, .probe = macsmc_power_probe, diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index bae3a9f856c20..a7e6eba6e7910 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2099,10 +2099,14 @@ config RTC_DRV_APPLE_PMU depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP help Read the PMU clock on M1 Pro/Max/Ultra, M2, M4 Pro and A18 Pro - systems and use its alarm to wake the system - from suspend-to-idle. Alarms are limited to one hour; deeper - suspend states are not supported. Continue setting the shared - clock through the SMC RTC. + systems and use its alarm to wake the system from suspend-to-idle, + and on A18 Pro from a firmware-driven suspend too. A18 Pro alarms + reach as far as the PMU's comparator, and the clock can be set here; + elsewhere alarms are limited to one hour and the shared clock is set + through the SMC RTC. + + Say Y where this is the system's first RTC (rtc0), so that it sets + the system time at boot. This driver can be built as the module rtc-apple-pmu. diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 6043d7ee87a84..cd46c9b9a10b1 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #define APPLE_PMU_RTC_ARM BIT(6) #define APPLE_PMU_RTC_ALARM BIT(0) #define APPLE_PMU_RTC_HZ_SHIFT 16 +/* The suspend-to-idle limit, where the alarm is all that wakes the system. */ #define APPLE_PMU_RTC_MAX_DELAY 3600 /* @@ -33,6 +35,8 @@ struct apple_pmu_rtc_hw { u8 control_event_bit; u8 monitor_bit; bool offset_in_seconds; + /* How far ahead an alarm may be, in seconds */ + u32 max_delay; }; static const struct apple_pmu_rtc_hw apple_pmu_rtc_baku = { @@ -43,6 +47,8 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_baku = { .event = 0xf80c, .event_bit = BIT(0), .offset_in_seconds = true, + /* As far as the comparator reaches: macOS's own deep-sleep wakes use it. */ + .max_delay = U32_MAX, }; static const struct apple_pmu_rtc_hw apple_pmu_rtc_abbey = { @@ -53,6 +59,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_abbey = { .event = 0x2000, .event_bit = BIT(1), .monitor_bit = BIT(0), + .max_delay = APPLE_PMU_RTC_MAX_DELAY, }; /* M2 (t8112): Baku's register block with the offset in 1/32768 s. */ @@ -64,6 +71,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = { .event = 0xf80c, .event_bit = BIT(0), .monitor_bit = BIT(0), + .max_delay = APPLE_PMU_RTC_MAX_DELAY, }; /* M1 Pro, Max and Ultra (t600x), offset in 1/32768 s */ @@ -76,6 +84,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_maverick = { .event_bit = BIT(0), .control_event_bit = BIT(1), .monitor_bit = BIT(0), + .max_delay = APPLE_PMU_RTC_MAX_DELAY, }; struct apple_pmu_rtc { @@ -149,6 +158,31 @@ static int apple_pmu_rtc_read_time(struct device *dev, struct rtc_time *time) return 0; } +/* + * Baku's offset is in whole seconds, and this sets it as the SMC RTC does: the + * counter rounded to the nearest second, so the clock is at most half a second off. + */ +static int apple_pmu_rtc_set_time(struct device *dev, struct rtc_time *time) +{ + struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); + __le64 value; + u64 counter; + s64 offset; + int error; + + guard(mutex)(&rtc->transaction); + error = apple_pmu_rtc_counter(rtc, &counter); + if (error) + return error; + offset = rtc_tm_to_time64(time) - + (s64)((counter + BIT(APPLE_PMU_RTC_HZ_SHIFT - 1)) >> APPLE_PMU_RTC_HZ_SHIFT); + if (offset < 0 || offset > RTC_TIMESTAMP_END_9999 - U32_MAX) + return -ERANGE; + value = cpu_to_le64(offset); + error = nvmem_cell_write(rtc->offset, &value, sizeof(value)); + return error < 0 ? error : 0; +} + /* All comparator/control accesses are serialized with the IRQ thread. */ static int apple_pmu_rtc_ack_alarm(struct apple_pmu_rtc *rtc) { @@ -228,7 +262,7 @@ static int apple_pmu_rtc_validate_alarm(struct apple_pmu_rtc *rtc, u64 comparato counter >>= APPLE_PMU_RTC_HZ_SHIFT; if (comparator <= counter) return -ETIME; - if (comparator - counter > APPLE_PMU_RTC_MAX_DELAY) + if (comparator - counter > rtc->hw->max_delay) return -ERANGE; return 0; } @@ -375,6 +409,7 @@ static int apple_pmu_rtc_init_alarm(struct apple_pmu_rtc *rtc) return apple_pmu_rtc_stop_alarm(rtc); } +/* The others' offsets are set through the SMC RTC. */ static const struct rtc_class_ops apple_pmu_rtc_ops = { .read_time = apple_pmu_rtc_read_time, .read_alarm = apple_pmu_rtc_read_alarm, @@ -382,11 +417,26 @@ static const struct rtc_class_ops apple_pmu_rtc_ops = { .alarm_irq_enable = apple_pmu_rtc_alarm_enable, }; +static const struct rtc_class_ops apple_pmu_rtc_seconds_ops = { + .read_time = apple_pmu_rtc_read_time, + .set_time = apple_pmu_rtc_set_time, + .read_alarm = apple_pmu_rtc_read_alarm, + .set_alarm = apple_pmu_rtc_set_alarm, + .alarm_irq_enable = apple_pmu_rtc_alarm_enable, +}; + static int apple_pmu_rtc_prepare(struct device *dev) { struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || rtc->io_failed) + if (rtc->io_failed) + return -EBUSY; + /* + * A firmware-driven suspend programs the wake sources itself, and the + * alarm stays armed in the PMU, which stays powered. Only a sleep state + * without either has no way to bring the alarm back. + */ + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !pm_suspend_via_firmware()) return -EBUSY; return 0; } @@ -401,6 +451,7 @@ static void apple_pmu_rtc_release(void *context) struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); disable_irq(rtc->interrupt); + dev_pm_clear_wake_irq(dev); mutex_lock(&rtc->transaction); if (apple_pmu_rtc_stop_alarm(rtc)) dev_err(dev, "cannot disable RTC alarm on removal\n"); @@ -434,23 +485,27 @@ static int apple_pmu_rtc_probe(struct platform_device *pdev) rtc->clock = devm_rtc_allocate_device(dev); if (IS_ERR(rtc->clock)) return PTR_ERR(rtc->clock); - rtc->clock->ops = &apple_pmu_rtc_ops; + rtc->clock->ops = rtc->hw->offset_in_seconds ? &apple_pmu_rtc_seconds_ops : + &apple_pmu_rtc_ops; rtc->clock->range_min = 0; rtc->clock->range_max = RTC_TIMESTAMP_END_9999; - rtc->clock->alarm_offset_max = APPLE_PMU_RTC_MAX_DELAY; + rtc->clock->alarm_offset_max = rtc->hw->max_delay; clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->clock->features); error = device_init_wakeup(dev, true); if (error) return error; /* - * This interrupt remains live during retained-power s2idle. The - * thread uses the parent PMIC's SPMI regmap to acknowledge the alarm. - * Off-domain wake routing is not established by this driver. + * Let the IRQ core wake the system before the thread acknowledges the + * alarm over the parent PMIC's SPMI regmap. Off-domain wake routing is + * still supplied by the platform sleep implementation. */ error = devm_request_threaded_irq(dev, rtc->interrupt, NULL, apple_pmu_rtc_interrupt, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND | IRQF_ONESHOT, + IRQF_NO_AUTOEN | IRQF_ONESHOT, dev_name(dev), dev); + if (error) + goto disable_wakeup; + error = dev_pm_set_wake_irq(dev, rtc->interrupt); if (error) goto disable_wakeup; error = apple_pmu_rtc_init_alarm(rtc); @@ -466,6 +521,7 @@ static int apple_pmu_rtc_probe(struct platform_device *pdev) return 0; disable_wakeup: + dev_pm_clear_wake_irq(dev); device_init_wakeup(dev, false); return error; } diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 46068f25693e6..68b1ee93c13c8 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -97,6 +97,44 @@ config APPLE_PMGR_DCS_KUNIT_TEST Also verify that boot constraints and concurrent endpoints retain their votes, and that suspend restores the last confirmed request. +config APPLE_CLUSTER_PWRGATE + bool "Apple SoC CPU cluster power gates" + depends on OF + default ARCH_APPLE + help + Sets the power-gate enable of each CPU cluster the device tree + lists, early in boot. Without it, a core that the boot loader + started and PMGR never powered on, such as the boot CPU, only + clock-gates in WFI. + +config APPLE_SOC_SLEEP + bool "Apple SoC system sleep at EL2" + default APPLE_T8140_SLEEP + depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) + depends on HOTPLUG_CPU && SUSPEND + select APPLE_TUNABLE + help + Powers the secondary CPUs off and suspends the system to RAM on an + Apple SoC whose kernel runs directly at EL2, with no monitor to ask. + It does nothing unless the boot object asks for it through + /chosen/aurora,s2r-mailbox. + + Only SoCs whose sleep has been measured and tested are supported, + each by its match data: today T8140. On every other SoC the driver + doesn't bind, and s2idle stays the only system sleep. + +config APPLE_SOC_SLEEP_KUNIT_TEST + bool "KUnit tests for the Apple SoC sleep match data" if !KUNIT_ALL_TESTS + depends on APPLE_SOC_SLEEP && KUNIT=y + default KUNIT_ALL_TESTS + help + Checks each matched SoC's sleep data: that only tested SoCs are + matched, that every list names only words of the SoC's full lists, + with no duplicates, and that the lists fit the driver's books. The + tests touch no hardware. + + If unsure, say N. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX @@ -216,8 +254,9 @@ config APPLE_SEP_KUNIT_TEST mailbox message encoding and decoding, the shared-memory registration limits, the rules that keep a keybag's lookup identity apart from the identity the enclave reports, the format and validation of the - Linux host-state store, and the xART guard's decisions and its list - of Linux's records. The tests touch no hardware. + Linux host-state store, the xART guard's decisions and its list + of Linux's records, and how the fingerprint sensor is put to sleep. + The tests touch no hardware. If unsure, say N. @@ -401,3 +440,9 @@ config APPLE_AURORA_BOOT_SECURITY_LOCKDOWN endmenu endif + +config APPLE_T8140_SLEEP + bool + transitional + help + Transitional config for APPLE_T8140_SLEEP to APPLE_SOC_SLEEP migration. diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 8b36ffee52eae..4b28f67cdc58c 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -13,6 +13,11 @@ apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o +obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o + +obj-$(CONFIG_APPLE_SOC_SLEEP) += apple-sleep.o +apple-sleep-y = sleep.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index c062835a60011..2347073ddcdd7 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -63,6 +63,7 @@ use kernel::{ atomic::{ Acquire, Atomic, + Relaxed, Release, // }, Arc, @@ -108,12 +109,16 @@ const AFK_OPC_INIT_UNK: u64 = 0x8c; const AFK_OPC_SEND: u64 = 0xa2; const AFK_OPC_START_ACK: u64 = 0x86; const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_SLEEP_ACK: u64 = 0x87; const AFK_OPC_RECV: u64 = 0x85; const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; const AFK_MSG_START: u64 = 0xa3 << 48; const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +/// Stops an endpoint for a system sleep. It keeps its rings, and `AFK_MSG_START` +/// brings it back. +const AFK_MSG_SLEEP: u64 = 0xa4 << 48; const AFK_RB_BLOCK_STEP: usize = 0x40; const EPIC_TYPE_NOTIFY: u32 = 0; const EPIC_CATEGORY_REPORT: u8 = 0x00; @@ -141,6 +146,8 @@ const EPIC_CALL_TIMEOUT_MS: u32 = 5000; /// Bound on the wait for the shutdown acknowledgments of all endpoints /// together, so that a firmware that stopped answering costs one timeout. const AFK_SHUTDOWN_TIMEOUT_MS: u32 = 5000; +/// Bound on the answer to each stop or start around a system sleep. +const AFK_SLEEP_TIMEOUT_MS: u32 = 1000; // The T8140 AOP boots through a second mailbox, its "setup port", besides the // RTKit one. Its management endpoint runs a HELLO / endpoint map / power @@ -209,6 +216,32 @@ mod aop_ring; /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one /// of the endpoints this driver drives. +/// Iterates over `list`, or over every AFK endpoint if `list` is empty. +struct AfkOrder { + list: &'static [u8], + next: usize, +} + +impl AfkOrder { + fn new(list: &'static [u8]) -> AfkOrder { + AfkOrder { list, next: 0 } + } +} + +impl Iterator for AfkOrder { + type Item = u8; + + fn next(&mut self) -> Option { + let at = self.next; + self.next += 1; + if self.list.is_empty() { + (at < AFK_ENDPOINT_COUNT as usize).then(|| AFK_ENDPOINT_START + at as u8) + } else { + self.list.get(at).copied() + } + } +} + fn afk_endpoint_index(ep: u8) -> Option { ep.checked_sub(AFK_ENDPOINT_START) .filter(|i| *i < AFK_ENDPOINT_COUNT) @@ -472,7 +505,7 @@ impl AFKEndpoint { self.recv_get_buf(client.dev.clone(), rtkit, msg)?; } AFK_OPC_INIT_UNK => {} // no-op - AFK_OPC_START_ACK => {} + AFK_OPC_START_ACK | AFK_OPC_SLEEP_ACK => client.sleep_answered(self.index), AFK_OPC_INIT_RX => { if self.rxbuf.is_some() { dev_err!( @@ -1285,6 +1318,21 @@ struct AopData { call_turn: [Mutex<()>; AFK_ENDPOINT_COUNT as usize], #[pin] ep_shutdown: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// Answers to the stop and start of each endpoint around a system sleep. + #[pin] + ep_sleep: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// The order endpoints stop for a system sleep and start after it. Empty + /// means all started ones in endpoint order. + sleep_endpoints: &'static [u8], + wake_endpoints: &'static [u8], + /// No call starts until the system has resumed. + sleeping: Atomic, + /// RTKit reported the firmware crashed. Nothing restarts it short of a reboot. + crashed: Atomic, + /// The endpoints stopped for this sleep, by index. + sleep_stopped: Atomic, + /// The firmware was told the AP is going to sleep. + ap_idle: Atomic, #[pin] hid_listeners: Mutex>, #[pin] @@ -1422,6 +1470,13 @@ impl AopData { }), call_turn <- pin_init::pin_init_array_from_fn(|_| new_mutex!(())), ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + ep_sleep <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + sleep_endpoints: cfg.sleep_endpoints, + wake_endpoints: cfg.wake_endpoints, + sleeping: Atomic::new(false), + crashed: Atomic::new(false), + sleep_stopped: Atomic::new(0), + ap_idle: Atomic::new(false), hid_listeners <- new_mutex!(KVec::new()), report_listeners <- new_mutex!(KVec::new()), source_ring <- new_mutex!(None), @@ -1631,6 +1686,127 @@ impl AopData { ret } + fn sleep_answered(&self, endpoint: u8) { + if let Some(index) = afk_endpoint_index(endpoint) { + self.ep_sleep[index].complete(()); + } + } + + /// Sends `msg` to one endpoint and waits for its answer. + fn sleep_switch(&self, endpoint: u8, msg: u64) -> Result<()> { + let index = afk_endpoint_index(endpoint).ok_or(EINVAL)?; + self.ep_sleep[index].reset(); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + rtk.send_message(endpoint, msg)?; + } + self.ep_sleep[index] + .wait_timeout(AFK_SLEEP_TIMEOUT_MS) + .ok_or(ETIMEDOUT) + } + + /// Tells the firmware about the AP's power state. Waits with the RTKit lock + /// dropped, as the answer shares a worker with messages that take it. + fn ap_power(&self, on: bool) -> Result<()> { + let rtk = (*self.rtkit.lock()) + .as_ref() + .map(|rtk| rtk.as_raw()) + .ok_or(ENODEV)?; + // SAFETY: the handle lives until removal, which holds the device lock + // that system sleep also runs under. + to_result(unsafe { bindings::apple_rtkit_ap_power(rtk, on) }) + } + + /// A crashed firmware hears nothing more, so it's left out of every sleep from then on, + /// rather than refusing them all. + fn crashed_skip(&self) -> bool { + self.crashed.load(Acquire) + } + + /// Puts back what `sleep_enter` did, in the order the firmware expects. + fn sleep_undo(&self) { + if self.crashed_skip() { + self.sleep_stopped.store(0, Relaxed); + self.ap_idle.store(false, Relaxed); + self.sleeping.store(false, Release); + return; + } + let stopped = self.sleep_stopped.xchg(0, Relaxed); + let was_idle = self.ap_idle.load(Relaxed); + if self.ap_idle.xchg(false, Relaxed) { + if let Err(e) = self.ap_power(true) { + dev_err!(self.dev, "Wake notice not answered ({:?})", e); + } + } + let order = AfkOrder::new(self.wake_endpoints); + for endpoint in order { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if stopped & (1 << index) == 0 { + continue; + } + if let Err(e) = self.sleep_switch(endpoint, AFK_MSG_START) { + dev_err!(self.dev, "Endpoint {:#04x} did not restart ({:?})", endpoint, e); + } + } + self.sleeping.store(false, Release); + if was_idle { + dev_dbg!(self.dev, "AP on notice sent and endpoints restarted after a sleep\n"); + } + } + + /// The AOP keeps running through a firmware-driven suspend. It stops its + /// endpoints and hears the AP's power state, then the reverse after a wake, + /// with no firmware restart. Measured on T8140, unverified on other SoCs. + fn sleep_enter(&self) -> Result<()> { + if !self.cpu_started.load(Acquire) { + return Ok(()); + } + if self.crashed_skip() { + dev_warn!(self.dev, "firmware crashed: sleeping without it\n"); + self.sleeping.store(true, Release); + return Ok(()); + } + self.sleeping.store(true, Release); + // Any call in progress finishes first; new ones see `sleeping`. + for turn in &self.call_turn { + drop(turn.lock()); + } + + let mut result = Ok(()); + for endpoint in AfkOrder::new(self.sleep_endpoints) { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if !self.endpoints[index].lock().started { + continue; + } + result = self.sleep_switch(endpoint, AFK_MSG_SLEEP).inspect_err(|e| { + dev_err!(self.dev, "Endpoint {:#04x} did not stop ({:?})", endpoint, e); + }); + if result.is_err() { + break; + } + self.sleep_stopped + .store(self.sleep_stopped.load(Relaxed) | (1 << index), Relaxed); + } + if result.is_ok() { + // Set first: a notice without an answer may still have been taken. + self.ap_idle.store(true, Relaxed); + result = self.ap_power(false).inspect_err(|e| { + dev_err!(self.dev, "Sleep notice not answered ({:?})", e); + }); + } + if result.is_err() { + self.sleep_undo(); + } else { + dev_dbg!(self.dev, "Endpoints stopped and AP idle notice answered for a sleep\n"); + } + result + } + fn patch_bootargs( &self, aop_mmio: &Mmio<'_, Region>, @@ -1945,8 +2121,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -1965,8 +2148,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -2613,6 +2803,7 @@ impl rtkit::Operations for AopData { } fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + data.crashed.store(true, Release); dev_err!(data.dev, "AOP firmware crashed"); } } @@ -2635,6 +2826,10 @@ struct AopHwConfig { epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. setup_port: bool, + /// The order endpoints stop for a system sleep; empty means endpoint order. + sleep_endpoints: &'static [u8], + /// The order they start after one. + wake_endpoints: &'static [u8], /// Its service endpoints are the ones its EPMAP announces, not T8140's. setup_epmap_endpoints: bool, /// Its boot waits for the system endpoints the EPMAP names as well. @@ -2661,6 +2856,8 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: false, @@ -2679,6 +2876,8 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: false, @@ -2696,6 +2895,8 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: false, @@ -2712,6 +2913,8 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: false, @@ -2733,6 +2936,8 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: true, @@ -2758,6 +2963,8 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: true, + sleep_endpoints: &[0x2b, 0x23, 0x22, 0x20, 0x21], + wake_endpoints: &[0x20, 0x21, 0x22, 0x2b, 0x23], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: true, @@ -2776,6 +2983,8 @@ const HW_CFG_T8152: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: false, setup_wait_system_endpoints: true, quiesce_on_unbind: true, @@ -2798,6 +3007,8 @@ const HW_CFG_T8142: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: true, + sleep_endpoints: &[], + wake_endpoints: &[], setup_epmap_endpoints: true, setup_wait_system_endpoints: true, quiesce_on_unbind: true, @@ -2814,6 +3025,50 @@ const HW_CFG_T6050: AopHwConfig = AopHwConfig { ..HW_CFG_T8142 }; +/// The data of a bound AOP device. +/// +/// # Safety +/// +/// `dev` is an AOP device between the end of probe and the start of removal. +unsafe fn aop_from_dev<'a>(dev: *mut bindings::device) -> &'a AopData { + // SAFETY: by the caller's contract, drvdata is the `AopDriver` probe set. + let driver = unsafe { &*bindings::dev_get_drvdata(dev).cast::() }; + let data: *const dyn AOP = &*driver.0; + // SAFETY: probe made this `Arc` from an `Arc`. + unsafe { &*data.cast::() } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_suspend_late(dev: *mut bindings::device) -> c_int { + // SAFETY: only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: by this function's contract. + match unsafe { aop_from_dev(dev) }.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_resume(dev: *mut bindings::device) -> c_int { + // SAFETY: by this function's contract. + unsafe { aop_from_dev(dev) }.sleep_undo(); + 0 +} + +static AOP_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend_late: Some(aop_suspend_late), + resume: Some(aop_resume), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + kernel::of_device_table!( OF_TABLE, ::IdInfo, @@ -2837,6 +3092,10 @@ impl platform::Driver for AopDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/apple-cluster-pwrgate.c b/drivers/soc/apple/apple-cluster-pwrgate.c new file mode 100644 index 0000000000000..67d493cff8af6 --- /dev/null +++ b/drivers/soc/apple/apple-cluster-pwrgate.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC CPU cluster power gates + * + * Each CPU cluster has a PMGR word whose top bit lets its cores power down + * in WFI. A core PMGR powers on comes up allowed to, but the boot CPU was + * started by the boot ROM, so it only follows the word, which the boot + * loader may leave clear. Set it for every cluster the device tree lists. + */ + +#include +#include +#include +#include +#include +#include + +#define CLUSTER_PWRGATE_EN BIT(31) + +static const struct of_device_id apple_cluster_pwrgate_of_match[] __initconst = { + { .compatible = "apple,cluster-pwrgate" }, + { } +}; + +/* Before the secondaries start. A core picks the word up at its next WFI. */ +static int __init apple_cluster_pwrgate_init(void) +{ + struct device_node *np; + + for_each_matching_node(np, apple_cluster_pwrgate_of_match) { + void __iomem *gate; + int i; + + if (!of_device_is_available(np)) + continue; + + /* One word per cluster. Shared with the sleep code, so not claimed. */ + for (i = 0; (gate = of_iomap(np, i)); i++) { + u32 was = readl(gate); + + writel(was | CLUSTER_PWRGATE_EN, gate); + pr_debug("%pOF: cluster %d power gate %#x, now %#x\n", np, i, was, + readl(gate)); + iounmap(gate); + } + } + + return 0; +} +early_initcall(apple_cluster_pwrgate_init); diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 946c95d7a46d3..644354f5ddb06 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -164,6 +164,7 @@ pub(crate) struct ControlState { retired: [u64; 4], retired_count: u32, entropy: EntropySink, + wake_seen: u8, } impl ControlState { @@ -181,6 +182,7 @@ impl ControlState { retry_at: None, retry_ms: ENTROPY_RETRY_MIN_MS, }, + wake_seen: 0, } } @@ -218,7 +220,7 @@ impl ControlState { } else { tag + 1 }; - if !self.tag_in_flight(tag) && !self.is_retired(tag) { + if tag != proto::TAG_SLEEP && !self.tag_in_flight(tag) && !self.is_retired(tag) { self.slots[idx] = Slot { used: true, tag, @@ -230,6 +232,45 @@ impl ControlState { Err(EBUSY) } + /// Claims the sleep request's own tag, which the pool never hands out. A + /// request still in flight under it means the last one was never answered. + pub(crate) fn alloc_sleep(&mut self) -> Result { + let tag = proto::TAG_SLEEP; + if self.tag_in_flight(tag) { + return Err(EBUSY); + } + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + self.reclaim_retired(tag); + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + Ok(idx) + } + + /// Whether any request is waiting for its reply. + pub(crate) fn requests_in_flight(&self) -> bool { + self.slots.iter().any(|s| s.used) + } + + /// Forget the wake messages seen, before asking for them. + pub(crate) fn wake_reset(&mut self) { + self.wake_seen = 0; + } + + /// A control message that isn't a reply: note it if it's a wake message. + pub(crate) fn wake_note(&mut self, ty: u8) { + if let Some(n) = proto::WAKE_TYPES.iter().position(|&t| t == ty) { + self.wake_seen |= 1 << n; + } + } + + /// All the wake messages have come. + pub(crate) fn wake_complete(&self) -> bool { + self.wake_seen == (1 << proto::WAKE_TYPES.len()) - 1 + } + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { self.slots[idx].reply.take() } @@ -419,6 +460,55 @@ mod tests { } } + #[test] + fn the_pool_never_hands_out_the_sleep_tag() { + let mut state = ControlState::new(); + for _ in 0..POOL_LEN * 2 { + let (idx, tag) = state.alloc().unwrap(); + assert_ne!(tag, proto::TAG_SLEEP); + state.release(idx); + } + } + + #[test] + fn a_sleep_request_gets_its_own_answer() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + assert!(state.requests_in_flight()); + assert_eq!(state.alloc_sleep(), Err(EBUSY)); + let reply = proto::ControlReply { + tag: proto::TAG_SLEEP, + data_lo: 0, + }; + assert!(matches!(state.deliver(reply), Delivery::Matched)); + assert!(state.take_reply(idx).is_some()); + state.release(idx); + assert!(!state.requests_in_flight()); + } + + #[test] + fn a_sleep_request_can_follow_one_that_timed_out() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + state.abandon(idx); + assert!(state.alloc_sleep().is_ok()); + } + + #[test] + fn a_wake_takes_both_messages() { + let mut state = ControlState::new(); + state.wake_reset(); + assert!(!state.wake_complete()); + state.wake_note(0x33); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[0]); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[1]); + assert!(state.wake_complete()); + state.wake_reset(); + assert!(!state.wake_complete()); + } + #[test] fn a_request_is_answered() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f4f674f548c5d..cad52f94fc81f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -46,6 +46,7 @@ #define APPLE_ASC_MBOX_A2I_RECV1 0x818 #define APPLE_ASC_MBOX_I2A_CONTROL 0x114 +#define APPLE_ASC_MBOX_CONTROL_RECV_ENABLE BIT(0) #define APPLE_ASC_MBOX_I2A_SEND0 0x820 #define APPLE_ASC_MBOX_I2A_SEND1 0x828 #define APPLE_ASC_MBOX_I2A_RECV0 0x830 @@ -228,6 +229,8 @@ struct apple_mbox_hw { /* Status word counts RX occupancy and free TX slots; paired transfers. */ bool counted_fifo; bool ap_initializes_mailboxes; + /* No interrupts: the consumer calls apple_mbox_poll() while it expects traffic. */ + bool polled; unsigned int control_full; unsigned int control_empty; @@ -340,14 +343,11 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); -static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +static void apple_mbox_ascwrap_v6_run_off(struct apple_mbox *mbox) { void __iomem *cpu_control; u32 val; - if (!mbox->ascwrap_v6_cpu_running) - return; - if (mbox->hw->is_ascwrap_v8) { u32 state, ack; @@ -377,12 +377,24 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) val = readl(cpu_control); writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); } +} +static void apple_mbox_ascwrap_v6_stopped_locked(struct apple_mbox *mbox) +{ mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + if (!mbox->ascwrap_v6_cpu_running) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); + apple_mbox_ascwrap_v6_stopped_locked(mbox); +} + /* * A failed resume leaves the runtime PM error set, and runtime PM then fails * every later resume with -EINVAL without retrying the power domain. The @@ -486,6 +498,44 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return NULL; + + return (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_control); + +/* No lock and no runtime PM: the registers alone, as apple_mbox_ascwrap_v6_stop_cpu() has them. */ +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs || + !READ_ONCE(mbox->ascwrap_v6_cpu_running)) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu_atomic); + +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) + ret = -ENODEV; + else if (mbox->ascwrap_v6_cpu_running) + apple_mbox_ascwrap_v6_stopped_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_stopped); + static bool apple_mbox_is_g17(const struct apple_mbox *mbox) { return mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX || @@ -704,7 +754,8 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, * it unmasked for them. The handler masks it when it fires, * and removal masks it otherwise. */ - t = wait_for_completion_interruptible_timeout( + /* Not for the freezer's signal: a message half sent can break a handshake. */ + t = wait_for_completion_killable_timeout( &mbox->tx_empty, min(remaining, msecs_to_jiffies(APPLE_MBOX_TX_RECHECK_MS))); if (t < 0) @@ -891,6 +942,12 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) return IRQ_HANDLED; } +bool apple_mbox_send_ready(struct apple_mbox *mbox) +{ + return !(readl_relaxed(mbox->regs + mbox->hw->a2i_control) & mbox->hw->control_full); +} +EXPORT_SYMBOL(apple_mbox_send_ready); + int apple_mbox_poll(struct apple_mbox *mbox) { unsigned long flags; @@ -907,6 +964,49 @@ int apple_mbox_poll(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_poll); +/* RAM-only capability check, before a client sends an irreversible request. */ +bool apple_mbox_poll_one_supported(struct apple_mbox *mbox) +{ + return !mbox->hw->is_ascwrap_v8 && !mbox->hw->counted_fifo && + !mbox->hw->has_irq_controls; +} +EXPORT_SYMBOL_GPL(apple_mbox_poll_one_supported); + +/* + * One message with the receive IRQ held. No MMIO follows its callback, including + * a status reread or IRQ ACK. 0 means empty, 1 consumed, <0 error. This narrow + * primitive is not a receive protocol for ACK/counting/PSD-based transports. + */ +int apple_mbox_poll_one(struct apple_mbox *mbox) +{ + struct apple_mbox_msg msg; + unsigned long flags; + u32 control; + int ret = 0; + + spin_lock_irqsave(&mbox->rx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + ret = -ENODEV; + goto out; + } + if (!apple_mbox_poll_one_supported(mbox)) { + ret = -EOPNOTSUPP; + goto out; + } + if (!mbox->rx) + goto out; + control = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + if (!apple_mbox_rx_empty(mbox, control)) { + msg = apple_mbox_read_msg(mbox); + mbox->rx(mbox, msg, mbox->cookie); + ret = 1; + } +out: + spin_unlock_irqrestore(&mbox->rx_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_poll_one); + /* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ static void apple_mbox_v8_rx_poll_work(struct work_struct *work) { @@ -1015,7 +1115,8 @@ int apple_mbox_start(struct apple_mbox *mbox) goto out_unlock; } - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; out_unlock: @@ -1034,7 +1135,8 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) apple_mbox_v8_rx_stop(mbox); } else { mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1048,6 +1150,25 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +/* + * Bit 0 of the receive control word, read and written back. A client whose + * firmware powers itself down clears it when the firmware has answered, and sets + * it before starting the firmware again. That it gates receiving is inferred. + */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&mbox->rx_lock, flags); + val = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + val = enabled ? val | APPLE_ASC_MBOX_CONTROL_RECV_ENABLE : + val & ~APPLE_ASC_MBOX_CONTROL_RECV_ENABLE; + writel_relaxed(val, mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} +EXPORT_SYMBOL_GPL(apple_mbox_set_recv_enabled); + /* * The mailbox's power domain was off while it stayed started (its consumer * powered its co-processor down and up again around it): set the @@ -1069,13 +1190,15 @@ EXPORT_SYMBOL(apple_mbox_restore); void apple_mbox_hold_rx(struct apple_mbox *mbox) { - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_hold_rx); void apple_mbox_release_rx(struct apple_mbox *mbox) { - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_release_rx); @@ -1116,7 +1239,7 @@ static void apple_mbox_quiesce(void *data) if (was_active) { if (apple_mbox_has_rx_poller(mbox)) apple_mbox_v8_rx_stop(mbox); - else + else if (mbox->irq_recv_not_empty >= 0) disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1136,7 +1259,8 @@ static void apple_mbox_quiesce(void *data) if (mbox->irq_send_empty >= 0) synchronize_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); - synchronize_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + synchronize_irq(mbox->irq_recv_not_empty); spin_lock_irqsave(&mbox->rx_lock, flags); had_rx = mbox->rx; @@ -1411,19 +1535,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; } - mbox->irq_recv_not_empty = - platform_get_irq_byname(pdev, "recv-not-empty"); - if (mbox->irq_recv_not_empty < 0) - return mbox->irq_recv_not_empty; + if (mbox->hw->polled) { + mbox->irq_recv_not_empty = -ENXIO; + mbox->irq_send_empty = -ENXIO; + } else { + mbox->irq_recv_not_empty = + platform_get_irq_byname(pdev, "recv-not-empty"); + if (mbox->irq_recv_not_empty < 0) + return mbox->irq_recv_not_empty; - /* - * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. - * apple_mbox_send then polls the A2I control register instead. - */ - mbox->irq_send_empty = - platform_get_irq_byname_optional(pdev, "send-empty"); - if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return mbox->irq_send_empty; + /* + * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. + * apple_mbox_send then polls the A2I control register instead. + */ + mbox->irq_send_empty = + platform_get_irq_byname_optional(pdev, "send-empty"); + if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) + return mbox->irq_send_empty; + } mbox->rx_util_min = -1; mbox->rx_util_min_applied = -1; @@ -1432,21 +1561,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); - if (!irqname) - return -ENOMEM; + if (mbox->irq_recv_not_empty >= 0) { + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); + if (!irqname) + return -ENOMEM; - if (apple_mbox_is_g17(mbox)) - ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, - apple_mbox_recv_irq, - IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, - irqname, mbox); - else - ret = devm_request_irq(dev, mbox->irq_recv_not_empty, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); - if (ret) - return ret; + if (apple_mbox_is_g17(mbox)) + ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, + apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_NO_SUSPEND, + irqname, mbox); + else + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (ret) + return ret; + } if (mbox->irq_send_empty >= 0) { irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", @@ -1627,6 +1759,24 @@ static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { .has_irq_controls = false, }; +/* The kind of port the secure RTBuddy proxy has, with the AOP setup port's registers. */ +static const struct apple_mbox_hw apple_mbox_secure_rtbuddy_hw = { + .polled = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_T8140_AOP_SETUP_A2I_CONTROL, + .a2i_send0 = APPLE_T8140_AOP_SETUP_A2I_SEND0, + .a2i_send1 = APPLE_T8140_AOP_SETUP_A2I_SEND1, + + .i2a_control = APPLE_T8140_AOP_SETUP_I2A_CONTROL, + .i2a_recv0 = APPLE_T8140_AOP_SETUP_I2A_RECV0, + .i2a_recv1 = APPLE_T8140_AOP_SETUP_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -1750,6 +1900,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6050-j775d-ascwrap-v6", .data = &apple_mbox_t6050_j775d_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, + { .compatible = "apple,secure-rtbuddy-proxy", .data = &apple_mbox_secure_rtbuddy_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* * t6030 gfx-asc mailbox in the boot.bin DTB: it carries no asc-mailbox-v4 fallback so that diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 3b9b47a6d6213..12d40d05b305f 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) #include @@ -39,6 +41,8 @@ struct apple_mcc { unsigned int nr_planes; bool enabled; bool retained; + /* Turned off for a deep sleep, to be turned back on after it. */ + bool sleep_disabled; }; static void apple_mcc_writel(struct apple_mcc *mcc, unsigned int plane, @@ -236,6 +240,43 @@ static void apple_mcc_shutdown(struct platform_device *pdev) apple_mcc_quiesce(mcc); } +/* + * A deep sleep takes the cache's power with the rest of the SoC: stop allocating and write it + * back first, as at shutdown, and allocate again after. Suspend-to-idle keeps it. + */ +static int apple_mcc_suspend_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + + if (!pm_suspend_via_firmware() || !mcc->enabled || mcc->retained) + return 0; + apple_mcc_quiesce(mcc); + mcc->sleep_disabled = true; + + return 0; +} + +static int apple_mcc_resume_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + int ret; + + if (!mcc->sleep_disabled) + return 0; + mcc->sleep_disabled = false; + /* Only a performance setting: a cache not ready after the wake stays off. */ + ret = apple_mcc_enable(mcc); + mcc->enabled = !ret; + if (ret) + dev_warn(dev, "system level cache allocation stays off after the sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_mcc_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_mcc_suspend_noirq, apple_mcc_resume_noirq) +}; + static const struct of_device_id apple_mcc_of_match[] = { { .compatible = "apple,t8132-mcc" }, { .compatible = "apple,t8140-mcc" }, @@ -247,6 +288,7 @@ static struct platform_driver apple_mcc_driver = { .driver = { .name = "apple-mcc", .of_match_table = apple_mcc_of_match, + .pm = pm_sleep_ptr(&apple_mcc_pm_ops), }, .probe = apple_mcc_probe, .remove = apple_mcc_remove, diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 3701f08c18487..28ec03e7e9fb7 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -8,6 +8,10 @@ //! is not frozen either, so entropy requests are held over the same window. //! The notifier itself is in `pm_shim.c`; the capture and entropy handling is //! on [`SepData`]. +//! +//! On a firmware-driven suspend the enclave also saves its state and gates its +//! own power domain when asked, as the sleep powers it down, and has to be +//! started again after a wake. See [`SleepGate`] and `SepData::sleep_enter`. use core::sync::atomic::{AtomicPtr, Ordering}; @@ -23,9 +27,120 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; + fn sep_pm_register( + dev: *mut c_void, + event: unsafe extern "C" fn(bool), + sleep: unsafe extern "C" fn() -> c_int, + wake: unsafe extern "C" fn(), + told: unsafe extern "C" fn() -> bool, + ) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; + fn sep_pm_clear_dev_disable(dev: *mut c_void) -> c_int; + fn sep_pm_dart_restore(dev: *mut c_void); + fn sep_pm_commit_prepare( + dev: *mut c_void, + mbox: *mut c_void, + msg0: u64, + msg1: u32, + reply: u64, + mask: u64, + ) -> c_int; + fn sep_pm_commit_finish(dev: *mut c_void) -> c_int; + pub(crate) static sep_pm_ops: kernel::bindings::dev_pm_ops; +} + +/// Who may use the mailbox around a firmware-driven suspend. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SleepGate { + /// Anyone. + Open = 0, + /// Replies to what the enclave asked, and the sleep request itself. + Draining = 1, + /// Nobody until the wake: the enclave is gated or about to be. + Told = 2, + /// Nobody: a previous wake failed. New firmware sleeps refuse until reboot. + Lost = 3, +} + +impl SleepGate { + pub(crate) fn from_u32(v: u32) -> SleepGate { + match v { + 0 => SleepGate::Open, + 1 => SleepGate::Draining, + 2 => SleepGate::Told, + _ => SleepGate::Lost, + } + } + + pub(crate) fn may_request(self) -> bool { + self == SleepGate::Open + } + + pub(crate) fn may_reply(self) -> bool { + matches!(self, SleepGate::Open | SleepGate::Draining) + } + + /// The enclave may be gated, so a wake has to look. + pub(crate) fn needs_wake(self) -> bool { + matches!(self, SleepGate::Told | SleepGate::Lost) + } +} + +/// Stages `request` for the irreversible commit, with RX held until physical wake. +/// Unsupported providers/transports refuse here; device suspend never sends SLEEP. +pub(crate) fn leave_for_commit( + dev: &kernel::device::Device, + mbox: *mut kernel::bindings::apple_mbox, + request: kernel::soc::apple::mailbox::Message, + reply: u64, + mask: u64, +) -> Result<()> { + // SAFETY: `dev` is the bound SEP device and `mbox` its mailbox, which outlives the sleep: + // `commit_finish` takes the request back at the resume. + let rc = unsafe { + sep_pm_commit_prepare( + dev.as_raw().cast(), + mbox.cast(), + request.msg0, + request.msg1, + reply, + mask, + ) + }; + if rc == 0 { + Ok(()) + } else { + Err(Error::from_errno(rc)) + } +} + +/// Finishes a staged request. A sent request requires a real firmware return; +/// the C boundary terminates an impossible sent-without-sleep unwind. `None` if unstaged. +pub(crate) fn commit_finish(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_commit_finish(dev.as_raw().cast()) }; + if rc < 0 { + None + } else { + Some(rc > 0) + } +} + +/// Puts the enclave's DART back if it lost its state, before the doorbell. +pub(crate) fn dart_restore(dev: &kernel::device::Device) { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_dart_restore(dev.as_raw().cast()) }; +} + +/// Lets a gated enclave start again. Writes only if the bit is set. +pub(crate) fn clear_dev_disable(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_clear_dev_disable(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) } /// Attaches every power domain of a multi-domain SEP node and keeps them on @@ -92,7 +207,41 @@ unsafe extern "C" fn pm_event(entering: bool) { sleep_event(&*sep, entering); } +/// Runs `f` on the registered instance. Only for callbacks that +/// `sep_pm_unregister` waits for or that the bound device excludes. +fn with_sep(none: R, f: impl FnOnce(&SepData) -> R) -> R { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return none; + } + // SAFETY: as in `pm_event`. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + f(&*sep) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's suspend callback. +unsafe extern "C" fn pm_sleep() -> c_int { + with_sep(0, |sep| match sep.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + }) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's resume callback. +unsafe extern "C" fn pm_wake() { + with_sep((), |sep| sep.sleep_exit()); +} + +/// # Safety +/// Called by `pm_shim.c`, whether or not the device suspended. +unsafe extern "C" fn pm_told() -> bool { + with_sep(false, |sep| sep.sleep_gate() == SleepGate::Told) +} + pub(crate) fn register(sep: Arc) -> Result<()> { + let dev: *mut c_void = sep.dev.as_raw().cast(); let ptr = sep.into_foreign(); if SEP @@ -111,7 +260,7 @@ pub(crate) fn register(sep: Arc) -> Result<()> { // SAFETY: the shim stores the callback and registers a notifier block // that lives for the module's lifetime. - let rc = unsafe { sep_pm_register(pm_event) }; + let rc = unsafe { sep_pm_register(dev, pm_event, pm_sleep, pm_wake, pm_told) }; if rc != 0 { let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); if !raw.is_null() { @@ -187,6 +336,33 @@ mod tests { } } + #[test] + fn the_gate_lets_replies_out_until_the_enclave_is_told() { + assert!(SleepGate::Open.may_request()); + assert!(SleepGate::Open.may_reply()); + assert!(!SleepGate::Draining.may_request()); + assert!(SleepGate::Draining.may_reply()); + assert!(!SleepGate::Told.may_request()); + assert!(!SleepGate::Told.may_reply()); + assert!(!SleepGate::Lost.may_request()); + assert!(!SleepGate::Lost.may_reply()); + } + + #[test] + fn only_a_told_or_lost_enclave_needs_waking() { + assert!(!SleepGate::Open.needs_wake()); + assert!(!SleepGate::Draining.needs_wake()); + assert!(SleepGate::Told.needs_wake()); + assert!(SleepGate::Lost.needs_wake()); + } + + #[test] + fn the_gate_round_trips() { + for gate in [SleepGate::Open, SleepGate::Draining, SleepGate::Told, SleepGate::Lost] { + assert_eq!(SleepGate::from_u32(gate as u32), gate); + } + } + #[test] fn going_to_sleep_holds_entropy_then_captures() { let calls = Calls::new(); diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 76846ad5aaac3..c1d415c0bd744 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -3,18 +3,51 @@ * System sleep for the SEP driver: the notifier only forwards "about to sleep" * and "back from sleep" to Rust, which ends any Touch ID capture before the * system suspends (see pm.rs), and the enclave's power domains are kept on - * across sleep. + * across sleep. The device's suspend and resume callbacks are the firmware + * suspend's: the enclave is told, and gates its own domain, whose state word is + * read here. On T8140 at EL2 the sleep driver sends the request at the + * irreversible commit boundary. Only errors before send may refuse. */ #include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include +#include + #include "shim.h" +/* The state word of a PMGR power domain, as pmgr-pwrstate has it. */ +#define SEP_PS_DEV_DISABLE BIT(10) +#define SEP_PS_ACTUAL GENMASK(7, 4) + +/* One-shot precommit test: no SLEEP request or commit coprocessor stop. */ +static bool sleep_refuse_before_send; +module_param(sleep_refuse_before_send, bool, 0644); +MODULE_PARM_DESC(sleep_refuse_before_send, + "Test only: refuse the next prepared deep sleep before sending SEP SLEEP"); + static void (*sep_pm_event)(bool entering); +static int (*sep_pm_sleep)(void); +static void (*sep_pm_wake)(void); +static bool (*sep_pm_told)(void); +/* In the device's .suspend, the only caller that can leave the request to the sleep. */ +static bool sep_pm_suspending; static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) @@ -46,15 +79,420 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(void (*event)(bool entering)) +/* The enclave's own domain is the first one its node lists. */ +static int sep_pm_domain(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + + *regmap = syscon_node_to_regmap(np->parent); + ret = PTR_ERR_OR_ZERO(*regmap); + if (!ret) + ret = of_property_read_u32(np, "reg", offset); + of_node_put(np); + + return ret; +} + +/* + * Returns 0 once the enclave has gated itself, 1 if the deadline expired, or a + * negative observation error (including -ENOENT if there is no domain to read). + * Keep a read's -ETIMEDOUT distinct from the polling deadline. DEV_DISABLE is + * the evidence: a running enclave idles at actual state 0 with auto-enable on. + */ +int sep_pm_wait_gated(void *ptr, unsigned int timeout_ms) +{ + struct regmap *regmap; + u32 offset, val; + int ret, poll_ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + poll_ret = read_poll_timeout(regmap_read, ret, ret || (val & SEP_PS_DEV_DISABLE), + 1000, timeout_ms * USEC_PER_MSEC, false, + regmap, offset, &val); + if (ret) + return ret; + + return !!poll_ret; +} + +int sep_pm_clear_dev_disable(void *ptr) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + ret = regmap_read(regmap, offset, &val); + if (ret || !(val & SEP_PS_DEV_DISABLE)) + return ret; + + return regmap_write(regmap, offset, val & ~SEP_PS_DEV_DISABLE); +} + +/* The enclave's DART, the first one its node lists, with a reference held. */ +static struct device *sep_pm_dart(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + + np = of_parse_phandle(dev->of_node, "iommus", 0); + if (!np) + return NULL; + pdev = of_find_device_by_node(np); + of_node_put(np); + + return pdev ? &pdev->dev : NULL; +} + +void sep_pm_dart_restore(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + + if (dart) { + apple_dart_sleep_restore(dart); + put_device(dart); + } +} + +#define SEP_PM_POLL_US 20 +#define SEP_PM_TIMEOUT_US (5000 * USEC_PER_MSEC) + +/* + * .suspend only stages this request and holds RX. The platform sends it after + * the last legitimate refusal. Once sent, only physical sleep/wake is supported; + * a failed/aborted SLEEP is fatal as on the native path, not an in-place restart. + */ +static struct sep_pm_commit { + struct device *dev; + struct device *dart; + struct apple_mbox *mbox; + void __iomem *ps; + struct apple_mbox_msg request; + u64 reply, reply_mask; + void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); + void *cookie; + bool registered, sent, answered, gate_valid; + u8 reply_result; + unsigned int answer_us; + int gate_ret; + u32 state; +} sep_pm_commit; + +/* The answer stops here. Anything else goes to the driver, which reads it after the wake. */ +static void sep_pm_commit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct sep_pm_commit *c = cookie; + + if (c->sent && (msg.msg0 & c->reply_mask) == c->reply) { + /* Control reply result is byte3; the match mask is below it. */ + c->reply_result = (msg.msg0 >> 24) & 0xff; + c->answered = true; + return; + } + if (c->rx) + c->rx(mbox, msg, c->cookie); +} + +static void sep_pm_commit_take_rx(struct sep_pm_commit *c, bool take) +{ + unsigned long flags; + + spin_lock_irqsave(&c->mbox->rx_lock, flags); + if (take) { + c->rx = c->mbox->rx; + c->cookie = c->mbox->cookie; + c->mbox->rx = sep_pm_commit_rx; + c->mbox->cookie = c; + } else { + c->mbox->rx = c->rx; + c->mbox->cookie = c->cookie; + } + spin_unlock_irqrestore(&c->mbox->rx_lock, flags); +} + +/* Nonzero returns are permitted only before the request was sent. */ +static int sep_pm_commit_stop(void *data) +{ + struct sep_pm_commit *c = data; + unsigned int us; + bool prepared = false; + int ret = 0; + + c->sent = false; + c->answered = false; + c->answer_us = 0; + c->reply_result = 0xff; + c->gate_ret = -EINVAL; + c->gate_valid = false; + c->state = 0; + + sep_pm_commit_take_rx(c, true); + /* No SLEEP sent yet: drain what was already waiting. */ + ret = apple_mbox_poll(c->mbox); + if (ret < 0) + goto out; + for (us = 0; !apple_mbox_send_ready(c->mbox); us += SEP_PM_POLL_US) { + if (us >= SEP_PM_TIMEOUT_US) { + ret = -ETIMEDOUT; + goto out; + } + udelay(SEP_PM_POLL_US); + } + ret = apple_dart_sleep_prepare(c->dart); + if (ret) + goto out; + prepared = true; + /* The PM core's earlier check precedes the potentially long preparation. */ + if (pm_wakeup_pending()) { + ret = -EBUSY; + goto out; + } + if (xchg(&sleep_refuse_before_send, false)) { + dev_info(c->dev, "test: precommit refusal before SEP SLEEP; request not sent\n"); + ret = -ECANCELED; + goto out; + } + /* Commit decision: a new wake from here must finish physical sleep/wake. */ + ret = apple_mbox_send(c->mbox, c->request, true); + if (ret) + goto out; + c->sent = true; + while (!c->answered && !((c->state = readl(c->ps)) & SEP_PS_DEV_DISABLE) && + c->answer_us < SEP_PM_TIMEOUT_US) { + ret = apple_mbox_poll_one(c->mbox); + if (ret < 0) + break; + if (!c->answered) { + udelay(SEP_PM_POLL_US); + c->answer_us += SEP_PM_POLL_US; + } + } +out: + /* Callback ownership is RAM only; RX remains held until a real wake. */ + sep_pm_commit_take_rx(c, false); + if (!c->sent) { + if (prepared) + apple_dart_sleep_restore(c->dart); + return ret; + } + /* Preserve the failure reason without another hardware query. */ + if (ret >= 0 && !c->answered) + ret = c->state & SEP_PS_DEV_DISABLE ? -EHOSTDOWN : -ETIMEDOUT; + if (c->answered && c->reply_result) + ret = -EREMOTEIO; + /* Native cmsgSLEEP errors/abort/timeout are fatal, never cancellation. */ + if (ret < 0) + panic("SEP SLEEP failed after send: ret %d reply_valid %d result %#x state %#x", + ret, c->answered, c->reply_result, c->state); + /* + * Native success-only receive close. No mailbox access follows it. The + * held RX IRQ adapts native workloop exclusion; access-until-close is + * the native contract, not proved atomic by the preceding PMGR sample. + */ + apple_mbox_set_recv_enabled(c->mbox, false); + c->gate_ret = readl_poll_timeout_atomic(c->ps, c->state, c->state & SEP_PS_DEV_DISABLE, + SEP_PM_POLL_US, SEP_PM_TIMEOUT_US); + c->gate_valid = true; + if (c->gate_ret) + panic("SEP SLEEP did not gate after send: %d state %#x", c->gate_ret, c->state); + + return 0; +} + +/* + * From .suspend, with the enclave drained and its receive interrupt held: leaves the + * request in @msg0 and @msg1 to the irreversible commit. Its answer is the message + * whose @mask bits read @reply. Unsupported paths must refuse without sending. + */ +int sep_pm_commit_prepare(void *ptr, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask) +{ + struct sep_pm_commit *c = &sep_pm_commit; + struct device *dev = ptr; + struct device_node *np; + struct resource res; + u32 offset; + int ret; + + /* Check capability before resources that legacy providers need not have. */ + if (!READ_ONCE(sep_pm_suspending) || !IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) || + !is_kernel_in_hyp_mode()) + return -EOPNOTSUPP; + if (c->registered) + return -EBUSY; + /* Reject ACK/counting/PSD transports before any request or MMIO. */ + if (!apple_mbox_poll_one_supported(mbox)) + return -EOPNOTSUPP; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + ret = of_property_read_u32(np, "reg", &offset); + if (!ret) + ret = np->parent ? of_address_to_resource(np->parent, 0, &res) : -EINVAL; + of_node_put(np); + if (ret) + return ret; + if (offset + 4 > resource_size(&res)) + return -EINVAL; + + c->dart = sep_pm_dart(dev); + if (!c->dart || !dev_get_drvdata(c->dart)) { + ret = -ENODEV; + goto err; + } + c->ps = ioremap(res.start + offset, 4); + if (!c->ps) { + ret = -ENOMEM; + goto err; + } + c->dev = dev; + c->mbox = mbox; + c->request.msg0 = msg0; + c->request.msg1 = msg1; + c->reply = reply; + c->reply_mask = mask; + c->sent = false; + /* Resume may follow a pre-tell abort: observations belong to this attempt. */ + c->answered = false; + c->answer_us = 0; + c->reply_result = 0xff; + c->gate_ret = -EINVAL; + c->gate_valid = false; + c->state = 0; + ret = apple_sleep_enclave_register(dev, sep_pm_commit_stop, c); + if (ret) + goto err; + c->registered = true; + + return 0; + +err: + if (c->ps) + iounmap(c->ps); + c->ps = NULL; + if (c->dart) + put_device(c->dart); + c->dart = NULL; + return ret; +} + +/* After the sleep, refused or not: 1 if the request went out, 0 if not, <0 if none was left. */ +int sep_pm_commit_finish(void *ptr) +{ + struct sep_pm_commit *c = &sep_pm_commit; + + if (!c->registered || c->dev != ptr) + return -ENOENT; + if (c->sent && !pm_resume_via_firmware()) + panic("SEP SLEEP was sent without a physical sleep/wake"); + apple_sleep_enclave_unregister(c->dev); + c->registered = false; + dev_dbg(c->dev, + "sleep commit finished: sent=%d reply_valid=%d result=%#x answer_us=%u gate_valid=%d gate_ret=%d state=%#x\n", + c->sent, c->answered, c->reply_result, c->answer_us, + c->gate_valid, c->gate_ret, c->state); + iounmap(c->ps); + c->ps = NULL; + put_device(c->dart); + c->dart = NULL; + + return c->sent; +} + +/* + * Only a firmware-driven suspend tells the enclave anything, so every other sleep + * stays as it was. Refusing is safe: the PM core runs the resume callback of a + * device whose suspend had passed when a later one failed, and sleep_enter undoes + * what it did if its own step fails. No SLEEP is sent during device suspend. + */ +static int __maybe_unused sep_pm_suspend(struct device *dev) +{ + int (*sleep)(void) = READ_ONCE(sep_pm_sleep); + int ret; + + if (!pm_suspend_via_firmware() || !sleep) + return 0; + + WRITE_ONCE(sep_pm_suspending, true); + ret = sleep(); + WRITE_ONCE(sep_pm_suspending, false); + + return ret; +} + +/* The last step before the platform's own: the domain has to be down by now. */ +static int __maybe_unused sep_pm_suspend_noirq(struct device *dev) +{ + bool (*told)(void) = READ_ONCE(sep_pm_told); + int ret; + + if (!pm_suspend_via_firmware() || !told || !told()) + return 0; + + ret = sep_pm_wait_gated(dev, 5000); + if (ret > 0) + ret = -ETIMEDOUT; + if (ret == -ENOENT) + return 0; + if (ret) { + dev_err(dev, "the enclave did not gate its power domain: %d\n", ret); + } else { + struct regmap *regmap; + u32 offset, val = 0; + + if (!sep_pm_domain(dev, ®map, &offset)) + regmap_read(regmap, offset, &val); + dev_dbg(dev, "the enclave gated its power domain, state %#x\n", val); + } + + return ret; +} + +static int __maybe_unused sep_pm_resume(struct device *dev) +{ + void (*wake)(void) = READ_ONCE(sep_pm_wake); + + if (wake) + wake(); + + return 0; +} + +const struct dev_pm_ops sep_pm_ops = { + .suspend = pm_sleep_ptr(sep_pm_suspend), + .resume = pm_sleep_ptr(sep_pm_resume), + .suspend_noirq = pm_sleep_ptr(sep_pm_suspend_noirq), +}; + +int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)) { int ret; WRITE_ONCE(sep_pm_event, event); + WRITE_ONCE(sep_pm_sleep, sleep); + WRITE_ONCE(sep_pm_wake, wake); + WRITE_ONCE(sep_pm_told, told); ret = register_pm_notifier(&sep_pm_nb); - if (ret) + if (ret) { WRITE_ONCE(sep_pm_event, NULL); - return ret; + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); + return ret; + } + + return 0; } /* Returns only once no notifier call is still running. */ @@ -62,6 +500,9 @@ void sep_pm_unregister(void) { unregister_pm_notifier(&sep_pm_nb); WRITE_ONCE(sep_pm_event, NULL); + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); } static void sep_pm_release_domains(void *data) diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 41091755a77c6..195452b8f0d02 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -1,15 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include #include #include +#include +#include #include +#include #include +#include +#include +#include /* Join removal and serialize the firmware's untagged power reply stream. */ static DEFINE_MUTEX(pmp_lock); static const void *pmp_data; static struct device *pmp_device; +/* The firmware is stopped for a system sleep. */ +static bool pmp_asleep; int apple_pmp_register(struct device *dev, const void *data) { @@ -66,6 +75,8 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; /* The matched native report must be ready; never use an elapsed-time fallback. */ ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) @@ -104,9 +115,153 @@ int apple_pmp_t6030_gpu_ready(void) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); if (ret) return ret; return apple_pmp_report_set_t6030_gfx(pmp_device, true); } EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); + +/* + * The PMP's idle is a point of no return: an idled PMP doesn't come back with its power kept, + * and only a real sleep restarts it. Where the sleep driver can take them, its idle + * and stop move to the sleep's last step, after every refusal: a refused sleep then finds it + * still running, and the GPU's request, withdrawn at the commit, reaches a running PMP. + */ +static struct apple_pmp_commit_stop pmp_commit; +static bool pmp_commit_registered, pmp_commit_used; + +/* Interrupts off, one CPU: what apple_pmp_sleep() does, through pointers taken before. */ +static int apple_pmp_commit_stop_fn(void *unused) +{ + struct apple_pmp_commit_stop *c = &pmp_commit; + unsigned int us; + size_t i; + int ret; + + ret = apple_rtkit_sleep_idle_polled(c->rtk, c->control, &us); + apple_rtkit_mailbox_recv_off(c->rtk); + for (i = 0; c->sram && i < c->count; i++) + c->words[i] = readl_relaxed(c->sram + 4 * i); + /* Told to idle, it can't carry on whatever it answered: the sleep goes on. */ + return ret ? 1 : 0; +} + +/* + * Retained J713 power votes have no teardown. On T8140 an unbind leaves the firmware running, + * and a commit with it running reset the SoC: its sleep registration is what refuses + * then, every sleep until reboot, since a rebind can't register again. + */ +bool apple_pmp_keep_bound(void) +{ + return apple_pmp_display_retained_board() || of_machine_is_compatible("apple,t8140"); +} + +/* Whether the sleep's last step takes the PMP: registered with the sleep driver, once. */ +static bool apple_pmp_commit_available(struct device *dev) +{ + void __iomem *asc; + int idx; + + if (pmp_commit_registered) + return true; + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return false; + if (apple_sleep_coproc_register(dev, asc, false, apple_pmp_commit_stop_fn, NULL)) { + iounmap(asc); + return false; + } + pmp_commit_registered = true; + + return true; +} + +static int apple_pmp_commit_prepare(struct device *dev) +{ + int ret; + + ret = apple_pmp_sleep_prepare(pmp_data, &pmp_commit); + if (ret) + return ret; + apple_sleep_coproc_set_ready(dev, true); + pmp_commit_used = true; + dev_dbg(dev, "idle and stop left for the sleep's commit\n"); + + return 0; +} + +static int apple_pmp_suspend_late(struct device *dev) +{ + bool at_commit; + int ret; + + if (!pm_suspend_via_firmware()) + return 0; + /* Retained power votes have no teardown. */ + if (apple_pmp_display_retained_board()) + return -EBUSY; + + guard(mutex)(&pmp_lock); + if (pmp_device != dev || pmp_asleep) + return 0; + at_commit = apple_pmp_commit_available(dev); + /* Idled any earlier, a refusal after it would leave it stopped. */ + if (!at_commit && apple_sleep_commit_stops_coprocs()) { + dev_err(dev, "no stop at the sleep's commit: refusing the sleep\n"); + return -EBUSY; + } + apple_pmp_report_sleep(dev, at_commit); + ret = at_commit ? apple_pmp_commit_prepare(dev) : apple_pmp_sleep(pmp_data); + if (ret) { + apple_pmp_report_wake(dev, false); + return dev_err_probe(dev, ret, "firmware did not go idle for sleep\n"); + } + pmp_asleep = true; + + return 0; +} + +static int apple_pmp_resume_early(struct device *dev) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_asleep) + return 0; + /* Left for the commit: if the sleep was refused before it, the firmware never idled. */ + if (pmp_commit_used) { + bool stopped = apple_sleep_coproc_stopped(dev); + + pmp_commit_used = false; + apple_sleep_coproc_set_ready(dev, false); + if (!pm_resume_via_firmware() && !stopped) { + pmp_asleep = false; + apple_pmp_report_wake(dev, true); + dev_info(dev, "sleep refused before the commit: firmware never stopped\n"); + return 0; + } + } + /* A stopped firmware doesn't restart with its power kept: the next real sleep does. */ + if (!pm_resume_via_firmware()) { + dev_warn(dev, "sleep refused: firmware stays stopped until a real sleep\n"); + return 0; + } + pmp_asleep = false; + ret = apple_pmp_wake(pmp_data); + if (ret) + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + else + dev_dbg(dev, "firmware back from sleep\n"); + /* Requests go back either way: consumers read them from the report too. */ + apple_pmp_report_wake(dev, true); + + return 0; +} + +const struct dev_pm_ops apple_pmp_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmp_suspend_late, apple_pmp_resume_early) +}; diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 3a9f970bf4893..ff2c1bf973e04 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -71,6 +71,7 @@ const BOOTARGS_UUID_MAGIC: u32 = 0x6469_7575; // "uuid" const BOOTARGS_UUID_VERSION: u32 = 5; const BOOTARGS_OFFSET: usize = 0x22c; const BOOTARGS_SIZE: usize = 0x230; +const ASC_CPU_CONTROL_OFFSET: usize = 0x44; const PMP_ENDPOINT: u8 = 0x20; const PM_COMMAND_MARKER: u64 = pmp_power::MARKER; const PM_COMMAND_SHIFT: u32 = pmp_power::SHIFT; @@ -120,11 +121,19 @@ unsafe impl AsBytes for IovaTableEntry {} // SAFETY: TODO: unsafe impl FromBytes for IovaTableEntry {} +/// A range of the firmware's SRAM as it was when the firmware stopped. +struct SramCopy { + offset: usize, + words: KVVec, +} + struct PmpState { iova_table: Option>, allocs: KVec, value_buf: Option, ioreg_entries: KVec, + /// Held from a system sleep's start until its end. + sram: Option, } impl PmpState { @@ -134,6 +143,7 @@ impl PmpState { allocs: KVec::with_capacity(10, GFP_KERNEL)?, value_buf: None, ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + sram: None, }) } fn find_alloc(&self, addr: u64) -> Option { @@ -169,10 +179,12 @@ struct PmpData { display: AtomicPtr, /// The first failure seen before or after the bridge was published. display_error: AtomicI32, + /// See [`PmpHwConfig::sram_backup`]. + sram_backup: Option<(usize, usize)>, } impl PmpData { - fn new(dev: &platform::Device>) -> Result> { + fn new(dev: &platform::Device>, cfg: &PmpHwConfig) -> Result> { let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = pmp_req.iomap_sized::()?.into_devres()?; let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; @@ -189,6 +201,7 @@ impl PmpData { asc_mmio, display: AtomicPtr::new(core::ptr::null_mut()), display_error: AtomicI32::new(0), + sram_backup: cfg.sram_backup, } ), GFP_KERNEL, @@ -438,6 +451,112 @@ impl PmpData { ) }; } + /// The handle, to wait on the firmware with the lock free: its messages take that lock. + fn rtkit_handle(&self) -> Result<*mut bindings::apple_rtkit> { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + Ok(rtk.as_raw()) + } + /// The ASC's CPU control register. + fn asc_cpu_control(&self) -> Result<*mut kernel::ffi::c_void> { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let asc_mmio = self.asc_mmio.access(bound_dev)?.as_view(); + Ok(MmioBackend::as_ptr(asc_mmio) + .cast::() + .wrapping_add(ASC_CPU_CONTROL_OFFSET) + .cast()) + } + /// Room for what SRAM needs kept, taken first so that running out refuses a sleep cleanly. + fn sram_alloc(&self, sram_size: usize) -> Result> { + let Some((offset, size)) = self.sram_backup else { + return Ok(None); + }; + if !pmp_bootargs::word_range_valid(offset, size, sram_size) { + return Err(EINVAL); + } + let mut words = KVVec::new(); + words.resize(size / 4, 0u32, GFP_KERNEL)?; + Ok(Some(SramCopy { offset, words })) + } + /// Stops the firmware for a system sleep, then keeps what its restart needs from SRAM. + fn sleep(&self) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + let mut copy = self.sram_alloc(io.size())?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + if let Err(error) = to_result(unsafe { bindings::apple_rtkit_sleep_stop(rtk, control) }) { + // SAFETY: as above. + if to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }).is_err() { + self.fail_display(error); + } + return Err(error); + } + if let Some(copy) = copy.as_mut() { + for (idx, word) in copy.words.iter_mut().enumerate() { + // The range was checked against the mapping when it was allocated. + *word = io.try_read32(copy.offset + idx * 4).unwrap_or(0); + } + } + self.state.lock().sram = copy; + Ok(()) + } + /// For a stop at the commit: the RTKit handle, the CPU control and the SRAM copy's room, + /// whose buffer stays where it is once moved into the state. + fn sleep_prepare(&self, stop: &mut bindings::apple_pmp_commit_stop) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let size = pmp_mmio.as_view().relaxed().size(); + let mut copy = self.sram_alloc(size)?; + stop.rtk = self.rtkit_handle()?; + stop.control = self.asc_cpu_control()?; + stop.sram = core::ptr::null_mut(); + stop.words = core::ptr::null_mut(); + stop.count = 0; + if let Some(copy) = copy.as_mut() { + stop.sram = MmioBackend::as_ptr(pmp_mmio) + .cast::() + .wrapping_add(copy.offset) + .cast(); + stop.words = copy.words.as_mut_ptr(); + stop.count = copy.words.len(); + } + self.state.lock().sram = copy; + Ok(()) + } + /// Writes SRAM back after a sleep that took its power, while the firmware is stopped. + fn sram_restore(&self) -> Result { + let Some(copy) = self.state.lock().sram.take() else { + return Ok(()); + }; + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + for (idx, word) in copy.words.iter().enumerate() { + io.try_write32(*word, copy.offset + idx * 4)?; + } + // The firmware mustn't start ahead of writes still on their way. + let _ = io.try_read32(copy.offset); + dev_dbg!(self.dev, "SRAM restored: {} bytes\n", copy.words.len() * 4); + Ok(()) + } + /// Brings the firmware back after a system sleep. It keeps its endpoints and everything the + /// host gave it, so nothing here repeats what it asked for at probe. + fn wake(&self) -> Result { + self.sram_restore()?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }) + } fn recv_message(&self, msg: u64) -> Result<()> { let pm_command = (msg >> PM_COMMAND_SHIFT) & 0xff; if msg & PM_COMMAND_MARKER != 0 && (pm_command == 0x0f || pm_command == 0x10) { @@ -611,6 +730,58 @@ pub unsafe extern "C" fn apple_pmp_send_power_command( 0 } +/// Stops the firmware for a system sleep. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.sleep() { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Takes what the sleep's last step needs to stop the firmware with interrupts off, and the room +/// for the SRAM it keeps. Nothing is stopped here. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver, and `stop` to writable memory. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep_prepare( + data: *const core::ffi::c_void, + stop: *mut bindings::apple_pmp_commit_stop, +) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + // SAFETY: The caller passes a valid pointer. + let stop = unsafe { &mut *stop }; + match data.sleep_prepare(stop) { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Brings the firmware back after a system sleep. A firmware that doesn't come back disables +/// further power commands. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_wake(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.wake() { + Ok(()) => 0, + Err(error) => { + data.fail_display(error); + error.to_errno() + } + } +} + struct NoBuffer; impl rtkit::Buffer for NoBuffer { fn iova(&self) -> Result { @@ -659,6 +830,9 @@ struct PmpHwConfig { /// When the firmware needs system endpoint 7 (CoreAnalytics) started /// and given its buffer before it acknowledges the IOP power state. coreanalytics: Ep7, + /// The range of the firmware's SRAM, as (offset, size) in bytes, that a system sleep keeps + /// and writes back. `None` where no sleep has been traced. + sram_backup: Option<(usize, usize)>, } /// Whether a PMP firmware needs endpoint 7 to finish its RTKit boot. @@ -676,6 +850,7 @@ const HW_CFG_DEFAULT: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; /// T8140: the DART translates 42 bits and its DMA window starts at 1 TiB. @@ -683,6 +858,7 @@ const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: Some((0x3c000, 0x5c000)), }; /// T6041: the DART window starts at 1 TiB; the boot argument header is @@ -692,6 +868,7 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: true, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 @@ -701,6 +878,7 @@ const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T6050: the DART window starts at 1 TiB (ADT dart-pmp0 vm-base) and spans @@ -709,12 +887,14 @@ const HW_CFG_T6050: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::RetainedBoard, + sram_backup: None, }; impl Drop for PmpDriver { @@ -742,9 +922,14 @@ impl platform::Driver for PmpDriver { type Data<'bound> = PmpDriver; fn suppress_bind_attrs() -> bool { - // Retained J713 power votes have no teardown path. - // SAFETY: read-only board identity check; other boards remain unchanged. - unsafe { bindings::apple_pmp_display_retained_board() } + // Retained J713 power votes have no teardown path, and an unbound T8140 PMP still runs. + // SAFETY: read-only machine identity checks; other machines remain unchanged. + unsafe { bindings::apple_pmp_keep_bound() } + } + + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pmp-export.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(bindings::apple_pmp_pm_ops) }) } const OF_ID_TABLE: Option> = Some(&OF_TABLE); @@ -760,7 +945,7 @@ impl platform::Driver for PmpDriver { // yet. unsafe { pdev.dma_set_mask_and_coherent(mask)? }; } - let data = PmpData::new(pdev)?; + let data = PmpData::new(pdev, cfg)?; let pmp_mmio = data.pmp_mmio.access(pdev.as_ref())?.as_view(); let asc_mmio = data.asc_mmio.access(pdev.as_ref())?.as_view(); let node = dev.fwnode().ok_or(EIO)?; diff --git a/drivers/soc/apple/pmp_bootargs.rs b/drivers/soc/apple/pmp_bootargs.rs index c085a323fee47..bd2abbc94029b 100644 --- a/drivers/soc/apple/pmp_bootargs.rs +++ b/drivers/soc/apple/pmp_bootargs.rs @@ -7,6 +7,11 @@ pub(super) fn region_valid(offset: usize, size: usize, sram_size: usize) -> bool size != 0 && offset <= sram_size && size <= sram_size - offset } +/// A range of SRAM that can be copied as 32-bit words. +pub(super) fn word_range_valid(offset: usize, size: usize, sram_size: usize) -> bool { + region_valid(offset, size, sram_size) && offset % 4 == 0 && size % 4 == 0 +} + fn header(bytes: &[u8]) -> Option<(u32, usize)> { let key = u32::from_le_bytes(bytes.get(..4)?.try_into().ok()?); let size = u32::from_le_bytes(bytes.get(4..8)?.try_into().ok()?) as usize; @@ -69,6 +74,15 @@ mod tests { assert!(!region_valid(1, usize::MAX, usize::MAX)); } + #[test] + fn a_word_range_is_aligned_and_inside_the_window() { + assert!(word_range_valid(0x3c000, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c002, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x5c002, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x64004, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0, 0xa0000)); + } + #[test] fn larger_sram_uses_its_actual_extent() { assert!(region_valid(0x90000, 0x1000, 0x100000)); diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 8fc7278550d9a..1bdea86a4c51e 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -90,6 +90,32 @@ pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; pub(crate) const TAG_ENTROPY: u8 = 0xE7; +/// The tag of the sleep request, kept out of the pool. +pub(crate) const TAG_SLEEP: u8 = 0x08; + +/// The sleep request's answer, in the bits [`SLEEP_REPLY_MASK`] keeps: the control endpoint, +/// its tag and the reply type. +pub(crate) const SLEEP_REPLY: u64 = (EP_CONTROL as u64) + | ((TAG_SLEEP as u64) << MSG_TAG_SHIFT) + | ((CONTROL_REPLY_TYPE as u64) << MSG_TYPE_SHIFT); +pub(crate) const SLEEP_REPLY_MASK: u64 = (1 << MSG_PARAM_SHIFT) - 1; + +// The answer in the trace, 0x0000_0000_0001_0800. +static_assert!(0x0000_0000_0001_0800 & SLEEP_REPLY_MASK == SLEEP_REPLY); + +/// How long the enclave gets to answer the sleep request. +pub(crate) const SLEEP_TIMEOUT_MS: u32 = 5000; + +/// The first message macOS sends a gated enclave after a sleep. The contents +/// vary from sleep to sleep, so it is taken to be a doorbell. This is one of them. +pub(crate) const WAKE_KICK: Message = Message { + msg0: 0xffff_ffff_a482_9303, + msg1: 0, +}; + +/// The control messages a woken enclave sends without being asked, in order. +pub(crate) const WAKE_TYPES: [u8; 2] = [0x0d, 0x10]; + pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; pub(crate) const TAG_POOL_FIRST: u8 = 0x01; @@ -97,6 +123,7 @@ pub(crate) const TAG_POOL_LAST: u8 = 0x7e; static_assert!(TAG_POOL_FIRST > 0); static_assert!(TAG_POOL_LAST < TAG_ENTROPY); +static_assert!(TAG_SLEEP >= TAG_POOL_FIRST && TAG_SLEEP <= TAG_POOL_LAST); pub(crate) struct ControlOp { ty: u8, @@ -161,6 +188,19 @@ pub(crate) fn op_get_entropy() -> ControlOp { } } +/// Asks the enclave to save its state and gate its own power domain. +pub(crate) fn op_sleep() -> ControlOp { + ControlOp { + ty: 0x0c, + param: 0x02, + data: 0, + expect_reply: true, + timeout_ms: SLEEP_TIMEOUT_MS, + reserved_tag: false, + name: c"SLEEP", + } +} + pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { Message { msg0: u64::from(EP_CONTROL) @@ -282,6 +322,34 @@ mod tests { assert_eq!(f.data_lo, 0x1234_5678); } + #[test] + fn sleep_request_matches_the_trace() { + let msg = encode_control(&op_sleep(), TAG_SLEEP); + assert_eq!(msg.msg0, 0x0000_0000_020c_0800); + assert_eq!(msg.msg1, 0); + } + + #[test] + fn its_answer_matches_the_trace() { + let reply = Message { + msg0: 0x0000_0000_0001_0800, + msg1: 0, + }; + let f = decode(&reply); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, CONTROL_REPLY_TYPE); + assert_eq!(f.tag, TAG_SLEEP); + } + + #[test] + fn wake_messages_match_the_trace() { + for (raw, ty) in [(0x000d_0000u64, WAKE_TYPES[0]), (0x0010_0000u64, WAKE_TYPES[1])] { + let f = decode(&Message { msg0: raw, msg1: 0 }); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, ty); + } + } + #[test] fn only_entropy_uses_the_reserved_tag() { assert!(op_get_entropy().uses_reserved_tag()); diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index e4d3dd0f65711..7c4521ef6783b 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -40,6 +40,12 @@ struct apple_rtkit_helper { * it does not list must not replace storage the firmware still owns. */ bool strict_buffers; + /* The firmware was told the AP is going to sleep. */ + bool ap_idle; + /* It was told, gave no answer, and wasn't told otherwise. */ + bool idle_unanswered; + /* Ends a sleep whose .prepare failed, which gets no .complete. */ + struct notifier_block pm_nb; void __iomem *asc_base; @@ -238,6 +244,8 @@ static int apple_rtkit_helper_inherit_buffers( return count; } +static int apple_rtkit_helper_pm_register(struct apple_rtkit_helper *helper); + static int apple_rtkit_helper_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -340,6 +348,9 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->rtk); if (ret) return ret; + ret = apple_rtkit_helper_pm_register(helper); + if (ret) + return ret; /* There is no safe module-unload boundary for the retained session. */ __module_get(THIS_MODULE); @@ -364,7 +375,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) if (ret != 0) return dev_err_probe(dev, ret, "Failed to wake up coprocessor\n"); - return 0; + return apple_rtkit_helper_pm_register(helper); } static void apple_rtkit_helper_remove(struct platform_device *pdev) @@ -381,10 +392,117 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); } +static int apple_rtkit_helper_ap_power(struct apple_rtkit_helper *helper, bool on) +{ + int ret = apple_rtkit_ap_power(helper->rtk, on); + + if (ret) + dev_err(helper->dev, "AP power notice (%s) not answered: %d\n", + on ? "on" : "idle", ret); + /* + * Nothing is sent to undo an unanswered idle notice, which macOS never does: + * the end of the sleep looks whether a late answer came, and only then sends AP on. + */ + if (ret && !on) + helper->idle_unanswered = true; + return ret; +} + +static int apple_rtkit_helper_ap_idle(struct apple_rtkit_helper *helper) +{ + int ret; + + if (helper->ap_idle) + return 0; + ret = apple_rtkit_helper_ap_power(helper, false); + if (ret) + return ret; + helper->ap_idle = true; + dev_dbg(helper->dev, "AP idle notice answered for a sleep\n"); + return 0; +} + +static void apple_rtkit_helper_ap_on(struct apple_rtkit_helper *helper) +{ + if (!helper->ap_idle) + return; + helper->ap_idle = false; + if (!apple_rtkit_helper_ap_power(helper, true)) + dev_dbg(helper->dev, "AP on notice answered after a sleep\n"); +} + +/* + * macOS tells this coprocessor first of all. Told before any driver stops its own, + * an unanswered notice refuses a sleep that has stopped nothing yet. + */ +static int __maybe_unused apple_rtkit_helper_prepare(struct device *dev) +{ + if (!pm_suspend_via_firmware()) + return 0; + return apple_rtkit_helper_ap_idle(dev_get_drvdata(dev)); +} + +static void apple_rtkit_helper_sleep_end(struct apple_rtkit_helper *helper) +{ + if (helper->idle_unanswered) { + bool late = !apple_rtkit_is_running(helper->rtk); + + helper->idle_unanswered = false; + dev_info(helper->dev, "after the unanswered idle notice: %s\n", + late ? "its answer came late, sending AP on" : + "still no answer, nothing sent"); + helper->ap_idle = late; + } + apple_rtkit_helper_ap_on(helper); +} + +/* A sleep refused before this device's .suspend never calls its .resume. */ +static void __maybe_unused apple_rtkit_helper_complete(struct device *dev) +{ + apple_rtkit_helper_sleep_end(dev_get_drvdata(dev)); +} + +/* The PM core calls no .complete after a failed .prepare: end that sleep here. */ +static int apple_rtkit_helper_pm_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct apple_rtkit_helper *helper = container_of(nb, struct apple_rtkit_helper, pm_nb); + + if (action == PM_POST_SUSPEND && helper->idle_unanswered) + apple_rtkit_helper_sleep_end(helper); + return NOTIFY_DONE; +} + +static void apple_rtkit_helper_pm_unregister(void *data) +{ + struct apple_rtkit_helper *helper = data; + + unregister_pm_notifier(&helper->pm_nb); +} + +static int apple_rtkit_helper_pm_register(struct apple_rtkit_helper *helper) +{ + int ret; + + helper->pm_nb.notifier_call = apple_rtkit_helper_pm_notify; + ret = register_pm_notifier(&helper->pm_nb); + if (ret) + return ret; + return devm_add_action_or_reset(helper->dev, apple_rtkit_helper_pm_unregister, helper); +} + static int __maybe_unused apple_rtkit_helper_suspend(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + /* + * A firmware-driven suspend keeps this coprocessor running. It hears the + * AP's power state and nothing else, so an adopted session stays live. + * That is measured on T8140 and unverified on other SoCs. + */ + if (pm_suspend_via_firmware()) + return apple_rtkit_helper_ap_idle(helper); + if (helper->adopted && pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; if (!helper->adopted || !of_machine_is_compatible("apple,j613")) @@ -401,6 +519,8 @@ static int __maybe_unused apple_rtkit_helper_resume(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + /* After a firmware-driven suspend's idle notice; nothing otherwise. */ + apple_rtkit_helper_ap_on(helper); if (!helper->adopted || !of_machine_is_compatible("apple,j613")) return 0; if (!(readl(helper->asc_base + APPLE_ASC_CPU_CONTROL) & APPLE_ASC_CPU_CONTROL_RUN)) { @@ -420,6 +540,8 @@ static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) } static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { + .prepare = pm_sleep_ptr(apple_rtkit_helper_prepare), + .complete = pm_sleep_ptr(apple_rtkit_helper_complete), .suspend = pm_sleep_ptr(apple_rtkit_helper_suspend), .resume = pm_sleep_ptr(apple_rtkit_helper_resume), .freeze = pm_sleep_ptr(apple_rtkit_helper_freeze), diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index fb365ebfe1921..7551fe6aca285 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -63,6 +63,17 @@ struct apple_rtkit { * hand out kdebug buffers, and poll the mailbox while waiting. */ bool sync_syslog_ack; + /* Echo each OSLOG channel-0 counter, as macOS does. */ + bool oslog_echo; + /* A counter queued for the receive work and not yet echoed. */ + u64 oslog_owed; + /* Device interrupts are off: poll the mailbox while waiting. */ + bool poll_waits; + /* The firmware is restarting with its buffers and rings still in place. */ + bool warm_restart; + u32 warm_last_mask; + /* A port that only answers the co-processor, see apple_rtkit_port_handshake(). */ + bool port_only; DECLARE_KFIFO(sync_ack_fifo, struct apple_mbox_msg, 64); struct work_struct sync_ack_work; @@ -93,6 +104,8 @@ struct apple_rtkit { /* Session inherited from the bootloader: INIT geometry never observed. */ bool syslog_inherited; bool crashlog_inherited; + /* The crash log buffer is the firmware's own, mapped here when it crashed. */ + bool crashlog_placed; bool oslog_inherited; /* A crash reported before the crashlog buffer was mapped: the log is diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 65faedc3da557..8535fd432f4b6 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -7,10 +7,14 @@ #include "rtkit-internal.h" #include +#include +#include #include #include #include #include +#include +#include enum { APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */ @@ -45,7 +49,7 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, - /* Sent by some co-processors once idle after an IOP IDLE request. */ + /* Sent by some co-processors once idle after an IOP IDLE request, argument 0. */ APPLE_RTKIT_MGMT_IDLE_NOTIFY = 0xc, }; @@ -63,6 +67,9 @@ enum { #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0) +#define APPLE_RTKIT_POLLED_STEP_US 10 +#define APPLE_RTKIT_POLLED_LOOPS 20000 + #define APPLE_RTKIT_CRASHLOG_CRASH 1 #define APPLE_RTKIT_BUFFER_REQUEST 1 @@ -117,6 +124,14 @@ enum { #define APPLE_RTKIT_CA_EVENT 2 #define APPLE_RTKIT_CA_EVENT_ACK 3 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +/* From the CPU control register. Both bits wait for the firmware's last message to be read. */ +#define APPLE_ASC_CPU_STATUS 4 +#define APPLE_ASC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_ASC_CPU_STATUS_IDLE BIT(5) +#define APPLE_ASC_CPU_STATUS_DOWN (APPLE_ASC_CPU_STATUS_STOPPED | APPLE_ASC_CPU_STATUS_IDLE) + #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 @@ -302,7 +317,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) * request early crashlog startup, need the endpoint before the map is * acknowledged: G17P can request its crash DVA right after. */ - if (apple_rtkit_early_crashlog(rtk) && base == 0 && + if (!rtk->warm_restart && apple_rtkit_early_crashlog(rtk) && base == 0 && test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) { ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); if (ret) { @@ -312,6 +327,15 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + /* macOS's secure side sends each part back unchanged. */ + if (rtk->port_only) { + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, msg); + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) { + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + } + return; + } if (rtk->retained_wake) { apple_rtkit_management_rx_epmap_retained(rtk, msg, base, bitmap); return; @@ -323,11 +347,29 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) else reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; + /* + * macOS's reply to a restarted DCP echoes the map, and adds bit 1 to the + * first part: 0x51d becomes 0x51f. It starts no endpoints. Measured under + * aura, 3 of 3. Why bit 1 isn't known, but ep 1 is the crash log's. + */ + if (rtk->warm_restart) { + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) + reply |= bitmap & rtk->warm_last_mask; + else + reply |= bitmap | BIT_ULL(1); + } + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (rtk->warm_restart) { + rtk->warm_restart = false; + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + return; + } if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) dev_info(rtk->dev, "RTKit: crashlog endpoint not in the endpoint map\n"); @@ -444,7 +486,7 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) apple_rtkit_management_rx_ap_pwr_ack(rtk, msg); break; case APPLE_RTKIT_MGMT_IDLE_NOTIFY: - dev_dbg(rtk->dev, "RTKit: co-processor idle\n"); + dev_dbg(rtk->dev, "RTKit: co-processor idle: 0x%llx\n", msg); break; default: dev_warn( @@ -569,10 +611,14 @@ static void apple_rtkit_free_buffer(struct apple_rtkit *rtk, if (bfr->size == 0) return; - if (rtk->ops->shmem_destroy) + if (bfr == &rtk->crashlog_buffer && rtk->crashlog_placed) { + memunmap(bfr->buffer); + rtk->crashlog_placed = false; + } else if (rtk->ops->shmem_destroy) { rtk->ops->shmem_destroy(rtk->cookie, bfr); - else if (bfr->buffer) + } else if (bfr->buffer) { dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova); + } bfr->buffer = NULL; bfr->iomem = NULL; @@ -804,6 +850,46 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, len, DMA_FROM_DEVICE); } +/* + * The buffer a crashing firmware placed at its own address, mapped if its IOMMU + * translates every page of it to one contiguous range, and acknowledged as a placed + * buffer is. The firmware doesn't run again until a reboot. + */ +static int apple_rtkit_crashlog_map(struct apple_rtkit *rtk, u64 msg) +{ + struct apple_rtkit_shmem *bfr = &rtk->crashlog_buffer; + struct iommu_domain *domain = iommu_get_domain_for_dev(rtk->dev); + dma_addr_t iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + size_t size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; + phys_addr_t phys, page; + size_t off; + void *va; + + if (!domain || !iova || !size) + return -ENODEV; + phys = iommu_iova_to_phys(domain, iova); + for (off = 0; phys && off < size; off += PAGE_SIZE) { + page = iommu_iova_to_phys(domain, iova + off); + if (page != phys + off) { + dev_err(rtk->dev, "RTKit: crash log at %pad isn't contiguous at +%#zx\n", + &iova, off); + return -EINVAL; + } + } + if (!phys) + return -ENOENT; + va = memremap(phys, size, MEMREMAP_WB); + if (!va) + return -ENOMEM; + bfr->buffer = va; + bfr->size = size; + bfr->iova = iova; + bfr->is_mapped = true; + rtk->crashlog_placed = true; + + return apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_CRASHLOG, msg, NULL, false); +} + #define APPLE_RTKIT_CRASHLOG_LATE_POLL_MS 250 #define APPLE_RTKIT_CRASHLOG_LATE_POLLS 20 @@ -887,6 +973,28 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } + /* + * A firmware that never announced this endpoint asks for its buffer only when + * it crashes, so there's no handshake to take the request for. Report the + * crash now, as the log may never come. A client that sets such a buffer up + * itself takes the path below, which also reads a log written later. + */ + if (!rtk->crashlog_buffer.size && !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints) && + !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { + int ret = apple_rtkit_crashlog_map(rtk, msg); + + dev_err(rtk->dev, "RTKit: co-processor has crashed: crash log buffer at %#llx %s (%d)\n", + FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), + ret ? "not mapped" : "mapped, waiting for its log", ret); + if (!rtk->crashed) { + rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + } + return; + } + if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) @@ -918,6 +1026,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) "RTKit: firmware-owned crash log at DVA %pad (0x%zx bytes)\n", &rtk->crashlog_buffer.iova, rtk->crashlog_buffer.size); rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, NULL, 0); return; @@ -938,7 +1047,11 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) "RTKit: Couldn't allocate crashlog shadow buffer\n"); } + /* Its driver heard of the crash when the buffer was asked for. */ + if (rtk->crashed) + return; rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); return; @@ -963,6 +1076,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) crashed: rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, NULL, 0); @@ -1116,6 +1230,22 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); int err; + /* + * A firmware counts its log writes and holds the next counter until the last is + * echoed, and with one outstanding takes no AP power notice: a DCP after a warm + * restart, the MTP and the SMC after about eight sleeps. macOS echoes every one. + */ + if (type == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo) { + /* A polled idle that came first echoed it already. */ + if (cmpxchg(&rtk->oslog_owed, msg, 0) != msg) + return; + dev_dbg(rtk->dev, "RTKit: oslog counter %016llx, echoed\n", msg); + err = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, msg, NULL, false); + if (err) + dev_err(rtk->dev, "RTKit: oslog echo failed: %d\n", err); + return; + } + /* Preserve the tested Neo/M5 and synchronous-log no-ACK policy. */ if (type == APPLE_RTKIT_OSLOG_LOG && (rtk->sync_syslog_ack || of_machine_is_compatible("apple,t8140") || @@ -1236,12 +1366,17 @@ static void apple_rtkit_rx_work(struct work_struct *work) kfree(rtk_work); } -static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +/* An OSLOG counter this firmware waits to see echoed. */ +static bool apple_rtkit_oslog_owed_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return ep == APPLE_RTKIT_EP_OSLOG && + FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg) == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo; +} + +static bool apple_rtkit_notification_msg(u8 ep, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - if (!rtk->sync_syslog_ack) - return false; if (ep == APPLE_RTKIT_EP_SYSLOG) return type == APPLE_RTKIT_SYSLOG_LOG; if (ep == APPLE_RTKIT_EP_IOREPORT) @@ -1250,6 +1385,11 @@ static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) return false; } +static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return rtk->sync_syslog_ack && apple_rtkit_notification_msg(ep, msg); +} + static void apple_rtkit_sync_ack_work(struct work_struct *work) { struct apple_rtkit *rtk = @@ -1320,6 +1460,8 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, if (!work) return; + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) + WRITE_ONCE(rtk->oslog_owed, msg.msg0); work->rtk = rtk; work->ep = ep; work->msg = msg.msg0; @@ -1365,6 +1507,82 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +static int apple_rtkit_poll_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->epmap_completion)) + return rtk->boot_result; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return try_wait_for_completion(&rtk->epmap_completion) ? rtk->boot_result : -ETIME; +} + +static int apple_rtkit_port_power(struct apple_rtkit *rtk, unsigned int state, + unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + reinit_completion(&rtk->ap_pwr_ack_completion); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state)); + if (ret) + return ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->ap_pwr_ack_completion)) + return 0; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return -ETIME; +} + +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + rtk->port_only = true; + + return apple_rtkit_poll_handshake(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_map); + +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + /* The co-processor starts its side of this port itself. */ + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_INIT, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_on); + +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + int ret; + + ret = apple_rtkit_port_map(rtk, timeout_ms); + if (ret) + return ret; + + return apple_rtkit_port_on(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_handshake); + +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_IDLE, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_idle); + void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { apple_mbox_hold_rx(rtk->mbox); @@ -1442,6 +1660,10 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) set_bit(system_endpoints[i], rtk->endpoints); + /* apple_rtkit_wake() must not wait for an EPMAP that already happened. */ + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->syslog_inherited = true; @@ -1584,6 +1806,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, rtk->handshake_timeout_ms = APPLE_RTKIT_HANDSHAKE_TIMEOUT_MS; rtk->sync_syslog_ack = of_property_read_bool(dev->of_node, "apple,sync-syslog-ack"); + /* As macOS does; a driver whose firmware shouldn't get them turns it off. */ + rtk->oslog_echo = of_machine_is_compatible("apple,t8140"); INIT_KFIFO(rtk->sync_ack_fifo); INIT_WORK(&rtk->sync_ack_work, apple_rtkit_sync_ack_work); @@ -1613,6 +1837,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + rtk->warm_last_mask = U32_MAX; + rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { @@ -1667,15 +1893,18 @@ static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c); /* - * A retained wake is not interrupted by a signal to its caller: it has + * A handshake given up halfway can leave a firmware started that nothing stops. The + * freezer signals every user task as a sleep begins, so only a fatal signal ends the + * wait, and during a system sleep not even that. Nor does a retained wake's: it has * already started the co-processor's CPU, and nothing would undo that. */ static long apple_rtkit_wait_timeout(struct apple_rtkit *rtk, struct completion *c, unsigned long timeout) { - if (READ_ONCE(rtk->retained_wake)) + if (READ_ONCE(rtk->retained_wake) || pm_suspend_target_state == PM_SUSPEND_MEM) return wait_for_completion_timeout(c, timeout); - return wait_for_completion_interruptible_timeout(c, timeout); + + return wait_for_completion_killable_timeout(c, timeout); } /* @@ -1689,7 +1918,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) long t; int ret; - if (!rtk->sync_syslog_ack) + if (!rtk->sync_syslog_ack && !rtk->poll_waits) return apple_rtkit_wait_for_completion(rtk, c); timeout = jiffies + msecs_to_jiffies(rtk->handshake_timeout_ms); @@ -1722,10 +1951,9 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted_with_buffers); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c) { - unsigned long timeout = msecs_to_jiffies(rtk->handshake_timeout_ms); long t; - t = apple_rtkit_wait_timeout(rtk, c, timeout); + t = apple_rtkit_wait_timeout(rtk, c, msecs_to_jiffies(rtk->handshake_timeout_ms)); if (t < 0) return t; else if (t == 0) @@ -1741,6 +1969,63 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, } EXPORT_SYMBOL_GPL(apple_rtkit_set_handshake_timeout); +/* + * Expect HELLO from a firmware being restarted warm: it keeps the buffers it was + * given and its rings, so unlike apple_rtkit_reinit() nothing is freed, and + * endpoints aren't started again. The caller then sets the run bit and calls + * apple_rtkit_boot(). + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk) +{ + unsigned long flags; + int ret; + + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + + reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + + rtk->crashed = false; + rtk->warm_restart = true; + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; + kfifo_reset(&rtk->sync_ack_fifo); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + WRITE_ONCE(rtk->shutting_down, false); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + ret = apple_mbox_start(rtk->mbox); + if (ret) { + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + } else { + apple_mbox_set_recv_enabled(rtk->mbox, true); + } + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_warm_restart); + +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask) +{ + rtk->warm_last_mask = last_mask; +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_warm_epmap_mask); + +int apple_rtkit_port_restart(struct apple_rtkit *rtk) +{ + int ret = apple_rtkit_warm_restart(rtk); + + rtk->warm_restart = false; + rtk->port_only = true; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_restart); + int apple_rtkit_reinit(struct apple_rtkit *rtk) { unsigned long flags; @@ -1846,7 +2131,8 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power, + void (*before_ap)(void *data), void *data) { int ret; @@ -1894,16 +2180,27 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) if (rtk->boot_result) return rtk->boot_result; - return early_ap_power ? 0 : - apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (early_ap_power) + return 0; + + if (before_ap) + before_ap(data); + + return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); } int apple_rtkit_boot(struct apple_rtkit *rtk) { - return apple_rtkit_boot_common(rtk, false); + return apple_rtkit_boot_common(rtk, false, NULL, NULL); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data) +{ + return apple_rtkit_boot_common(rtk, false, before_ap, data); +} +EXPORT_SYMBOL_GPL(apple_rtkit_boot_with); + int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { if (rtk->crashed) @@ -1948,26 +2245,66 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poweroff); -int apple_rtkit_idle(struct apple_rtkit *rtk) +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data) { int ret; /* if OFF is used here the co-processor will not wake up again */ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: AP idle not acknowledged: %d\n", ret); return ret; + } + + if (after_ap) + after_ap(data); ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: IOP idle not acknowledged: %d\n", ret); return ret; + } rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE; return 0; } +EXPORT_SYMBOL_GPL(apple_rtkit_idle_with); + +int apple_rtkit_idle(struct apple_rtkit *rtk) +{ + return apple_rtkit_idle_with(rtk, NULL, NULL); +} EXPORT_SYMBOL_GPL(apple_rtkit_idle); +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_off); + +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on) +{ + return apple_rtkit_set_ap_power_state(rtk, on ? APPLE_RTKIT_PWR_STATE_ON : + APPLE_RTKIT_PWR_STATE_IDLE); +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power); + +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on) +{ + int ret; + + rtk->poll_waits = true; + ret = apple_rtkit_ap_power(rtk, on); + rtk->poll_waits = false; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power_noirq); + int apple_rtkit_quiesce(struct apple_rtkit *rtk) { int ret; @@ -2012,7 +2349,7 @@ static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) if (ret) return ret; - return apple_rtkit_boot_common(rtk, early_ap_power); + return apple_rtkit_boot_common(rtk, early_ap_power, NULL, NULL); } int apple_rtkit_wake(struct apple_rtkit *rtk) @@ -2027,6 +2364,225 @@ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); +static int apple_rtkit_sleep_idle(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + u32 status; + int ret; + + ret = apple_rtkit_idle(rtk); + if (ret) + return ret; + + ret = readl_poll_timeout(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == APPLE_ASC_CPU_STATUS_DOWN, + 100, 5000); + /* Both idles took, so it doesn't run on: stop it as if the status had shown it. */ + if (ret) + dev_warn(rtk->dev, "RTKit: idle, but not stopped: status %#x\n", status); + + return 0; +} + +/* With interrupts off: room in the FIFO, then the message. */ +static bool apple_rtkit_polled_send(struct apple_rtkit *rtk, struct apple_mbox_msg msg) +{ + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return false; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + + return !apple_mbox_send(rtk->mbox, msg, true); +} + +/* + * Only what an idle needs: the state acks, notifications answered as they come, and OSLOG + * counters echoed, without which the firmware takes no AP power notice. + */ +static void apple_rtkit_polled_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct apple_rtkit *rtk = cookie; + u8 ep = msg.msg1; + + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) { + if (!apple_rtkit_polled_send(rtk, msg)) + dev_err(rtk->dev, "RTKit: oslog echo failed while idling\n"); + return; + } + if (ep == APPLE_RTKIT_EP_MGMT) { + u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg.msg0); + unsigned int state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg.msg0); + + if (type == APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK) + rtk->iop_power_state = state; + else if (type == APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK) + rtk->ap_power_state = state; + } else if (apple_rtkit_notification_msg(ep, msg.msg0) && apple_mbox_send_ready(mbox)) { + apple_mbox_send(mbox, msg, true); + } +} + +static int apple_rtkit_polled_state(struct apple_rtkit *rtk, u8 type, unsigned int *state, + unsigned int *us) +{ + struct apple_mbox_msg msg = { + .msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type) | + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_IDLE), + .msg1 = APPLE_RTKIT_EP_MGMT, + }; + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return -ETIMEDOUT; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + apple_mbox_send(rtk->mbox, msg, true); + + for (i = 0; i < APPLE_RTKIT_POLLED_LOOPS; i++) { + apple_mbox_poll(rtk->mbox); + if (READ_ONCE(*state) == APPLE_RTKIT_PWR_STATE_IDLE) + return 0; + udelay(APPLE_RTKIT_POLLED_STEP_US); + *us += APPLE_RTKIT_POLLED_STEP_US; + } + + return -ETIMEDOUT; +} + +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us) +{ + unsigned long flags; + u64 owed; + u32 status; + int ret; + + *us = 0; + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_polled_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + /* What's already waiting is answered first, or the firmware may not go idle. */ + apple_mbox_poll(rtk->mbox); + /* Then a counter left for the receive work, which can't run now. */ + owed = xchg(&rtk->oslog_owed, 0); + if (owed) { + struct apple_mbox_msg echo = { .msg0 = owed, .msg1 = APPLE_RTKIT_EP_OSLOG }; + + if (!apple_rtkit_polled_send(rtk, echo)) + dev_err(rtk->dev, "RTKit: oslog echo failed before the idle\n"); + } + + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + &rtk->ap_power_state, us); + if (!ret) + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + &rtk->iop_power_state, us); + if (!ret) + ret = readl_poll_timeout_atomic(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == + APPLE_ASC_CPU_STATUS_DOWN, + APPLE_RTKIT_POLLED_STEP_US, 5000); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_idle_polled); + +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_recv_off); + +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + ret = apple_rtkit_sleep_idle(rtk, cpu_control); + if (ret) + return ret; + + apple_rtkit_mailbox_off(rtk); + + /* Twice, each time from what it reads back. */ + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_stop); + +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + /* + * No real sleep: a firmware that idled doesn't come back from that with its power kept, + * and stays down until the next sleep restarts it. One whose idle failed still runs. + */ + if (!pm_resume_via_firmware()) { + if (!(readl(cpu_control) & APPLE_ASC_CPU_CONTROL_RUN) || + (rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON) + return -EAGAIN; + /* Its AP idle took and its IOP idle didn't: endpoints stay shut until AP on. */ + if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON) + return apple_rtkit_ap_power(rtk, true); + return 0; + } + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return); + +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret = apple_rtkit_warm_restart(rtk); + + if (ret) + return ret; + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_power_cycled_return); + +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo) +{ + rtk->oslog_echo = echo; + dev_dbg(rtk->dev, "RTKit: OSLOG counters %s\n", echo ? "echoed" : "dropped"); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_oslog_echo); + +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + if (!pm_resume_via_firmware()) + return -EOPNOTSUPP; + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return_start); + /* * Wake a co-processor whose CPU was stopped after apple_rtkit_idle(), with * its firmware, endpoints and shared buffers kept (and its power domain @@ -2059,7 +2615,7 @@ int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), voi ret = start(ctx); if (ret) goto out; - ret = apple_rtkit_boot_common(rtk, false); + ret = apple_rtkit_boot_common(rtk, false, NULL, NULL); out: WRITE_ONCE(rtk->retained_wake, false); if (ret) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index f9e79924e53a6..4fa31a0e54f47 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #define APPLE_SART_MAX_ENTRIES 16 @@ -90,6 +91,15 @@ struct apple_sart { unsigned long protected_entries; unsigned long used_entries; + /* The registers as they stood before a firmware-driven suspend */ + struct { + u8 flags; + phys_addr_t paddr; + size_t size; + } saved[APPLE_SART_MAX_ENTRIES]; + bool saved_valid; + /* saved[] still holds the last sleep's entries, for apple_sart_replay() */ + bool replay_valid; }; static void sart0_get_entry(struct apple_sart *sart, int index, u8 *flags, @@ -398,6 +408,98 @@ static void apple_sart_shutdown(struct platform_device *pdev) } } +/* + * A SART in a domain that a firmware-driven suspend powers off comes back empty, + * and its coprocessor can't reach its firmware or buffers until the entries are + * back. This includes the bootloader's, which Linux otherwise never writes. + */ +static int apple_sart_suspend_late(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + int i; + + sart->saved_valid = false; + sart->replay_valid = false; + if (pm_suspend_target_state != PM_SUSPEND_MEM || !pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) + sart->ops->get_entry(sart, i, &sart->saved[i].flags, + &sart->saved[i].paddr, &sart->saved[i].size); + sart->saved_valid = true; + + return 0; +} + +static int apple_sart_resume_noirq(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + unsigned int restored = 0; + int i; + + if (!sart->saved_valid || !pm_resume_via_firmware()) + return 0; + sart->saved_valid = false; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + /* An unused entry stays as the power left it, whatever that reads. */ + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + /* Every one in use, the same or not: macOS rewrites 0 to 7 at every wake. */ + sart->ops->set_entry(sart, i, sart->saved[i].flags, + sart->saved[i].paddr >> sart->ops->paddr_shift, + sart->saved[i].size >> sart->ops->size_shift); + restored++; + } + dev_dbg(dev, "%u entries restored after sleep\n", restored); + sart->replay_valid = true; + + return 0; +} + +/* + * A coprocessor's driver calls this right before its run bit. macOS writes the entries back + * there, config first, then address, then size, so this does the same on SARTv3. Unused + * entries stay as they read. + */ +int apple_sart_replay(struct apple_sart *sart) +{ + int i, bad = 0; + + if (sart->ops != &sart_ops_v3 || !sart->replay_valid) + return -EOPNOTSUPP; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + phys_addr_t paddr; + size_t size; + u8 flags; + + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + writel(sart->saved[i].flags, sart->regs + APPLE_SART3_CONFIG(i)); + writel(sart->saved[i].paddr >> APPLE_SART3_PADDR_SHIFT, + sart->regs + sart->ops->paddr_reg + 4 * i); + writel(sart->saved[i].size >> APPLE_SART3_SIZE_SHIFT, + sart->regs + sart->ops->size_reg + 4 * i); + sart->ops->get_entry(sart, i, &flags, &paddr, &size); + if (flags == sart->saved[i].flags && paddr == sart->saved[i].paddr && + size == sart->saved[i].size) + continue; + bad++; + dev_warn(sart->dev, "entry %d reads %#x %pa+%#zx, wrote %#x %pa+%#zx\n", + i, flags, &paddr, size, sart->saved[i].flags, &sart->saved[i].paddr, + sart->saved[i].size); + } + + return bad; +} +EXPORT_SYMBOL_GPL(apple_sart_replay); + +static const struct dev_pm_ops apple_sart_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_sart_suspend_late, NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, apple_sart_resume_noirq) +}; + static const struct of_device_id apple_sart_of_match[] = { { .compatible = "apple,t6050-sart", @@ -423,6 +525,7 @@ static struct platform_driver apple_sart_driver = { .driver = { .name = "apple-sart", .of_match_table = apple_sart_of_match, + .pm = pm_sleep_ptr(&apple_sart_pm_ops), }, .probe = apple_sart_probe, .shutdown = apple_sart_shutdown, diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index f3dfabb46693d..ffe3979c02840 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,21 @@ MODULE_PARM_DESC(sensor_spi_mode, struct sep_sensor_lifetime { /* Serializes sensor operations with this device's resource teardown. */ struct mutex lock; + /* The power last written; a failed write counts as on. */ + bool powered; + /* Set across a system sleep: bus and power requests are refused. */ + bool asleep; }; +/* The power line or rails and the bus, with no settling delays. */ +struct sep_sensor_hw { + int (*set_power)(int on); + int (*send)(const void *tx, size_t len); +}; + +/* Puts the sensor in its low-power state; macOS sends it before a sleep. */ +static const u8 sep_sensor_cmd_sleep[] = { 0x80, 0x31, 0x00, 0x07, 0x00, 0x00, 0x00 }; + static DEFINE_MUTEX(sep_sensor_registry_lock); static struct sep_sensor_lifetime *sep_active_sensor; @@ -58,6 +72,7 @@ static struct sep_sensor_lifetime *sep_lock_sensor(void) } static void sep_sensor_irq_teardown_locked(void); +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len); static struct spi_device *sep_spi; static struct gpio_desc *sep_power; @@ -328,6 +343,52 @@ static void sep_release_power(void) sep_power_source = SEP_POWER_NONE; } +static int sep_sensor_set_power_now(int on) +{ + if (sep_power_source == SEP_POWER_SMC_RAILS) + return sep_smc_set_power(on); + if (!sep_power) + return -ENODEV; + gpiod_set_value_cansleep(sep_power, on); + return 0; +} + +static const struct sep_sensor_hw sep_hw = { + .set_power = sep_sensor_set_power_now, + .send = sep_sensor_xfer_tx_locked, +}; + +static int sep_sensor_write_power(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw, int on) +{ + int rc = hw->set_power(on); + + /* A failed write may have taken effect. */ + sensor->powered = rc || on; + return rc; +} + +/* + * As macOS does for a sleep: the sleep command, then the power off. The next + * bring-up's power cycle powers the sensor again, so a resume does not. + * Returns the power-off's error. + */ +static int sep_sensor_sleep_locked(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw) +{ + WRITE_ONCE(sensor->asleep, true); + if (!sensor->powered) + return 0; + /* Best effort: only the power-off matters. */ + hw->send(sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + return sep_sensor_write_power(sensor, hw, 0); +} + +static int sep_sensor_awake_locked(struct sep_sensor_lifetime *sensor) +{ + return READ_ONCE(sensor->asleep) ? -ESHUTDOWN : 0; +} + #define SEP_CS_TIMING_SOFTWARE 0 #define SEP_CS_TIMING_HOOK_BYPASSED 1 #define SEP_CS_TIMING_HARDWARE 2 @@ -462,6 +523,35 @@ static void sep_sensor_remove(struct spi_device *spi) mutex_unlock(&sep_sensor_registry_lock); } +static int sep_sensor_suspend(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + bool was_on; + int rc; + + mutex_lock(&sensor->lock); + was_on = sensor->powered; + rc = sep_sensor_sleep_locked(sensor, &sep_hw); + mutex_unlock(&sensor->lock); + /* Failing the sleep would keep the machine awake; the next bring-up recovers. */ + if (rc) + dev_warn(dev, "sep sensor: power-off for sleep failed (%d)\n", rc); + else if (was_on) + dev_dbg(dev, "sep sensor: off for sleep\n"); + return 0; +} + +/* No lock: a capture still waiting on the data-ready line may hold it. */ +static int sep_sensor_resume(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + + WRITE_ONCE(sensor->asleep, false); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(sep_sensor_pm_ops, sep_sensor_suspend, sep_sensor_resume); + /* * Its own compatible, deliberately not spidev's: capture must stay in the * kernel, and a spidev node would leak an image to userspace. No @@ -485,6 +575,7 @@ static struct spi_driver sep_sensor_driver = { .driver = { .name = "apple-mesa", .of_match_table = sep_sensor_of_match, + .pm = pm_sleep_ptr(&sep_sensor_pm_ops), }, .id_table = sep_sensor_spi_ids, .probe = sep_sensor_probe, @@ -528,26 +619,16 @@ static int sep_sensor_cs_timing_mode_locked(void) } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -static int sep_sensor_power_cycle_locked(void) +static int sep_sensor_power_cycle_locked(struct sep_sensor_lifetime *sensor) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(0); - - if (rc) - return rc; - msleep(SEP_SENSOR_OFF_DELAY_MS); - rc = sep_smc_set_power(1); - if (rc) - return rc; - msleep(SEP_SENSOR_ON_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; + int rc = sep_sensor_write_power(sensor, &sep_hw, 0); - gpiod_set_value_cansleep(sep_power, 0); + if (rc) + return rc; msleep(SEP_SENSOR_OFF_DELAY_MS); - gpiod_set_value_cansleep(sep_power, 1); + rc = sep_sensor_write_power(sensor, &sep_hw, 1); + if (rc) + return rc; msleep(SEP_SENSOR_ON_DELAY_MS); return 0; } @@ -588,24 +669,13 @@ static const char *sep_sensor_firmware_name_locked(unsigned int index) } /* Powers the sensor on/off, holding the hardware settling delay. */ -static int sep_sensor_power_locked(int on) +static int sep_sensor_power_locked(struct sep_sensor_lifetime *sensor, int on) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(on ? 1 : 0); + int rc = sep_sensor_write_power(sensor, &sep_hw, on ? 1 : 0); - if (rc) - return rc; - msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; - - gpiod_set_value_cansleep(sep_power, on ? 1 : 0); - if (on) - msleep(SEP_SENSOR_ON_DELAY_MS); - else - msleep(SEP_SENSOR_OFF_DELAY_MS); + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); return 0; } @@ -846,7 +916,7 @@ int sep_sensor_power_cycle(void) if (!sensor) return -ENODEV; - result = sep_sensor_power_cycle_locked(); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_cycle_locked(sensor); mutex_unlock(&sensor->lock); return result; } @@ -894,7 +964,7 @@ int sep_sensor_power(int on) if (!sensor) return -ENODEV; - result = sep_sensor_power_locked(on); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_locked(sensor, on); mutex_unlock(&sensor->lock); return result; } @@ -962,7 +1032,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_locked(tx, rx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_locked(tx, rx, len); mutex_unlock(&sensor->lock); return result; } @@ -974,7 +1044,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_tx_locked(tx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_tx_locked(tx, len); mutex_unlock(&sensor->lock); return result; } @@ -986,7 +1056,13 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + result = sep_sensor_awake_locked(sensor) ?: + sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); mutex_unlock(&sensor->lock); return result; } + +/* Like the module's Rust tests, which need KUnit built in. */ +#if IS_ENABLED(CONFIG_APPLE_SEP_KUNIT_TEST) && IS_BUILTIN(CONFIG_KUNIT) +#include "sensor_shim_test.c" +#endif diff --git a/drivers/soc/apple/sensor_shim_test.c b/drivers/soc/apple/sensor_shim_test.c new file mode 100644 index 0000000000000..a0374e65bbf5f --- /dev/null +++ b/drivers/soc/apple/sensor_shim_test.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* KUnit tests for the fingerprint sensor's system sleep. No hardware is touched. */ + +#include +#include + +struct sep_fake_hw { + /* 's' for a send, '0' and '1' for power writes, in order. */ + char calls[8]; + unsigned int ncalls; + u8 sent[16]; + size_t sent_len; + int send_rc; + int power_rc; +}; + +static struct sep_fake_hw *sep_fake(void) +{ + return kunit_get_current_test()->priv; +} + +static void sep_fake_note(struct sep_fake_hw *f, char call) +{ + if (f->ncalls < ARRAY_SIZE(f->calls)) + f->calls[f->ncalls++] = call; +} + +static int sep_fake_send(const void *tx, size_t len) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, 's'); + f->sent_len = min(len, sizeof(f->sent)); + memcpy(f->sent, tx, f->sent_len); + return f->send_rc; +} + +static int sep_fake_set_power(int on) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, on ? '1' : '0'); + return f->power_rc; +} + +static const struct sep_sensor_hw sep_fake_hw = { + .set_power = sep_fake_set_power, + .send = sep_fake_send, +}; + +static int sep_sensor_test_init(struct kunit *test) +{ + test->priv = kunit_kzalloc(test, sizeof(struct sep_fake_hw), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, test->priv); + return 0; +} + +static void sep_sensor_sleep_sends_the_command_then_cuts_power(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_EQ(test, f->sent_len, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_MEMEQ(test, f->sent, sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_leaves_an_unpowered_sensor_alone(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = false }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_EXPECT_EQ(test, f->ncalls, 0U); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_cuts_power_when_the_command_fails(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->send_rc = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_FALSE(test, sensor.powered); +} + +static void sep_sensor_sleep_counts_a_failed_power_off_as_on(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_requests_wait_for_the_resume(struct kunit *test) +{ + struct sep_sensor_lifetime sensor = { .powered = true }; + struct device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + sep_sensor_sleep_locked(&sensor, &sep_fake_hw); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), -ESHUTDOWN); + + dev_set_drvdata(dev, &sensor); + KUNIT_EXPECT_EQ(test, sep_sensor_resume(dev), 0); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + /* The resume powers nothing: the next bring-up does. */ + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, ((struct sep_fake_hw *)test->priv)->ncalls, 2U); +} + +static void sep_sensor_power_writes_are_tracked(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = {}; + + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), 0); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), 0); + KUNIT_EXPECT_FALSE(test, sensor.powered); + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + sensor.powered = false; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); +} + +/* Framed like the driver's other commands: 0x80, opcode, then the length at 3. */ +static void sep_sensor_sleep_command_is_framed(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, sizeof(sep_sensor_cmd_sleep), (size_t)7); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[0], (u8)0x80); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[1], (u8)0x31); + KUNIT_EXPECT_EQ(test, (size_t)get_unaligned_le16(&sep_sensor_cmd_sleep[3]), + sizeof(sep_sensor_cmd_sleep)); +} + +static struct kunit_case sep_sensor_sleep_cases[] = { + KUNIT_CASE(sep_sensor_sleep_sends_the_command_then_cuts_power), + KUNIT_CASE(sep_sensor_sleep_leaves_an_unpowered_sensor_alone), + KUNIT_CASE(sep_sensor_sleep_cuts_power_when_the_command_fails), + KUNIT_CASE(sep_sensor_sleep_counts_a_failed_power_off_as_on), + KUNIT_CASE(sep_sensor_requests_wait_for_the_resume), + KUNIT_CASE(sep_sensor_power_writes_are_tracked), + KUNIT_CASE(sep_sensor_sleep_command_is_framed), + {} +}; + +static struct kunit_suite sep_sensor_sleep_suite = { + .name = "apple_sep_sensor_sleep", + .init = sep_sensor_test_init, + .test_cases = sep_sensor_sleep_cases, +}; + +kunit_test_suite(sep_sensor_sleep_suite); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 72e532ce11f0b..9987fcd18dbab 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -65,8 +65,10 @@ use kernel::{ sync::{ aref::ARef, atomic::{ + Acquire, Atomic, - Relaxed, // + Relaxed, + Release, // }, barrier::{ dma_mb, @@ -659,6 +661,12 @@ struct SepData { // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, + // A `pm::SleepGate`. + sleep_gate: Atomic, + + // The mailbox's receive interrupt is held off for the enclave's sleep. + sleep_rx_held: Atomic, + #[pin] ool_xarm: Mutex>, @@ -1152,6 +1160,8 @@ impl SepData { sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), + sleep_gate: Atomic::new(pm::SleepGate::Open as u32), + sleep_rx_held: Atomic::new(false), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), @@ -1411,7 +1421,200 @@ impl SepData { if self.shutting_down.load(Relaxed) { return Err(ENODEV); } - Self::ring(self.mbox.lock().as_ref().ok_or(ENODEV)?, msg) + // Checked with the mailbox lock held: a sender can wait for it for longer + // than the system takes to tell the enclave it's going to sleep. + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_request() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + /// As [`Self::send`], for an answer to something the enclave asked: it + /// waits for these before it can answer the sleep request. + fn send_reply(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_reply() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + pub(crate) fn sleep_gate(&self) -> pm::SleepGate { + pm::SleepGate::from_u32(self.sleep_gate.load(Acquire)) + } + + /// Under the mailbox lock, so that a sender holding it finishes first. + fn set_sleep_gate(&self, gate: pm::SleepGate) { + let _mbox = self.mbox.lock(); + self.sleep_gate.store(gate as u32, Release); + } + + /// Prepare a firmware-driven suspend without sending SLEEP. The platform + /// owns the irreversible request, after every returnable refusal. + pub(crate) fn sleep_enter(&self) -> Result<()> { + match self.sleep_gate() { + // A failed earlier wake is not repaired by another precommit attempt. + pm::SleepGate::Lost => return Err(EIO), + pm::SleepGate::Open => {} + _ => return Err(EBUSY), + } + // Do not commit a running enclave whose sleep protocol is not ready. + if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { + dev_warn_ratelimited!( + self.dev, + "sleep refused: the enclave did not finish coming up\n" + ); + return Err(EBUSY); + } + + self.set_sleep_gate(pm::SleepGate::Draining); + let idle = control::wait_idle( + proto::CONTROL_TIMEOUT_MS, + 10, + || self.control.lock().requests_in_flight(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_err_ratelimited!( + self.dev, + "sleep refused: a request is still waiting for its reply\n" + ); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(EBUSY); + } + + if let Err(e) = self.sleep_leave_for_commit() { + dev_err_ratelimited!( + self.dev, + "sleep refused: irreversible request could not be staged ({:?})\n", + e + ); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(e); + } + Ok(()) + } + + /// Stage without sending; the platform's commit owns SLEEP with IRQs off. + fn sleep_leave_for_commit(&self) -> Result<()> { + let request = proto::encode_control(&proto::op_sleep(), proto::TAG_SLEEP); + match self.mbox.lock().as_ref() { + Some(mbox) => pm::leave_for_commit( + &self.dev, + mbox.as_raw(), + request, + proto::SLEEP_REPLY, + proto::SLEEP_REPLY_MASK, + ), + None => Err(ENODEV), + }?; + self.sleep_hold_rx(); + dev_dbg!(self.dev, "SEP SLEEP staged for the irreversible commit\n"); + Ok(()) + } + + /// Holds off the receive interrupt, whose handler reads the mailbox. + fn sleep_hold_rx(&self) { + if !self.sleep_rx_held.xchg(true, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.hold_rx(); + } + } + } + + fn sleep_release_rx(&self) { + if self.sleep_rx_held.xchg(false, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.release_rx(); + } + } + } + + /// A refused precommit attempt was never sent; reopen normal admission. + /// A sent request can be restarted only after actual physical sleep/wake. + pub(crate) fn sleep_exit(&self) { + // Staged only: an unsent refusal releases RX without a restart. + match pm::commit_finish(&self.dev) { + Some(true) => self.set_sleep_gate(pm::SleepGate::Told), + Some(false) => self.sleep_release_rx(), + None => {} + } + let gate = self.sleep_gate(); + if gate == pm::SleepGate::Draining { + self.set_sleep_gate(pm::SleepGate::Open); + return; + } + if !gate.needs_wake() { + return; + } + // SAFETY: only reads a flag. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if !real && gate == pm::SleepGate::Lost { + return; + } + if !real { + // commit_finish already terminates a current sent-without-sleep case. + // An old lost state must not cause mailbox or DART accesses here. + self.set_sleep_gate(pm::SleepGate::Lost); + return; + } + let answered = self.sleep_kick(); + match answered { + Ok(()) => { + self.set_sleep_gate(pm::SleepGate::Open); + // One line per sleep it was told of: nothing else says it came back. + dev_info_ratelimited!(self.dev, "the enclave is back from its sleep request\n"); + } + Err(e) => { + dev_err_ratelimited!( + self.dev, + "the enclave did not come back ({:?}); it stays unused until a reboot\n", + e + ); + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + } + } + } + + /// Starts a gated enclave and waits for the messages that say it's up. + fn sleep_kick(&self) -> Result<()> { + pm::clear_dev_disable(&self.dev)?; + pm::dart_restore(&self.dev); + // Arm completion before the held receive IRQ can deliver an early wake. + self.control.lock().wake_reset(); + self.sleep_release_rx(); + match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, proto::WAKE_KICK)?; + } + None => return Err(ENODEV), + } + let mut remaining = time::msecs_to_jiffies(proto::CONTROL_TIMEOUT_MS); + let mut guard = self.control.lock(); + loop { + if guard.wake_complete() { + return Ok(()); + } + if remaining == 0 { + return Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } } /// Puts `msg` in the mailbox once the enclave can see what the CPU wrote @@ -1437,6 +1640,16 @@ impl SepData { return Err(e); } + self.control_wait(op, idx, tag) + } + + /// Waits for the answer to the request sent under `idx` and `tag`. + fn control_wait( + &self, + op: &proto::ControlOp, + idx: usize, + tag: u8, + ) -> Result> { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { @@ -1580,7 +1793,7 @@ impl SepData { } let mbox = self.mbox.lock(); let mbox = mbox.as_ref().ok_or(ENODEV)?; - if !self.control.lock().entropy_may_send() { + if !self.sleep_gate().may_request() || !self.control.lock().entropy_may_send() { return Err(EAGAIN); } Self::ring(mbox, msg) @@ -2024,7 +2237,7 @@ impl SepData { ); } let msg = crate::xarm::encode_xarm_reply(reply); - let _ = self.send(msg); + let _ = self.send_reply(msg); } fn release_deferred_query(&self) { @@ -2811,6 +3024,8 @@ impl SepData { fn on_control(&self, msg: Message, f: proto::Fields) { if f.ty != proto::CONTROL_REPLY_TYPE { + self.control.lock().wake_note(f.ty); + self.control_wq.notify_all(); return; } @@ -3163,6 +3378,11 @@ impl platform::Driver for SepDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pm_shim.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(pm::sep_pm_ops) }) + } + fn probe<'bound>( pdev: &'bound platform::Device>, _info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index d2bf1d4f2ba3e..aa5112b1d5577 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -19,15 +19,23 @@ #include struct apple_sep_fv_new_file_key; +struct dev_pm_ops; void sep_cancel_work_sync(void *work); void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void (*event)(bool entering)); +int sep_pm_register(void *dev, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); +int sep_pm_wait_gated(void *dev, unsigned int timeout_ms); +int sep_pm_clear_dev_disable(void *dev); +void sep_pm_dart_restore(void *dev); +int sep_pm_commit_prepare(void *dev, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask); +int sep_pm_commit_finish(void *dev); +extern const struct dev_pm_ops sep_pm_ops; /* -- hwrng_shim.c ------------------------------------------------------- */ diff --git a/drivers/soc/apple/sleep-test.c b/drivers/soc/apple/sleep-test.c new file mode 100644 index 0000000000000..90b7661afd198 --- /dev/null +++ b/drivers/soc/apple/sleep-test.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by sleep.c: each matched SoC's data. A word a list names but the SoC's full list + * lacks is skipped silently at the wake, and a duplicate splits its books. + */ +#include + +/* Matched only once its sleep has been tested on hardware. */ +static const char *const apple_sleep_test_tested[] = { + "apple,t8140-sleep", +}; + +static bool apple_sleep_test_has(const struct apple_sleep_words *all, u32 offset) +{ + return sleep_ps_find(all->offset, all->count, offset) >= 0; +} + +static void apple_sleep_test_in(struct kunit *test, const char *compat, const char *list, + const struct apple_sleep_words *all, u32 offset) +{ + KUNIT_EXPECT_TRUE_MSG(test, apple_sleep_test_has(all, offset), + "%s: %s word %#x isn't in its full list", compat, list, offset); +} + +static void apple_sleep_test_list_in(struct kunit *test, const char *compat, const char *list, + const struct apple_sleep_words *all, + const struct apple_sleep_words *words) +{ + int i; + + for (i = 0; i < words->count; i++) + apple_sleep_test_in(test, compat, list, all, words->offset[i]); +} + +static void apple_sleep_test_full(struct kunit *test, const char *compat, const char *list, + const struct apple_sleep_words *all) +{ + int i; + + KUNIT_EXPECT_GT_MSG(test, all->count, 0, "%s: %s is empty", compat, list); + for (i = 0; i < all->count; i++) { + KUNIT_EXPECT_EQ_MSG(test, all->offset[i] % 4, 0, "%s: %s word %#x", compat, list, + all->offset[i]); + KUNIT_EXPECT_EQ_MSG(test, sleep_ps_find(all->offset, all->count, all->offset[i]), i, + "%s: %s has %#x twice", compat, list, all->offset[i]); + } +} + +static void apple_sleep_test_only_tested(struct kunit *test) +{ + const struct of_device_id *id; + int n = 0; + + for (id = apple_sleep_of_match; id->compatible[0]; id++, n++) { + KUNIT_EXPECT_NOT_NULL_MSG(test, id->data, "%s has no data", id->compatible); + KUNIT_ASSERT_LT(test, n, ARRAY_SIZE(apple_sleep_test_tested)); + KUNIT_EXPECT_STREQ(test, id->compatible, apple_sleep_test_tested[n]); + } + KUNIT_EXPECT_EQ(test, n, ARRAY_SIZE(apple_sleep_test_tested)); +} + +static void apple_sleep_test_words(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = apple_sleep_of_match; id->compatible[0]; id++) { + const struct apple_sleep_soc *soc = id->data; + const struct apple_sleep_domains *d = &soc->ps; + const struct apple_sleep_words *all = &soc->ps_all, *aon = &soc->ps_aon_all; + const char *c = id->compatible; + u32 named[] = { + d->disp[0][0], d->disp[0][1], d->disp[0][2], d->disp[1][0], d->disp[1][1], + d->disp[1][2], d->sio_cpu, d->sep, d->usb_aon, d->usb_common, d->usb, + d->ap_tmm, d->gpio, d->ans, d->apcie_gp, d->apcie_sys_gp, d->apcie_st, + d->apcie_sys_st, d->apcie_phy_sw, + }; + int i; + + apple_sleep_test_full(test, c, "ps_all", all); + apple_sleep_test_full(test, c, "ps_aon_all", aon); + for (i = 0; i < ARRAY_SIZE(named); i++) + apple_sleep_test_in(test, c, "named", all, named[i]); + apple_sleep_test_in(test, c, "debug switch", aon, soc->ps_aon_debug_switch); + apple_sleep_test_list_in(test, c, "ps_macos_on", all, &soc->ps_macos_on); + apple_sleep_test_list_in(test, c, "ps_macos_off", all, &soc->ps_macos_off); + apple_sleep_test_list_in(test, c, "ps_wake_order", all, &soc->ps_wake_order); + apple_sleep_test_list_in(test, c, "ps_still_on", all, &soc->ps_still_on); + apple_sleep_test_list_in(test, c, "ps_aon_macos_off", aon, &soc->ps_aon_macos_off); + apple_sleep_test_list_in(test, c, "ps_aon_macos_no_auto", aon, + &soc->ps_aon_macos_no_auto); + apple_sleep_test_list_in(test, c, "ps_aon_wake_order", aon, + &soc->ps_aon_wake_order); + for (i = 0; i < soc->nstray; i++) + apple_sleep_test_in(test, c, soc->stray[i].name, all, soc->stray[i].offset); + for (i = 0; i < soc->ngate_off; i++) + apple_sleep_test_in(test, c, soc->gate_off[i].name, all, + soc->gate_off[i].offset); + for (i = 0; i < soc->ngate_coprocs; i++) { + KUNIT_EXPECT_TRUE(test, soc->gate_coprocs[i].path[0] == '/'); + if (soc->gate_coprocs[i].ps) + apple_sleep_test_in(test, c, soc->gate_coprocs[i].path, all, + soc->gate_coprocs[i].ps); + } + } +} + +/* The books keep one bit for each stray word and one gate for each coprocessor. */ +static void apple_sleep_test_books(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = apple_sleep_of_match; id->compatible[0]; id++) { + const struct apple_sleep_soc *soc = id->data; + + KUNIT_EXPECT_LE(test, soc->nstray, BITS_PER_LONG); + KUNIT_EXPECT_LE(test, soc->ngate_coprocs, + ARRAY_SIZE(((struct apple_sleep *)0)->gate)); + } +} + +static void apple_sleep_test_wake_names(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = apple_sleep_of_match; id->compatible[0]; id++) { + const struct apple_sleep_soc *soc = id->data; + int i, j; + + KUNIT_EXPECT_NE(test, soc->wake_last_word, 0); + for (i = 0; i < soc->nwake_names; i++) { + u32 n = soc->wake_names[i].id; + + KUNIT_EXPECT_LT_MSG(test, n, WAKE_WORDS * 32, "%s", + soc->wake_names[i].name); + if (n / 32 == WAKE_WORDS - 1) + KUNIT_EXPECT_TRUE_MSG(test, soc->wake_last_word & BIT(n % 32), "%s", + soc->wake_names[i].name); + for (j = 0; j < i; j++) + KUNIT_EXPECT_NE(test, soc->wake_names[j].id, n); + } + } +} + +static struct kunit_case apple_sleep_test_cases[] = { + KUNIT_CASE(apple_sleep_test_only_tested), + KUNIT_CASE(apple_sleep_test_words), + KUNIT_CASE(apple_sleep_test_books), + KUNIT_CASE(apple_sleep_test_wake_names), + {} +}; + +static struct kunit_suite apple_sleep_test_suite = { + .name = "apple-sleep-soc-data", + .test_cases = apple_sleep_test_cases, +}; +kunit_test_suite(apple_sleep_test_suite); diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c new file mode 100644 index 0000000000000..c78be9b28b3da --- /dev/null +++ b/drivers/soc/apple/sleep.c @@ -0,0 +1,3688 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC system sleep + * + * On some Apple SoCs a kernel can run directly at EL2 behind the CTXR execute + * window (Documentation/arch/arm64/ctxr-execmem.rst), with no monitor below it. + * So no PSCI answers a request to power a CPU off or to suspend the system, and + * the kernel does both itself. + * + * iBoot locks every core's reset vector to the boot object. A core that comes + * back from a power-off therefore restarts inside the boot object (aura), and + * that has to send it back here. The boot object opts in by publishing a mailbox + * in /chosen/aurora,s2r-mailbox, which also gives its spin-table CPUs the parking + * spin table (arch/arm64/kernel/cpu_ops.c). Without it this driver does nothing: + * + * - A secondary core goes back through its spin-table release address, which + * the boot object keeps, into secondary_holding_pen. + * - The boot core takes cpu_resume from the mailbox: EL2, MMU off. + * + * The register addresses are in the device tree. The ADT has every one of them, + * as /arm-io/pmgr's or /arm-io/wdt's reg entries. Register offsets and order not + * otherwise explained here were measured from a macOS sleep under aura. + * + * What the kernel does in suspend-to-RAM is what macOS does in its last 14 ms, + * with the order kept. What differs between SoCs, their power words and the + * lists macOS was measured with, is each SoC's match data. Only SoCs measured + * and tested that way are matched: on the others this driver doesn't bind, and + * s2idle stays the only sleep. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Must match aura's struct aura_s2r_mailbox. */ +#define AURA_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ + +struct aura_s2r_mailbox { + u64 magic; /* aura, before the kernel starts */ + u64 boot_mpidr; /* aura: the only core that may take resume_pa */ + u64 resume_pa; /* kernel, armed before a commit; aura clears it */ + u64 resumes; /* aura: resume_pa taken this many times */ +}; + +/* PS word */ +#define PS_AUTO_ENABLE BIT(28) +#define PS_UNLATCH BIT(12) +#define PS_DEV_DISABLE BIT(10) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_OFF_TIMEOUT_US 5000 +#define PS_BUSY_SETTLE_US 1000 + +/* CPU start block, as aura's smp_start_cpu() uses it */ +#define CPU_START_STOP 0x0 +#define CPU_START_SYS 0x4 +#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) + +/* Power-gate page: reads back this value already. Purpose unknown. */ +#define PWRGATE_ON 0x80000000 + +/* Three clock muxes macOS writes before the commit. */ +#define CLK_MUX_PARKED 0x80100000 +#define CLK_MUX_34 0x0 +#define CLK_MUX_38 0x4 +#define CLK_MUX_3C 0x8 + +/* Cluster control: only these three words, the next one is locked. */ +#define CLUSTER_E 0x0 +#define CLUSTER_P 0x4 +#define CLUSTER_AVE 0x8 +#define CLUSTER_ON 0xf +/* Set when a complex's up sequence ends, and cleared by writing 1. */ +#define CLUSTER_UP_DONE BIT(5) +#define CLUSTER_E_READY BIT(5) +#define CLUSTER_TIMEOUT_US 5000 + +/* CLLT: names from nothing. Bits measured cleared by macOS before the commit. */ +#define CLLT_0 0x30 +#define CLLT_1 0x34 +#define CLLT_2 0x3c +#define CLLT_0_CLEAR GENMASK(7, 0) +#define CLLT_1_CLEAR GENMASK(7, 0) +#define CLLT_2_CLEAR GENMASK(17, 16) + +/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ +#define WDT_WD0_COUNT 0x00 +#define WDT_WD0_CTL 0x0c +#define WDT_WD1_COUNT 0x10 +#define WDT_WD2_COUNT 0x20 +#define WDT_WD2_ALARM 0x28 +#define WDT_WD2_CTL 0x2c +#define WDT_TICKS_PER_S 24000000 +/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ +#define WDT_SLEEP_SECS 5 +/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ +#define WDT_SUSPEND_SECS 120 + +#define AON_AP_DEADLINE 0x08 + +/* Wake sources: a mask, so 0 enables. Status words are sticky. */ +#define WAKE_MASK(word) (0x08 + (word) * 4) +#define WAKE_STATUS(word) (0x28 + (word) * 4) +#define WAKE_WORDS 8 +#define WAKE_SOURCES_MAX 32 + +/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ +#define SPMI_STATUS 0x00 +#define SPMI_STATUS_RX_EMPTY BIT(24) +#define SPMI_CMD 0x04 +#define SPMI_REPLY 0x08 +#define SPMI_NOTIFY_ENABLE 0x40 +#define SPMI_NOTIFY_ENABLE_SLEEP 0x2000001 +#define SPMI_NOTIFY_REPLY BIT(0) +#define SPMI_NOTIFY_ACK 0x80 +#define SPMI_CMD_ACTIVE BIT(15) +#define SPMI_OP_EXT_WRITE(n) ((n) - 1) +#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) +#define SPMI_OP_ZERO_WRITE 0x80 +#define SPMI_OP_SLEEP 0x11 +#define SPMI_OP_WAKEUP 0x13 +#define SPMI_POLL_US 20000 + +/* A port controller on SPMI: registers go through a page select, then a data window. */ +#define HPM_REG_SELECT 0x00 +#define HPM_REG_DATA 0x20 +#define HPM_REG_DATA_CMD 0xa0 +#define HPM_PAGE_CMD1 0x08 +#define HPM_PAGE_DATA1 0x09 +#define HPM_CMD_SSPS 0x53505353 +#define HPM_CMD_COLD_RESET 0x64696147 +#define HPM_SSPS_S3 3 +/* The first look after the reset, and how long a contract that never seemed to go is trusted to have gone. */ +#define HPM_RESET_FIRST_MS 50 +#define HPM_RESET_FLOOR_MS 1000 +#define HPM_PAGE_POWER_STATUS 0x3f +#define HPM_POWER_CONNECTED BIT(0) +#define HPM_POWER_TYPEC_MODE GENMASK(3, 2) +#define HPM_POWER_TYPEC_PD 3 +/* A contract that comes back does by 0.7 s; one that doesn't is back after another reset. */ +#define HPM_CONTRACT_MS 1500 +#define HPM_RESET_TRIES 3 + +/* A PMU register write on the sleep's SPMI bus */ +#define SPMI_CMD_WRITE (BIT(15) | (0xe << 8) | 0x30) +#define PMU_BOOT_STAGE 0xf701 +#define PMU_BOOT_STAGE_SLEEP 0x50 + +/* Three of the memory controller's channels, and what macOS steps down on each. */ +#define MCC_CHANNELS 3 +#define MCC_ENABLES 0x4 +#define MCC_LANES 0x8 +#define MCC_LANES_MASK 0xc +#define MCC_LANE_STROBE(n) BIT(16 + 4 * (n)) +#define MCC_LANE(n) BIT(4 * (n)) + +/* Two words macOS zeroes just before the power gates. */ +#define PTD_A 0x0 +#define PTD_B 0x8 + +/* Perf-state words, each like DCS's. The SoC's is first. */ +#define CLK_PERF_SOC 0x0 +#define SOC_SLEEP_STATE 3 + +/* The PMC's voter agents, each with a word holding its floor for every rail. */ +#define PMC_VOTER_AGENTS 4 +#define PMC_VOTER_STRIDE 0x4000 + +/* DCS: the low nibble asks for a state, and [11:8] is the state it has reached. */ +#define DCS_STATE GENMASK(11, 8) +#define DCS_BUSY BIT(31) +#define DCS_SLEEP_STATE 4 +#define DCS_STEP_STATE 6 +#define DCS_TIMEOUT_US 5000 + +/* + * Word 0 of the wake block. Bit 0 lets the sleep keep DRAM for the USB host's always-on side. + * macOS sets it from the host's start to its stop, not at the commit. + */ +#define WAKE_SLP_DDR 0x00 +#define WAKE_SLP_DDR_USB BIT(0) + +/* + * AMCC error enables. Any error enabled and pending at the commit turns the + * sleep around within 20 ms with no error handler to service it. Linux has none. + * Measured: with macOS's handler hidden, as Linux's position is, 6 of 6 sleeps + * held with both of these off, and bit 25 alone still bounced. Their status + * stays set; only the enables matter. A deep sleep sets them again, so this has + * to be done before every commit. + */ +#define AMCC_WORD0_ENABLE 0x0 +#define AMCC_WORD1_ENABLE 0x8 +#define AMCC_WORD0_DSID_AGE_OUT BIT(25) +#define AMCC_WORD1_UNEXP_RT_HIT BIT(8) + +/* A coprocessor's ASC */ +#define ASC_CPU_CONTROL 0x44 +#define ASC_CPU_CONTROL_RUN BIT(4) +#define ASC_CPU_STATUS 0x48 + +/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ +#define APCIE_AXI_EN0 0x104 +#define APCIE_AXI_EN1 0x108 +#define APCIE_CLK_CTRL 0x0 +#define APCIE_CLK_BUSY BIT(31) +#define APCIE_CLK_RUN BIT(5) +#define APCIE_CLK_OFF BIT(1) +#define APCIE_CLK_OFF2 BIT(2) +#define APCIE_CLK_START 0x4 +#define APCIE_CLK_START_GO BIT(31) +#define APCIE_PHY_COMMON 0x4000 +#define APCIE_PHY_CTRL 0x8000 +#define APCIE_PHY_CTRL_CLK0REQ BIT(0) +#define APCIE_PHY_CTRL_CLK1REQ BIT(1) +#define APCIE_PHY_CTRL_CLK0ACK BIT(2) +#define APCIE_PHY_CTRL_CLK1ACK BIT(3) +#define APCIE_PHY_CTRL_RESET BIT(4) +#define APCIE_PHY_CTRL_UP BIT(27) +#define APCIE_PHY_RUN 0x8004 +#define APCIE_PHY_RUN_EN BIT(0) +#define APCIE_PHY_RUN_EN2 BIT(4) +#define APCIE_PHY_READY 0x8008 +#define APCIE_PHY_IP_PLL_CTL 0x90 +#define APCIE_PHY_IP_PLL_HOLD BIT(16) +#define APCIE_PORT_INTSTAT 0x100 +#define APCIE_PORT_INTMSK 0x104 +#define APCIE_PORT_INT_LINK GENMASK(7, 4) +#define APCIE_PORT_L2 0x88 +#define APCIE_PORT_L2_ENTER 0x11 +#define APCIE_PORT_L2_BUSY BIT(0) +#define APCIE_PORT_PERST_GUARD 0x13c +#define APCIE_PORT_LINKSTS 0x208 +#define APCIE_PORT_LINKSTS_UP BIT(0) +#define APCIE_PORT_APPCLK 0x800 +#define APCIE_PORT_APPCLK_EN BIT(0) +#define APCIE_PORT_APPCLK_CGDIS BIT(8) +#define APCIE_PORT_PERST 0x82c +#define APCIE_PORT_PHY_LINK BIT(31) +#define APCIE_PORT_PHY_EN0 BIT(9) +#define APCIE_PORT_PHY_EN1 BIT(10) +#define APCIE_ENGINE_CTRL 0x0 +#define APCIE_ENGINE_IRQ_MASK 0x8 +#define APCIE_ENGINE_STOP BIT(0) +#define APCIE_FABRIC_STOP 0x4 +#define APCIE_PORT_STOP_US 5000 +#define APCIE_RC_ZERO 0x04 +#define APCIE_RC_READY_START 0x50 +#define APCIE_RC_READY_CFG 0x54 +#define APCIE_RC_READY_STAT 0x58 +#define APCIE_POLL_US 50000 + +/* The coprocessors the last mile stops. Each driver idles its own first. */ +#define SLEEP_COPROCS 4 + +static struct apple_sleep_coproc { + struct device *dev; + void __iomem *asc; + bool pcie_ans; + int (*last_check)(void *data); + int (*last_stop)(void *data); + void *data; + bool ready; + bool stopped; + u32 status; + int stop_ret; + /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ + bool first; + u32 ps; +} apple_sleep_coprocs[SLEEP_COPROCS]; +static DEFINE_MUTEX(apple_sleep_coprocs_lock); + +/* Irreversible enclave stop; nonzero is legal only if SLEEP was not sent. */ +static struct { + struct device *dev; + int (*stop)(void *data); + void *data; +} apple_sleep_enclave; + +/* + * Written with bits 0 and 63 by every core right before its last WFI, measured + * from macOS. This is not CYC_OVRD, which is undefined on this SoC. + */ +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) + +#define SLEEP_DISPLAY_COUNT 2 + +/* CPU clusters: 0 is E, 1 is P */ +#define SLEEP_CLUSTERS 2 + +/* cpm-dvfm */ +#define DVFM_CTL 0x178 +#define DVFM_WORD_0 0x140 +#define DVFM_WORD_1 0x160 +#define DVFM_WORD_2 0x190 +#define DVFM_WORD_3 0x198 +#define DVFM_ENABLE BIT_ULL(63) + +/* In macOS's order, on the way down. */ +static const u32 sleep_dvfm_words[] = { + DVFM_WORD_0, DVFM_WORD_1, DVFM_WORD_2, DVFM_WORD_3, +}; + +/* A power word by its offset, named for the log */ +struct apple_sleep_word { + const char *name; + u32 offset; +}; + +/* Power words by their offsets */ +struct apple_sleep_words { + const u32 *offset; + unsigned int count; +}; + +#define SLEEP_WORDS(a) { .offset = (a), .count = ARRAY_SIZE(a) } + +/* + * A coprocessor that must be bound for a sleep, by its path. @must_stop: it must also have + * stopped by the last step. @ps: its CPU's own domain, where a driver may power it off. + */ +struct apple_sleep_gate_coproc { + const char *path; + bool must_stop; + u32 ps; +}; + +/* A source in the ADT's pmgr/ap-wake-sources: for the log, and to tell the SMC's apart */ +struct apple_sleep_wake_name { + u32 id; + const char *name; +}; + +/* The power words in "ps" that the sleep drives itself */ +struct apple_sleep_domains { + /* CPU, front end and system, of the DCP and then of the external DCP */ + u32 disp[SLEEP_DISPLAY_COUNT][3]; + u32 sio_cpu; + u32 sep; + /* The USB host's complex: always-on, common and the controller */ + u32 usb_aon; + u32 usb_common; + u32 usb; + /* Only the deep path's wake turns it on. */ + u32 ap_tmm; + u32 gpio; + u32 ans; + /* ANS's PCIe block */ + u32 apcie_gp; + u32 apcie_sys_gp; + u32 apcie_st; + u32 apcie_sys_st; + u32 apcie_phy_sw; +}; + +/* + * What differs between SoCs. Power words are offsets in "ps", or in "ps-aon" where named + * so, and the lists keep the order macOS was measured with. + */ +struct apple_sleep_soc { + struct apple_sleep_domains ps; + /* In "ps-aon": held up by a live debug link, and off on macOS */ + u32 ps_aon_debug_switch; + /* In "pwrgate": each pair written as macOS does */ + u32 pwrgate_wake[2]; + u32 pwrgate_cpm[2]; + u32 pwrgate_disp[2]; + /* Every power word, saved at the commit and checked at the wake */ + struct apple_sleep_words ps_all; + struct apple_sleep_words ps_aon_all; + /* On at macOS's commit and left off by Linux */ + struct apple_sleep_words ps_macos_on; + /* Off at macOS's commit, in the order it lowers them */ + struct apple_sleep_words ps_macos_off; + struct apple_sleep_words ps_aon_macos_off; + /* In "ps-aon": on at macOS's commit, with AUTO_ENABLE off */ + struct apple_sleep_words ps_aon_macos_no_auto; + /* Written back on by macOS's wake before any driver runs, in its order */ + struct apple_sleep_words ps_wake_order; + struct apple_sleep_words ps_aon_wake_order; + /* Polled off just before the complex words */ + struct apple_sleep_words ps_still_on; + /* The wake's timed power-gate polls: off with BUSY clear at the commit, on macOS */ + struct apple_sleep_words ps_engine_polled; + /* Lowered at the last step if still on, with no user left */ + const struct apple_sleep_word *stray; + unsigned int nstray; + /* Off on macOS by the last step: one still on only warns. */ + const struct apple_sleep_word *gate_off; + unsigned int ngate_off; + const struct apple_sleep_gate_coproc *gate_coprocs; + unsigned int ngate_coprocs; + const struct apple_sleep_wake_name *wake_names; + unsigned int nwake_names; + /* The sources in the wake block's last word */ + u32 wake_last_word; + /* The PCIe host whose port the radio is on, for its PERST#, or NULL */ + const char *radio_pcie; +}; + +struct apple_sleep { + struct device *dev; + const struct apple_sleep_soc *soc; + void __iomem *ps; + void __iomem *pwrgate; + void __iomem *cpu_start; + void __iomem *cllt; + void __iomem *cluster; + void __iomem *commit; + void __iomem *wdt; + void __iomem *aon_misc; + void __iomem *wake; + void __iomem *amcc[4]; + void __iomem *pmc_dcp; + void __iomem *spmi; + void __iomem *pmc_voters; + void __iomem *dcs; + void __iomem *ps_aon; + void __iomem *spmi_a0; + void __iomem *hpm_spmi; + /* The port controller whose cold reset ends the debug link. */ + struct device_node *hpm_np; + u64 siq_cfg; + u32 hpm_sid; + int usb_down_ret; + /* Cut, or found already down, for this sleep. */ + bool usb_down_ran; + /* The PMGR block "ps" maps, for the power controllers' nodes. */ + struct resource ps_res; + /* Its words a driver took down with apple_sleep_domain_set() and hasn't raised. */ + unsigned long *driver_down; + void __iomem *clk_mux; + void __iomem *mcc[MCC_CHANNELS]; + void __iomem *ptd; + void __iomem *clk_perf; + void __iomem *cluster_dcs; + /* As the match found them: one for each word of the SoC's lists */ + u32 *ps_on_saved; + u32 *ps_off_saved; + u32 *ps_aon_saved; + u32 *ps_aon_auto_saved; + /* For each word of ps_all and ps_aon_all: at the commit, as the wake found it, late */ + u32 *ps_commit; + u32 *ps_aon_commit; + u32 *ps_woke; + u32 *ps_aon_woke; + bool *ps_late; + bool *ps_aon_late; + /* The rewake's own books */ + bool *ps_done; + bool *ps_aon_done; + u32 spmi_saved[2]; + bool matched; + void __iomem *dvfm[SLEEP_CLUSTERS]; + struct aura_s2r_mailbox *mbox; + + u32 wake_enable[WAKE_SOURCES_MAX]; + int wake_enable_count; + u32 wake_saved[WAKE_WORDS]; + /* The sources this sleep left enabled. */ + u32 wake_armed[WAKE_WORDS]; + + struct { + struct device_node *np; + void __iomem *asc; + bool bound; + } display[SLEEP_DISPLAY_COUNT]; + + /* Filled in for the report after a return. */ + u32 wake_status[WAKE_WORDS]; + u64 dvfm_ctl[SLEEP_CLUSTERS]; + u64 dvfm_saved[SLEEP_CLUSTERS][ARRAY_SIZE(sleep_dvfm_words)]; + bool dvfm_parked; + u64 resumes_before; + u64 resumes_after; + int cluster_e_ret; + int cluster_ave_ret; + int cluster_ave_commit_ret; + unsigned int ps_rewoken; + unsigned int ps_rewake_timeouts; + u64 cntkctl; + int cpus_stop_ret; + int ans_off_ret; + int pcie_ans_on_ret; + void __iomem *apcie_axi; + void __iomem *apcie_cio3pll; + void __iomem *apcie_clkgen; + void __iomem *apcie_phy; + void __iomem *apcie_rc; + bool apcie_port_stopped; + /* A real sleep left the radio's port down, until its host sets it up again */ + bool apcie_port_down; + void __iomem *apcie_port; + void __iomem *apcie_port_phy; + void __iomem *apcie_engine; + void __iomem *apcie_fabric; + struct apple_tunable *apcie_axi_tunable; + struct apple_tunable *apcie_cio3pll_tunable; + struct apple_tunable *apcie_clkgen_tunable; + struct apple_tunable *apcie_phy_tunable; + void __iomem *apcie_phy_ip; + struct apple_tunable *apcie_phy_ip_pll; + struct apple_tunable *apcie_phy_ip_lanes; + bool pcie_ans_stopped; + /* The words apple_sleep_lower_stray() lowered, one bit each */ + unsigned long stray_lowered; + /* What must be bound for a sleep, and what must have stopped by the last step */ + struct apple_sleep_gate { + const char *name; + /* Compared only, never dereferenced: nodes outlive the gate. */ + const struct device_node *np; + void __iomem *asc; + bool bound; + bool must_stop; + bool sep; + u32 ps; + } gate[16]; + unsigned int gate_count; + /* The AP pins, which a deep sleep resets */ + void __iomem *pins; + u32 *pins_saved; + unsigned long *pins_skip; + unsigned int npins; + unsigned int pin_irq_groups; + bool pins_saved_valid; + bool pins_lost; + /* The radio port's PERST# among them, or -1 */ + int radio_perst; + bool radio_perst_low; + bool returned; + /* This wake came back down the deep path, not the DRAM-kept one. */ + bool deep_ran; + /* This sleep's wake came from the SMC's sources alone. */ + bool smc_woke_alone; + /* A power word the DRAM-kept wake polls stayed latched: this sleep went deep. */ + bool deep_once; + /* Each latched word the commit cleared, as it found it */ + struct { + u32 offset; + u32 val; + } ps_unlatched[48]; + unsigned int ps_unlatched_count; +}; + +static struct apple_sleep *apple_sleep; + +static struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev == dev) + return &apple_sleep_coprocs[i]; + + return NULL; +} + +/* A registered coprocessor its driver has readied, whose run bit the last step clears. */ +static bool apple_sleep_coproc_ready_node(const struct device_node *np) +{ + int i; + + for (i = 0; np && i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && + apple_sleep_coprocs[i].dev->of_node == np) + return true; + return false; +} + +static bool sleep_usb_asleep; + +void apple_sleep_usb_set_asleep(bool asleep) +{ + WRITE_ONCE(sleep_usb_asleep, asleep); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_asleep); + +static unsigned long sleep_usb_wake_ports; + +void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ + WRITE_ONCE(sleep_usb_wake_ports, ports); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); + +bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return port < BITS_PER_LONG && (READ_ONCE(sleep_usb_wake_ports) & BIT(port)); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); + +/* The USB host's vote, kept until the word is mapped. Only bit 0 is touched, as on macOS. */ +static DEFINE_SPINLOCK(sleep_slp_ddr_lock); +static bool sleep_slp_ddr_usb; +static void __iomem *sleep_slp_ddr_word; + +/* + * A coprocessor that runs through a sleep crashed. The AOP is the always-on master SLP_DDR keeps + * DRAM up for: after its firmware crashed, the next SLP_DDR wake stalled in the sleep engine, + * while the deep path doesn't need it. Nothing restarts it short of a reboot. + */ +static bool sleep_slp_ddr_crashed; + +/* This sleep only: the DRAM-kept wake would stall on something this commit leaves wrong. */ +static bool sleep_slp_ddr_deep_once; + +static void sleep_slp_ddr_write(void) +{ + u32 val; + + lockdep_assert_held(&sleep_slp_ddr_lock); + if (!sleep_slp_ddr_word) + return; + val = readl(sleep_slp_ddr_word) & ~WAKE_SLP_DDR_USB; + if (sleep_slp_ddr_usb && !sleep_slp_ddr_crashed && !sleep_slp_ddr_deep_once) + val |= WAKE_SLP_DDR_USB; + writel(val, sleep_slp_ddr_word); +} + +/* A driver found something the DRAM-kept wake would stall on: this sleep goes deep. */ +static bool sleep_deep_requested; + +bool apple_sleep_request_deep_once(void) +{ + if (!READ_ONCE(apple_sleep)) + return false; + WRITE_ONCE(sleep_deep_requested, true); + + return true; +} +EXPORT_SYMBOL_GPL(apple_sleep_request_deep_once); + +static void sleep_slp_ddr_set_deep_once(bool deep) +{ + guard(spinlock_irqsave)(&sleep_slp_ddr_lock); + sleep_slp_ddr_deep_once = deep; + sleep_slp_ddr_write(); +} + +void apple_sleep_usb_slp_ddr(bool up) +{ + guard(spinlock_irqsave)(&sleep_slp_ddr_lock); + sleep_slp_ddr_usb = up; + sleep_slp_ddr_write(); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_slp_ddr); + +/* + * From RTKit, for every firmware crash. One this driver stops and restarts around each sleep + * doesn't run through it; for any other, every later sleep takes the deep path. It may come + * from a poll with interrupts off, so the table is read without its lock: entries only change + * at registration. + */ +void apple_sleep_coproc_crashed(struct device *dev) +{ + if (!apple_sleep || apple_sleep_coproc_find(dev)) + return; + scoped_guard(spinlock_irqsave, &sleep_slp_ddr_lock) { + if (sleep_slp_ddr_crashed) + return; + sleep_slp_ddr_crashed = true; + sleep_slp_ddr_write(); + } + dev_warn(apple_sleep->dev, "%s crashed: later sleeps take the deep path\n", dev_name(dev)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_crashed); + +static bool sleep_debug_link_cut; + +/* Only the debug port's controller took the cold reset. */ +static bool sleep_debug_port_is(struct device *dev) +{ + return apple_sleep && apple_sleep->hpm_np && dev->of_node == apple_sleep->hpm_np; +} + +bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return sleep_debug_port_is(dev) && xchg(&sleep_debug_link_cut, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_was_cut); + +bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return sleep_debug_port_is(dev) && READ_ONCE(sleep_debug_link_cut); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_peek); + +/* The radio's host set its port up again after a real sleep: the next commit may stop it. */ +void apple_sleep_radio_port_up(void) +{ + if (apple_sleep) + WRITE_ONCE(apple_sleep->apcie_port_down, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_port_up); + +#define PIN_REG(n) (4 * (n)) +#define PIN_DATA BIT(0) +#define PIN_LOCK BIT(21) +#define PIN_CTL 0xc48 +#define PIN_IRQ(g, w) (0x800 + 0x40 * (g) + 4 * (w)) + +/* Before any device suspends, with the pins' domain on. Skipped pins aren't read. */ +static void sleep_pins_save(struct apple_sleep *s) +{ + unsigned int pin; + + s->pins_lost = false; + s->pins_saved_valid = false; + if (!s->pins) + return; + for (pin = 0; pin < s->npins; pin++) + if (!test_bit(pin, s->pins_skip)) + s->pins_saved[pin] = readl(s->pins + PIN_REG(pin)); + s->pins_saved_valid = true; +} + +/* + * A deep sleep resets every pin, and the pin controller's driver puts back only the + * ones it configured. macOS puts all of them back right before ANS's run bit, and + * ANS's link doesn't come up without that: 1 written at 0xc48, every interrupt + * group's pending bits cleared, then each pin written back as it reads, then as it + * was, its lock bit last. + */ +void apple_sleep_restore_pins(void) +{ + struct apple_sleep *s = apple_sleep; + unsigned int pin, grp, w, changed = 0; + + if (!s || !s->pins_lost) + return; + s->pins_lost = false; + + writel(1, s->pins + PIN_CTL); + for (grp = 0; grp < s->pin_irq_groups; grp++) + for (w = 0; w < DIV_ROUND_UP(s->npins, 32); w++) + writel(U32_MAX, s->pins + PIN_IRQ(grp, w)); + for (pin = 0; pin < s->npins; pin++) { + void __iomem *reg = s->pins + PIN_REG(pin); + u32 want = s->pins_saved[pin], cur; + + if (test_bit(pin, s->pins_skip)) + continue; + cur = readl(reg); + changed += cur != want; + writel(cur, reg); + if (want & PIN_LOCK) + writel(want & ~PIN_LOCK, reg); + writel(want, reg); + } + dev_info(s->dev, "%u of %u AP pins changed by the sleep, all put back\n", changed, + s->npins - bitmap_weight(s->pins_skip, s->npins)); +} +EXPORT_SYMBOL_GPL(apple_sleep_restore_pins); + +/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data) +{ + struct apple_sleep_coproc *c; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_coproc_find(dev)) + return -EEXIST; + c = apple_sleep_coproc_find(NULL); + if (!c) + return -ENOSPC; + c->asc = asc; + c->pcie_ans = pcie_ans; + c->last_check = NULL; + c->last_stop = last_stop; + c->data = data; + c->ready = false; + c->stopped = false; + c->first = false; + c->ps = 0; + c->dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); + +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data) +{ + struct apple_sleep_coproc *c; + struct device_node *np; + u32 ps = 0; + int ret; + + if (!last_stop) + return -EINVAL; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np) { + of_property_read_u32(np, "reg", &ps); + of_node_put(np); + } + ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); + if (ret) + return ret; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) { + c->first = true; + c->ps = ps; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); + +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return -ENOENT; + c->last_check = last_check; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); + +int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data) +{ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + if (!stop) + return -EINVAL; + if (!apple_sleep_commit_stops_coprocs()) + return -EOPNOTSUPP; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev) + return -EBUSY; + apple_sleep_enclave.stop = stop; + apple_sleep_enclave.data = data; + apple_sleep_enclave.dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); + +void apple_sleep_enclave_unregister(struct device *dev) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev == dev) + memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); + +void apple_sleep_coproc_unregister(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) + memset(c, 0, sizeof(*c)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); + +void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return; + c->ready = ready; + c->stopped = false; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); + +bool apple_sleep_coproc_stopped(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + return c && c->stopped; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); + +bool apple_sleep_commit_stops_coprocs(void) +{ + return READ_ONCE(apple_sleep) && pm_suspend_via_firmware(); +} +EXPORT_SYMBOL_GPL(apple_sleep_commit_stops_coprocs); + +/* + * The last thing that can fail on the way to the commit: no CPU but this one is up and + * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only + * through a real sleep, so every refusal has to be behind this, and nothing after it may + * go back. In the order measured on macOS, on ANS itself: whatever its driver does last, + * status and control read, then 0 written twice. + */ +static void apple_sleep_coproc_run_off(struct apple_sleep_coproc *c) +{ + /* Its own last stop cleared its run bits. */ + if (!c->asc) { + c->stopped = true; + return; + } + c->status = readl(c->asc + ASC_CPU_STATUS); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + c->stopped = true; +} + +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +static bool apple_sleep_coproc_lowers(u32 ps) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && + apple_sleep_coprocs[i].first && apple_sleep_coprocs[i].ps == ps) + return true; + return false; +} + +/* The last refusals, each coprocessor's own: nothing has stopped yet. */ +static int apple_sleep_coprocs_check(struct apple_sleep *s) +{ + int i, ret; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->stopped || !c->last_check) + continue; + ret = c->last_check(c->data); + if (ret) { + dev_err(s->dev, "refused: %s: %d\n", dev_name(c->dev), ret); + return ret; + } + } + + return 0; +} + +/* + * The other cores stop at the commit, and one left awake there keeps the engine waiting + * until WD2 resets the SoC. Each answers here, before the display goes off, and waits for the + * stop or a refusal. + */ +static int apple_sleep_cpus_hold(struct apple_sleep *s) +{ + int ret = spin_table_hold_parked(); + + if (ret) + dev_err(s->dev, "refused: a CPU didn't answer the hold: %d\n", ret); + + return ret; +} + +/* A nonzero result must mean no SLEEP was sent; post-send errors are fatal. */ +static int apple_sleep_enclave_stop(struct apple_sleep *s) +{ + int ret; + + if (!apple_sleep_enclave.stop) + return 0; + ret = apple_sleep_enclave.stop(apple_sleep_enclave.data); + if (ret) + dev_err(s->dev, "refused before SEP SLEEP was sent: %d\n", ret); + + return ret; +} + +/* + * In two passes, after the last refusal: from the first stop only a real sleep restarts the + * GPU, so a failure here goes on to the commit. Those registered first stop whole first, + * before macOS's off words drop under them: the GPU withdraws its PMP request there, and the + * PMP has to see that before its own stop. The rest after the match. + */ +static void apple_sleep_coprocs_stop(struct apple_sleep *s, bool first) +{ + int i; + + if (first) + s->pcie_ans_stopped = false; + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != first) + continue; + s->pcie_ans_stopped |= c->pcie_ans; + if (c->stopped) + continue; + c->stop_ret = c->last_stop ? c->last_stop(c->data) : 0; + apple_sleep_coproc_run_off(c); + } + dsb(sy); +} + +/* MPIDR's cluster is affinity 2, its core affinity 0, as 0.0.n and 1.1.n on T8140. */ +static unsigned int sleep_cpu_cluster(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 2); +} + +static unsigned int sleep_cpu_core(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 0); +} + +static u32 sleep_cpu_start_bit(u64 mpidr) +{ + return BIT(4 * sleep_cpu_cluster(mpidr) + sleep_cpu_core(mpidr)); +} + +static void sleep_deep_wfi_arm(void) +{ + /* A timer left running ends the WFI, and costs power until reset. */ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_ENABLE); +} + +/* A core's last instructions: nothing brings it back but a reset. */ +static void __noreturn sleep_deep_wfi(void) +{ + sleep_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void sleep_cpu_stop(u64 mpidr) +{ + writel(sleep_cpu_start_bit(mpidr), apple_sleep->cpu_start + CPU_START_STOP); + dsb(sy); +} + +static void sleep_cpu_off(u64 mpidr) +{ + sleep_deep_wfi(); +} + +static int sleep_cpu_on(u64 mpidr) +{ + struct apple_sleep *s = apple_sleep; + unsigned int cluster = sleep_cpu_cluster(mpidr); + + writel(sleep_cpu_start_bit(mpidr), s->cpu_start + CPU_START_SYS); + writel(BIT(sleep_cpu_core(mpidr)), s->cpu_start + CPU_START_CLUSTER(cluster)); + + return 0; +} + +static const struct spin_table_power_ops sleep_cpu_power_ops = { + .cpu_stop = sleep_cpu_stop, + .cpu_off = sleep_cpu_off, + .cpu_on = sleep_cpu_on, +}; + +/* + * As macOS lowers a domain: AUTO_ENABLE off first if it's set, then target 0. + * Writing the flags clears them. Values measured, for instance ANS 0x1f0020ff, + * 0x0f0020ff, 0x0f0023f0. + */ +static u32 sleep_ps_lower_start(struct apple_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + + return val; +} + +static void sleep_ps_lower_finish(struct apple_sleep *s, u32 offset, u32 val) +{ + writel((val & ~PS_TARGET) | PS_FLAGS, s->ps + offset); +} + +static void sleep_ps_lower(struct apple_sleep *s, u32 offset) +{ + sleep_ps_lower_finish(s, offset, sleep_ps_lower_start(s, offset)); +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Measured + * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. + */ +static int sleep_ps_raise_start(struct apple_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 now; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + + return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, + PS_OFF_TIMEOUT_US); +} + +static void sleep_ps_raise_finish(struct apple_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, + reg); +} + +static int sleep_ps_raise(struct apple_sleep *s, u32 offset) +{ + int ret = sleep_ps_raise_start(s, offset); + + sleep_ps_raise_finish(s, offset); + + return ret; +} + +/* A domain the hardware idled under AUTO_ENABLE comes back once that's cleared. */ +static bool sleep_ps_wait_on(struct apple_sleep *s, u32 offset, u32 val) +{ + if (!FIELD_GET(PS_TARGET, val)) + return false; + + return !readl_poll_timeout_atomic(s->ps + offset, val, FIELD_GET(PS_ACTUAL, val), 1, + PS_OFF_TIMEOUT_US); +} + +static int sleep_ps_wait_off(struct apple_sleep *s, u32 offset) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, !FIELD_GET(PS_ACTUAL, val), + 10, PS_OFF_TIMEOUT_US); +} + +static bool sleep_ps_on(struct apple_sleep *s, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)); +} + +/* Below full power the hardware has idled it, and its registers may not answer. */ +static bool sleep_ps_full(struct apple_sleep *s, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)) == FIELD_MAX(PS_ACTUAL); +} + +/* @pd's word in "ps": a power controller node, child of the PMGR that "ps" maps. */ +static int sleep_ps_offset(struct apple_sleep *s, struct device_node *pd) +{ + struct device_node *pmgr __free(device_node) = of_get_parent(pd); + struct resource res; + u32 off; + + if (!pmgr || of_address_to_resource(pmgr, 0, &res) || res.start != s->ps_res.start || + of_property_read_u32(pd, "reg", &off) || off % 4 || + off + 4 > resource_size(&s->ps_res)) + return -EINVAL; + + return off; +} + +int apple_sleep_domain_check(struct device_node *pd) +{ + int off; + + if (!apple_sleep) + return -ENODEV; + off = sleep_ps_offset(apple_sleep, pd); + return off < 0 ? off : 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_domain_check); + +/* + * A power domain lowered or raised by its own driver for a sleep, before the commit or after + * a refused one, as macOS's drivers do through its PMGR. A lower waits for off. Until the + * driver raises it, the PMGR's driver leaves it alone at the wake. + */ +int apple_sleep_domain_set(struct device_node *pd, bool on) +{ + struct apple_sleep *s = apple_sleep; + int off, ret = 0; + u32 val; + + if (!s) + return -ENODEV; + off = sleep_ps_offset(s, pd); + if (off < 0) + return off; + + val = readl(s->ps + off); + if (on) { + if (FIELD_GET(PS_TARGET, val) != 0xf || FIELD_GET(PS_ACTUAL, val) != 0xf) + ret = sleep_ps_raise(s, off); + else if (!(val & PS_AUTO_ENABLE)) + sleep_ps_raise_finish(s, off); + if (!ret) + clear_bit(off / 4, s->driver_down); + return ret; + } + set_bit(off / 4, s->driver_down); + if (FIELD_GET(PS_TARGET, val)) + sleep_ps_lower(s, off); + return sleep_ps_wait_off(s, off); +} +EXPORT_SYMBOL_GPL(apple_sleep_domain_set); + +bool apple_sleep_domain_is_down(struct device_node *pd) +{ + struct apple_sleep *s = apple_sleep; + int off; + + if (!s) + return false; + off = sleep_ps_offset(s, pd); + return off >= 0 && test_bit(off / 4, s->driver_down); +} +EXPORT_SYMBOL_GPL(apple_sleep_domain_is_down); + +/* + * After a refused sleep, the sleep engine's own restart of a firmware's CPU whose domain shares + * a power switch and can't gate alone, as SIO's: each of the device's own domains reset through + * target 4 and 0, released with the domain off, then raised again, its DARTs' state kept. The + * caller has cleared the run bit, and starts the firmware after. + */ +#define PS_BUSY BIT(11) +#define PS_RESET BIT(31) +#define PS_ENGINE_RESET_4 0x8f000244 +#define PS_ENGINE_RESET_0 0x80000000 +#define ASC_CPU_STATUS_READY BIT(1) +#define ENGINE_RESET_PS 4 +#define ENGINE_RESET_DARTS 2 + +static int sleep_engine_reset_wait(struct apple_sleep *s, u32 offset, u32 actual) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, + FIELD_GET(PS_ACTUAL, val) == actual && !(val & PS_BUSY), + 10, PS_OFF_TIMEOUT_US); +} + +/* A domain of the PMGR block mapped here as "ps", and its offset in it. */ +static bool sleep_ps_domain(struct apple_sleep *s, struct device_node *pd, u32 *offset) +{ + int idx = of_property_match_string(s->dev->of_node, "reg-names", "ps"); + struct device_node *parent = of_get_parent(pd); + struct resource ps, res; + bool ours; + + ours = idx >= 0 && parent && !of_address_to_resource(s->dev->of_node, idx, &ps) && + !of_address_to_resource(parent, 0, &res) && res.start == ps.start && + !of_property_read_u32(pd, "reg", offset); + of_node_put(parent); + + return ours; +} + +/* The @index-th DART in @np's iommus, a reference held, or NULL past the last. */ +static struct device *sleep_iommus_dart(struct device_node *np, int index) +{ + struct of_phandle_args args; + struct platform_device *pdev; + + if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, &args)) + return NULL; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + + return pdev ? &pdev->dev : NULL; +} + +int apple_sleep_engine_reset(struct device *dev) +{ + struct apple_sleep *s = apple_sleep; + struct device *darts[ENGINE_RESET_DARTS] = {}; + u32 ps[ENGINE_RESET_PS], saved[ENGINE_RESET_PS], val; + struct of_phandle_iterator it; + void __iomem *asc; + int i, n = 0, idx, err, ret = 0; + + if (!s) + return -ENODEV; + of_for_each_phandle(&it, err, dev->of_node, "power-domains", NULL, 0) { + if (!sleep_ps_domain(s, it.node, &val)) + continue; + if (n == ENGINE_RESET_PS) { + of_node_put(it.node); + return -E2BIG; + } + ps[n++] = val; + } + if (!n) + return -ENODEV; + + idx = of_property_match_string(dev->of_node, "reg-names", "coproc"); + if (idx < 0) + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return -ENODEV; + + for (i = 0; i < ENGINE_RESET_DARTS; i++) { + darts[i] = sleep_iommus_dart(dev->of_node, i); + if (!darts[i]) + break; + apple_dart_state_save(darts[i]); + } + + for (i = 0; i < n && !ret; i++) { + saved[i] = readl(s->ps + ps[i]); + writel(PS_ENGINE_RESET_4, s->ps + ps[i]); + ret = sleep_engine_reset_wait(s, ps[i], 4); + if (!ret) { + writel(PS_ENGINE_RESET_0, s->ps + ps[i]); + ret = sleep_engine_reset_wait(s, ps[i], 0); + } + } + if (!ret) { + usleep_range(1000, 2000); + for (i = 0; i < n; i++) + writel(0, s->ps + ps[i]); + for (i = n - 1; i >= 0 && !ret; i--) { + writel(PS_TARGET, s->ps + ps[i]); + ret = sleep_engine_reset_wait(s, ps[i], FIELD_MAX(PS_ACTUAL)); + if (!ret) + writel(saved[i] & ~(PS_FLAGS | PS_RESET | PS_ACTUAL), + s->ps + ps[i]); + } + } + if (!ret) + ret = readl_poll_timeout(asc + ASC_CPU_STATUS, val, val & ASC_CPU_STATUS_READY, 10, + 100 * USEC_PER_MSEC); + + for (i = 0; i < ENGINE_RESET_DARTS && darts[i]; i++) { + apple_dart_state_restore(darts[i]); + put_device(darts[i]); + } + iounmap(asc); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_sleep_engine_reset); + +/* The order macOS does it in, across clusters. */ +static void sleep_dvfm_park(struct apple_sleep *s) +{ + int cluster, word; + + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) { + s->dvfm_ctl[cluster] = readq(s->dvfm[cluster] + DVFM_CTL); + for (word = 0; word < ARRAY_SIZE(sleep_dvfm_words); word++) + s->dvfm_saved[cluster][word] = + readq(s->dvfm[cluster] + sleep_dvfm_words[word]); + } + + writeq(0, s->dvfm[0] + DVFM_CTL); + writeq(0, s->dvfm[1] + DVFM_CTL); + for (word = 0; word < 2; word++) + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + sleep_dvfm_words[word]); + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + sleep_dvfm_words[word]); + s->dvfm_parked = true; +} + +static void sleep_dvfm_unpark(struct apple_sleep *s) +{ + int cluster, word; + + if (!s->dvfm_parked) + return; + s->dvfm_parked = false; + + writeq(s->dvfm_ctl[0], s->dvfm[0] + DVFM_CTL); + writeq(s->dvfm_ctl[1], s->dvfm[1] + DVFM_CTL); + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + sleep_dvfm_words[word]); + for (word = 1; word >= 0; word--) + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + sleep_dvfm_words[word]); +} + +/* + * Every device has suspended by the last step, so a stray word still on has no user: it's + * lowered there, and a sleep with one that won't go off is refused. They come back on after + * the sleep or a refusal, for genpd's books. + */ +static void apple_sleep_stray_restore(struct apple_sleep *s) +{ + const struct apple_sleep_word *stray = s->soc->stray; + int i; + + for (i = s->soc->nstray - 1; i >= 0; i--) + if (s->stray_lowered & BIT(i)) + sleep_ps_raise(s, stray[i].offset); + s->stray_lowered = 0; +} + +static int apple_sleep_lower_stray(struct apple_sleep *s) +{ + const struct apple_sleep_word *stray = s->soc->stray; + int i, ret = 0; + + s->stray_lowered = 0; + for (i = 0; i < s->soc->nstray; i++) { + u32 off = stray[i].offset; + + if (!sleep_ps_on(s, off)) + continue; + /* Its driver's own last stop lowers it, at the commit. */ + if (apple_sleep_coproc_lowers(off)) { + dev_dbg(s->dev, "%s: left on for its driver's last stop\n", + stray[i].name); + continue; + } + sleep_ps_lower(s, off); + s->stray_lowered |= BIT(i); + if (sleep_ps_wait_off(s, off)) { + dev_err(s->dev, "refused: %s won't go off: %#x\n", + stray[i].name, readl(s->ps + off)); + ret = -EBUSY; + } else { + dev_info(s->dev, "%s was on at the last step, lowered\n", + stray[i].name); + } + } + if (ret) + apple_sleep_stray_restore(s); + + return ret; +} + +/* + * The SoC's coprocessors that must be bound for a sleep, the ones that must also have stopped + * by the last step, and the domains macOS has off by then. Taken in process context, since + * the checks run with interrupts off. A driver whose stop failed has already failed the + * suspend, naming itself, so what's left is whether it's bound at all, and what the hardware + * shows. + */ +static void apple_sleep_gate_prepare(struct apple_sleep *s) +{ + const struct apple_sleep_gate_coproc *coprocs = s->soc->gate_coprocs; + int i; + + s->gate_count = 0; + for (i = 0; i < s->soc->ngate_coprocs && s->gate_count < ARRAY_SIZE(s->gate); i++) { + struct device_node *c = of_find_node_by_path(coprocs[i].path); + struct platform_device *pdev = c ? of_find_device_by_node(c) : NULL; + struct apple_sleep_gate *g; + + if (!c) { + dev_warn(s->dev, "%s isn't in this tree\n", coprocs[i].path); + continue; + } + g = &s->gate[s->gate_count++]; + g->name = c->name; + g->np = c; + g->bound = pdev && device_is_bound(&pdev->dev); + g->sep = of_node_name_eq(c, "sep"); + g->must_stop = coprocs[i].must_stop; + g->ps = coprocs[i].ps; + if (g->must_stop && !g->asc) { + int idx = of_property_match_string(c, "reg-names", "coproc"); + + if (idx < 0) + idx = of_property_match_string(c, "reg-names", "asc"); + if (idx >= 0) + g->asc = of_iomap(c, idx); + } + if (pdev) + put_device(&pdev->dev); + of_node_put(c); + } +} + +/* + * Names everything wrong, so one boot shows the whole list. A coprocessor that must stop and + * still runs refuses the sleep; the rest only warns, as a real sleep is the only full test. + */ +static bool apple_sleep_gate_failed(struct apple_sleep *s) +{ + int i, bad = 0, running = 0; + + for (i = 0; i < s->gate_count; i++) { + struct apple_sleep_gate *g = &s->gate[i]; + + /* Even an always-on domain may read actual0: require a prepared protocol. */ + if (g->sep && (!g->bound || !apple_sleep_enclave.stop)) { + dev_err(s->dev, "refused: SEP has no prepared irreversible sleep request\n"); + bad++; + running++; + continue; + } + if (!g->bound) { + dev_err(s->dev, "%s isn't bound\n", g->name); + bad++; + /* Nothing will stop it, and a commit over a running CPU resets the SoC. */ + if (g->must_stop && g->asc && g->ps && sleep_ps_full(s, g->ps) && + (readl(g->asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + dev_err(s->dev, "%s runs unbound: control %#x status %#x\n", g->name, + readl(g->asc + ASC_CPU_CONTROL), + readl(g->asc + ASC_CPU_STATUS)); + running++; + } + continue; + } + /* A CPU whose domain is off isn't running, and its registers aren't there. */ + if (g->ps && !sleep_ps_on(s, g->ps)) + continue; + if (g->must_stop && (!g->asc || + ((readl(g->asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN) && + !apple_sleep_coproc_ready_node(g->np)))) { + dev_err(s->dev, "%s didn't stop: control %#x status %#x\n", g->name, + g->asc ? readl(g->asc + ASC_CPU_CONTROL) : 0, + g->asc ? readl(g->asc + ASC_CPU_STATUS) : 0); + bad++; + running++; + } + } + + for (i = 0; i < s->soc->ngate_off; i++) { + const struct apple_sleep_word *w = &s->soc->gate_off[i]; + + if (sleep_ps_on(s, w->offset)) { + dev_err(s->dev, "%s is on: %#x\n", w->name, readl(s->ps + w->offset)); + bad++; + } + } + + if (running) { + dev_err(s->dev, "refused: %d coprocessors that must stop or gate are still running\n", + running); + return true; + } + if (bad) + dev_warn(s->dev, "going ahead with %d problems\n", bad); + else + dev_info(s->dev, "%u coprocessors up and stopped\n", s->gate_count); + + return false; +} + +static void sleep_wake_unmask(u32 *val, u32 id) +{ + if (id / 32 < WAKE_WORDS) + val[id / 32] &= ~BIT(id % 32); +} + +/* + * Mask every source but the ones asked for. These words read all zero until + * something writes them, which leaves every source enabled, listed in the ADT or + * not. macOS reads all ones outside the ADT's ap-wake-sources (measured). The AOP's + * source 0 is masked with the rest: that held under macOS with its handler hidden, + * as did leaving it enabled with the AMCC enables off. The two together are unmeasured. + */ +static void sleep_wake_masks(struct apple_sleep *s, u32 *val) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + val[i] = i == WAKE_WORDS - 1 ? s->soc->wake_last_word : U32_MAX; + for (i = 0; i < s->wake_enable_count; i++) + sleep_wake_unmask(val, s->wake_enable[i]); +} + +/* The last checks, then the stray words lowered: what has stopped may not return until a wake. */ +static int apple_sleep_refuse(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + bool refused = false, idled = false; + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev) + continue; + if (c->ready) { + idled |= c->pcie_ans; + } else { + /* Nothing may lower a coprocessor's domain under its driver. */ + dev_err(s->dev, "refused: %s is running and its driver has not quiesced it\n", + dev_name(c->dev)); + refused = true; + } + } + + if (num_online_cpus() != 1 || + (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { + dev_err(s->dev, "refused: wrong CPU, or other CPUs online\n"); + refused = true; + } + + /* Only a driver takes the display to where its domain can be lowered. */ + if (!s->display[0].bound) { + dev_err(s->dev, "refused: no driver bound to the display coprocessor\n"); + refused = true; + } + + /* ANS's link needs these after a real power loss. */ + if (idled && !s->apcie_phy_tunable) { + dev_err(s->dev, "refused: no tunables to bring ANS's PCIe block back\n"); + refused = true; + } + + /* Only ANS's driver can stop it, and it doesn't come back without a real sleep. */ + if (!idled && (sleep_ps_on(s, d->ans) || sleep_ps_on(s, d->apcie_sys_gp))) { + dev_err(s->dev, "refused: ANS or PCIe is on with no driver to stop it\n"); + refused = true; + } + + /* The engine waits for DISP_CPU with no timeout, so this must never slip. */ + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + u32 ps = d->disp[i][0]; + + if (s->display[i].bound && sleep_ps_on(s, ps) && + (readl(s->display[i].asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + dev_err(s->dev, "refused: a bound display coprocessor is still running\n"); + refused = true; + } + } + + if (apple_sleep_gate_failed(s)) + refused = true; + + if (refused) + return -EBUSY; + + return apple_sleep_lower_stray(s); +} + +static void sleep_wake_arm(struct apple_sleep *s) +{ + u32 val[WAKE_WORDS]; + int i; + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_saved[i] = readl(s->wake + WAKE_MASK(i)); + sleep_wake_masks(s, val); + for (i = 0; i < WAKE_WORDS; i++) { + writel(val[i], s->wake + WAKE_MASK(i)); + s->wake_armed[i] = ~val[i]; + } +} + +static void sleep_wake_restore(struct apple_sleep *s) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + writel(s->wake_saved[i], s->wake + WAKE_MASK(i)); +} + +/* + * The sleep engine waits on DISP_CPU with no timeout, so the display domains + * must be off. A coprocessor's driver stops its firmware first, and nothing here + * talks to the firmware. Otherwise no driver is bound, and the bootloader left the + * firmware running: on the DCP, stop its CPU cold. That's unproven. The coprocessor + * restarts from its reset vector after any sleep, and the engine itself clears the + * DCP's run bit on the way down. + * + * Domain order and values as macOS's: DCPEXT then DCP, CPU, front end, system. + * The rest of the display's domains go off with these, by nothing Linux or macOS + * writes. The coprocessors' secure mailboxes are not used here. Whether that + * holds is open. + */ +static int sleep_display_off(struct apple_sleep *s) +{ + const u32 (*domains)[3] = s->soc->ps.disp; + int i, ret = 0; + + for (i = SLEEP_DISPLAY_COUNT - 1; i >= 0; i--) { + void __iomem *asc = s->display[i].asc; + + /* The engine doesn't wait for DISPEXT0. */ + if (i == 1 && !s->display[i].bound) + continue; + + if (sleep_ps_on(s, domains[i][0]) && asc && + (readl(asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + writel(0, asc + ASC_CPU_CONTROL); + writel(0, asc + ASC_CPU_CONTROL); + } + + /* Reads as a count of the DCPs powered, and macOS ends at 0. */ + writel(i, s->pmc_dcp); + if (i == 0) { + if (readl(s->ps + domains[0][0]) & PS_AUTO_ENABLE) + writel(readl(s->ps + domains[0][0]) & + ~(PS_AUTO_ENABLE | PS_FLAGS), s->ps + domains[0][0]); + writel(3, s->pwrgate + s->soc->pwrgate_disp[0]); + writel(0, s->pwrgate + s->soc->pwrgate_disp[1]); + } + sleep_ps_lower(s, domains[i][0]); + sleep_ps_lower(s, domains[i][1]); + sleep_ps_lower(s, domains[i][2]); + } + + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + int err; + + if (i == 1 && !s->display[i].bound) + continue; + + err = sleep_ps_wait_off(s, domains[i][0]); + + if (err) { + dev_err(s->dev, "refused: display domain %#x stayed on: %#x\n", + domains[i][0], readl(s->ps + domains[i][0])); + ret = err; + } + } + + return ret; +} + +/* One command on the port controller's bus, polled: no other CPU, and interrupts are off. */ +static int sleep_hpm_cmd(struct apple_sleep *s, u32 op, u32 extra, const u32 *in, u32 *out) +{ + void __iomem *b = s->hpm_spmi; + u32 status, reply; + int ret; + + while (!(readl(b + SPMI_STATUS) & SPMI_STATUS_RX_EMPTY)) + readl(b + SPMI_REPLY); + + writel(extra << 16 | SPMI_CMD_ACTIVE | s->hpm_sid << 8 | op, b + SPMI_CMD); + if (in) + writel(*in, b + SPMI_CMD); + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, SPMI_POLL_US); + if (ret) + return ret; + reply = readl(b + SPMI_REPLY); + writel(1, b + SPMI_NOTIFY_ACK); + if (out) { + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, + SPMI_POLL_US); + if (ret) + return ret; + *out = readl(b + SPMI_REPLY); + } + + /* A zero write's reply carries the value written. */ + return op == SPMI_OP_ZERO_WRITE || (reply & 0xff) == op ? 0 : -EIO; +} + +static int sleep_hpm_select(struct apple_sleep *s, u32 page) +{ + u32 val = ~0U; + int i, ret; + + ret = sleep_hpm_cmd(s, SPMI_OP_ZERO_WRITE, page << 8, NULL, NULL); + for (i = 0; !ret && i < 2000; i++) { + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(1), HPM_REG_SELECT, NULL, &val); + if (!ret && (val & 0xff) == page) + return 0; + /* A select it dropped, waking, is sent again. page | 0x80 is one in progress. */ + if (!ret && (val & 0xff) != (page | 0x80) && i % 20 == 19) { + ret = sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_ZERO_WRITE, page << 8, NULL, NULL); + } + udelay(10); + } + + return ret ?: -ETIMEDOUT; +} + +/* + * What macOS sends the other port controller about a second before a sleep, measured + * in three runs: system power state 3, then the bus's sleep command. + */ +static int sleep_hpm_sleep(struct apple_sleep *s) +{ + u32 data = HPM_SSPS_S3, cmd = HPM_CMD_SSPS, val = ~0U; + int i, ret; + + ret = sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_DATA1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_WRITE(1), HPM_REG_DATA_CMD, &data, NULL); + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA_CMD, &cmd, NULL); + /* Selected afresh for each read, as macOS does. */ + for (i = 0; !ret && val && i < 100; i++) { + udelay(1000); + ret = sleep_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); + } + if (!ret && val) + ret = -ETIMEDOUT; + /* Asleep either way: a failure here only means it wasn't told first. */ + sleep_hpm_cmd(s, SPMI_OP_SLEEP, 0, NULL, NULL); + + return ret; +} + +/* + * The partner takes the reset as an unplug, attaches again and renegotiates its contract. + * Left to finish across the commit, that sometimes failed: neither side had a contract or + * saw the other, and no VDM got through until a replug. Returns the ms it took. + */ +static int sleep_hpm_wait_contract(struct apple_sleep *s, int *gone_ms) +{ + ktime_t start = ktime_get(); + bool wake = true, up; + u32 val = 0; + int ms, ret; + + /* + * It reads back as before for a moment after the reset. Count it as back once it was + * seen gone (no contract, or no answer while the controller restarts), or after the + * floor if it never seemed to go. + */ + *gone_ms = -1; + mdelay(HPM_RESET_FIRST_MS); + for (;;) { + ms = ktime_ms_delta(ktime_get(), start); + if (ms >= HPM_CONTRACT_MS) + return -ETIMEDOUT; + ret = wake ? sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL) : 0; + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_POWER_STATUS); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); + up = !ret && (val & HPM_POWER_CONNECTED) && + FIELD_GET(HPM_POWER_TYPEC_MODE, val) == HPM_POWER_TYPEC_PD; + if (!up && *gone_ms < 0) + *gone_ms = ms; + if (up && (*gone_ms >= 0 || ms >= HPM_RESET_FLOOR_MS)) + return ms; + wake = ret; + mdelay(10); + } +} + +/* A live debug link holds the debug switch domain on. */ +static bool sleep_debug_link_up(struct apple_sleep *s) +{ + return s->hpm_spmi && + FIELD_GET(PS_ACTUAL, readl(s->ps_aon + s->soc->ps_aon_debug_switch)); +} + +/* + * The debug link stays up with the ATC0 domains off, and it holds the debug domains + * active, which macOS has idle. The only known way to end it is the port controller's + * cold reset. To the SMC that's a cable pulled: it records the port's events as wake + * reasons once it has been warned of the sleep, and wakes the AP right after the commit. + * So the SMC's driver warns it only after this. tipd rearms the link after a refusal. + */ +static bool apple_sleep_usb_down(struct apple_sleep *s) +{ + u32 cmd = HPM_CMD_COLD_RESET; + u32 enable; + int i, ret, ms, gone; + + if (s->usb_down_ran) + return false; + s->usb_down_ran = true; + + if (!sleep_debug_link_up(s)) + return false; + + dev_info(s->dev, "debug link: cutting it for the sleep\n"); + enable = readl(s->hpm_spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + /* A reset that the partner doesn't come back from is done again. */ + for (i = 1; i <= HPM_RESET_TRIES; i++) { + ret = sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA, &cmd, NULL); + if (ret) + break; + WRITE_ONCE(sleep_debug_link_cut, true); + ms = sleep_hpm_wait_contract(s, &gone); + if (ms >= 0) { + dev_info(s->dev, "debug link: contract back %d ms after reset %d (gone at %d ms)\n", + ms, i, gone); + break; + } + dev_warn(s->dev, "debug link: no contract with the partner after reset %d\n", i); + } + if (!ret) + ret = sleep_hpm_sleep(s); + writel(1, s->hpm_spmi + SPMI_NOTIFY_ACK); + writel(enable, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + s->usb_down_ret = ret; + + return true; +} + +bool apple_sleep_debug_link_cut(void) +{ + return apple_sleep ? apple_sleep_usb_down(apple_sleep) : false; +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); + +bool apple_sleep_debug_link_up(void) +{ + struct apple_sleep *s = apple_sleep; + + return s && !s->usb_down_ran && sleep_debug_link_up(s); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_up); + +/* + * macOS takes the wake for a lid closing during the sleep as a dark one: every driver wakes, + * the display stays off, and the root domain sleeps again as soon as it sees the lid closed. + */ +void apple_sleep_lid_closed_wake(void) +{ + struct apple_sleep *s = apple_sleep; + + if (!s || !s->smc_woke_alone) + return; + dev_info(s->dev, "the lid closed during the sleep: dark wake\n"); + pm_set_dark_wake(); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_wake); + +/* + * The SMC woke the AP for the port alone, and this sleep cut the debug link: the cut's own + * events. Sleep again. The link stays down until tipd rearms it at a full wake, so the next + * entry doesn't cut it. + */ +void apple_sleep_debug_link_cut_wake(void) +{ + struct apple_sleep *s = apple_sleep; + + if (!s || !s->smc_woke_alone || !READ_ONCE(sleep_debug_link_cut)) + return; + dev_info(s->dev, "debug link: its cut woke the sleep: dark wake\n"); + pm_set_dark_wake(); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_wake); + +/* + * The radio's link and the engine that starts it, in macOS's order, measured on one + * sleep: the link into L2, the engine stopped, PERST# asserted, the PHY's clocks off. + * macOS does it 40 ms before ANS's idle. Here it follows, so that a refused sleep finds + * the link untouched. A link that's down or a radio that's gone only makes the poll time + * out, and the power goes next either way. + */ +static void sleep_apcie_port_stop(struct apple_sleep *s) +{ + void __iomem *port = s->apcie_port; + void __iomem *phy = s->apcie_port_phy; + u32 val; + + writel(readl(port + APCIE_PORT_INTMSK) | APCIE_PORT_INT_LINK, port + APCIE_PORT_INTMSK); + writel(APCIE_PORT_INT_LINK, port + APCIE_PORT_INTSTAT); + + writel(readl(port + APCIE_PORT_APPCLK) | APCIE_PORT_APPCLK_CGDIS, port + APCIE_PORT_APPCLK); + /* A loader can leave the port ready with the link never trained. */ + if (readl(port + APCIE_PORT_LINKSTS) & APCIE_PORT_LINKSTS_UP) { + writel(APCIE_PORT_L2_ENTER, port + APCIE_PORT_L2); + readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), + 10, APCIE_PORT_STOP_US); + } + /* The radio keeps its power and firmware, in reset until its host releases it. */ + if (s->radio_perst >= 0 && sleep_ps_on(s, s->soc->ps.gpio)) { + void __iomem *reg = s->pins + PIN_REG(s->radio_perst); + + val = readl(reg); + writel(s->radio_perst_low ? val & ~PIN_DATA : val | PIN_DATA, reg); + } + writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); + + if (s->apcie_engine && s->apcie_fabric) { + writel(0, s->apcie_engine + APCIE_ENGINE_IRQ_MASK); + writel(3, s->apcie_fabric + APCIE_FABRIC_STOP); + writel(readl(s->apcie_engine + APCIE_ENGINE_CTRL) | APCIE_ENGINE_STOP, + s->apcie_engine + APCIE_ENGINE_CTRL); + } + udelay(1000); + + /* What this bit does isn't known. macOS sets it around PERST#. */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(0, port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), port + APCIE_PORT_PERST_GUARD); + + writel(readl(phy) & ~APCIE_PORT_PHY_EN1, phy); + writel(readl(phy) & ~APCIE_PORT_PHY_EN0, phy); + writel(readl(phy) | APCIE_PHY_CTRL_RESET, phy); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK0REQ, phy); + readl_poll_timeout_atomic(phy, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK1REQ, phy); + + writel(readl(port + APCIE_PORT_APPCLK) & ~APCIE_PORT_APPCLK_EN, port + APCIE_PORT_APPCLK); + writel(readl(port + APCIE_PORT_INTSTAT), port + APCIE_PORT_INTSTAT); +} + +static void sleep_apcie_port_stop_once(struct apple_sleep *s) +{ + /* Only with the port set up, by the loader or by its host after a wake, or it aborts. */ + if (s->apcie_port_stopped || s->apcie_port_down || !s->apcie_port || !s->apcie_port_phy || + !sleep_ps_on(s, s->soc->ps.apcie_gp) || !sleep_ps_on(s, s->soc->ps.apcie_phy_sw)) + return; + s->apcie_port_stopped = true; + sleep_apcie_port_stop(s); +} + +/* A poll here only paces the steps: the power goes next whatever it reads. */ +static void sleep_apcie_phy_off(struct apple_sleep *s) +{ + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + + writel(readl(ctrl) | APCIE_PHY_CTRL_RESET, ctrl); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK1REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK1ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK0REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_UP, ctrl); + writel(readl(run) & ~APCIE_PHY_RUN_EN2, run); + writel(readl(run) & ~APCIE_PHY_RUN_EN, run); +} + +static void sleep_apcie_clk_off(struct apple_sleep *s) +{ + void __iomem *clk = s->apcie_clkgen + APCIE_CLK_CTRL; + u32 val; + + writel(readl(clk) & ~APCIE_CLK_RUN, clk); + writel(readl(clk) | APCIE_CLK_OFF, clk); + readl_poll_timeout_atomic(clk, val, !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + writel(readl(clk) | APCIE_CLK_OFF2, clk); +} + +/* + * ANS's own link runs on this block, and macOS takes it down with the domains. Its + * order, measured on one sleep, in the same microseconds as the power words. A block's + * registers fault with its domain off, and a loader may have left any of these off. + */ +static void apple_sleep_lower_apcie(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + u32 val; + + sleep_apcie_port_stop_once(s); + /* The PHY's registers need both domains on. */ + if (s->apcie_phy && sleep_ps_on(s, d->apcie_phy_sw) && sleep_ps_on(s, d->apcie_sys_st)) + sleep_apcie_phy_off(s); + sleep_ps_lower(s, d->apcie_phy_sw); + + if (s->apcie_clkgen && sleep_ps_on(s, d->apcie_sys_st)) + sleep_apcie_clk_off(s); + sleep_ps_lower(s, d->apcie_sys_st); + sleep_ps_lower(s, d->apcie_st); + + /* + * The AXI enables are the block's link into the fabric. macOS reads the domain on + * before clearing them: powered off with them set, SoC NI 1 latches a PCIe receive + * link error once the block's last domain goes off. + */ + val = sleep_ps_lower_start(s, d->apcie_sys_gp); + if (s->apcie_axi && sleep_ps_wait_on(s, d->apcie_sys_gp, val)) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + sleep_ps_lower_finish(s, d->apcie_sys_gp, val); + + sleep_ps_lower(s, d->apcie_gp); + sleep_ps_lower(s, d->ans); +} + +/* Read-modify-write in table order, with no write where nothing changes. */ +static void sleep_tunable_apply_changed(void __iomem *regs, const struct apple_tunable *t) +{ + size_t i; + + for (i = 0; i < t->sz; i++) { + void __iomem *reg = regs + t->values[i].offset; + u32 old = readl(reg); + u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); + + if (val != old) + writel(val, reg); + } +} + +/* + * The power loss takes the PHY's PLL and lane settings with it, and ANS's link can't + * train without them. Applied as macOS does: the PLL table, then the lanes, then the + * PLL's hold released. That takes about the settling time the PHY needs here. + */ +static void sleep_apcie_phy_ip_apply(struct apple_sleep *s) +{ + void __iomem *ctl = s->apcie_phy_ip + APCIE_PHY_IP_PLL_CTL; + + sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); + sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); + writel(readl(ctl) & ~APCIE_PHY_IP_PLL_HOLD, ctl); +} + +/* + * Brought up from power-off in macOS's order, measured on one sleep: all of it follows + * the domains on the waking core, well before ANS runs. Whether ANS needs each step is + * inferred. Nothing else does this if no host driver is bound. + */ +static int apple_sleep_raise_apcie(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + int ret; + + ret = sleep_ps_raise(s, d->ans); + ret |= sleep_ps_raise(s, d->apcie_gp); + ret |= sleep_ps_raise_start(s, d->apcie_sys_gp); + if (!ret) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) | BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) | BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + sleep_ps_raise_finish(s, d->apcie_sys_gp); + ret |= sleep_ps_raise(s, d->apcie_st); + ret |= sleep_ps_raise(s, d->apcie_sys_st); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_axi, s->apcie_axi_tunable); + apple_tunable_apply(s->apcie_cio3pll, s->apcie_cio3pll_tunable); + apple_tunable_apply(s->apcie_clkgen, s->apcie_clkgen_tunable); + writel(readl(s->apcie_clkgen + APCIE_CLK_START) | APCIE_CLK_START_GO, + s->apcie_clkgen + APCIE_CLK_START); + /* Both off bits: the lowering sets them, and only a power-off clears OFF. */ + writel(readl(s->apcie_clkgen + APCIE_CLK_CTRL) & ~(APCIE_CLK_OFF2 | APCIE_CLK_OFF), + s->apcie_clkgen + APCIE_CLK_CTRL); + ret = readl_poll_timeout_atomic(s->apcie_clkgen + APCIE_CLK_CTRL, val, + !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + if (ret) + return ret; + + ret = sleep_ps_raise(s, d->apcie_phy_sw); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_phy, s->apcie_phy_tunable); + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK0REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK1REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) & ~APCIE_PHY_CTRL_RESET, ctrl); + + writel(readl(run) | APCIE_PHY_RUN_EN, run); + sleep_apcie_phy_ip_apply(s); + writel(readl(run) | APCIE_PHY_RUN_EN2, run); + ret = readl_poll_timeout_atomic(s->apcie_phy + APCIE_PHY_READY, val, val & BIT(0), 1, + APCIE_POLL_US); + if (ret) + return ret; + writel(readl(s->apcie_phy + APCIE_PHY_COMMON) | BIT(0), s->apcie_phy + APCIE_PHY_COMMON); + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + + writel(0, s->apcie_rc + APCIE_RC_ZERO); + writel(0x140, s->apcie_rc + APCIE_RC_READY_CFG); + writel(1, s->apcie_rc + APCIE_RC_READY_START); + + return readl_poll_timeout_atomic(s->apcie_rc + APCIE_RC_READY_STAT, val, val & BIT(0), 1, + APCIE_POLL_US); +} + +/* About 4 ms after the other cores stop on macOS: deepest first, ANS last. */ +static void apple_sleep_lower_io(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + + /* + * USB's complex, deepest first. macOS's host lowers the controller's and the common + * domain in its own suspend, so here they're only lowered for a host that didn't. A host + * that sleeps in place keeps the always-on one, as macOS's does. + */ + if (FIELD_GET(PS_TARGET, readl(s->ps + d->usb))) + sleep_ps_lower(s, d->usb); + if (FIELD_GET(PS_TARGET, readl(s->ps + d->usb_common))) + sleep_ps_lower(s, d->usb_common); + if (!READ_ONCE(sleep_usb_asleep)) + sleep_ps_lower(s, d->usb_aon); + if (s->pcie_ans_stopped) { + apple_sleep_lower_apcie(s); + s->ans_off_ret = sleep_ps_wait_off(s, d->ans); + } +} + +static void apple_sleep_set_wd2(struct apple_sleep *s, unsigned int secs) +{ + writel(0, s->wdt + WDT_WD2_CTL); + writel(0, s->wdt + WDT_WD2_COUNT); + writel(secs * WDT_TICKS_PER_S, s->wdt + WDT_WD2_ALARM); + writel(1, s->wdt + WDT_WD2_CTL); +} + +#define PS_PARENT_MISSING BIT(11) +#define PS_AUTO_STATE GENMASK(27, 24) +#define PS_REWAKE_PASSES 2 + +static void sleep_ps_save(struct apple_sleep *s) +{ + const struct apple_sleep_words *all = &s->soc->ps_all, *aon = &s->soc->ps_aon_all; + int i; + + for (i = 0; i < all->count; i++) + s->ps_commit[i] = readl(s->ps + all->offset[i]); + for (i = 0; i < aon->count; i++) + s->ps_aon_commit[i] = readl(s->ps_aon + aon->offset[i]); +} + +static int sleep_ps_find(const u32 *all, int n, u32 offset) +{ + int i; + + for (i = 0; i < n; i++) + if (all[i] == offset) + return i; + return -1; +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Never + * DEV_DISABLE, which keeps it off: the SEP's is set at the commit, and macOS's wake writes + * that word without it. The hardware owns the auto state field, so it stays as read. + */ +static int sleep_ps_restore(void __iomem *reg, u32 saved) +{ + u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_PARENT_MISSING | PS_DEV_DISABLE | + PS_FLAGS | PS_ACTUAL)) | (readl(reg) & PS_AUTO_STATE); + u32 now; + int ret; + + /* + * Off under AUTO_ENABLE at the commit, as the SEP's is: the hardware powers it when + * it's wanted, and forcing it on doesn't. macOS writes that word once, AUTO set. + */ + if ((saved & PS_AUTO_ENABLE) && !FIELD_GET(PS_ACTUAL, saved)) { + writel(keep | PS_AUTO_ENABLE, reg); + return 0; + } + + writel(keep, reg); + ret = readl_poll_timeout_atomic(reg, now, + FIELD_GET(PS_ACTUAL, now) == FIELD_GET(PS_TARGET, saved), 1, + PS_OFF_TIMEOUT_US); + if (saved & PS_AUTO_ENABLE) + writel(keep | PS_AUTO_ENABLE, reg); + + return ret; +} + +/* Whether the domain at @offset was on as the wake found it. */ +static bool sleep_ps_woke_on(struct apple_sleep *s, u32 offset) +{ + const struct apple_sleep_words *all = &s->soc->ps_all; + int i = sleep_ps_find(all->offset, all->count, offset); + + return i >= 0 && FIELD_GET(PS_ACTUAL, s->ps_woke[i]); +} + +/* Linux had it on and the wake left it otherwise. With AUTO_ENABLE, off is allowed. */ +static bool sleep_ps_dropped(u32 saved, u32 woke) +{ + u32 control = PS_AUTO_ENABLE | PS_DEV_DISABLE | PS_TARGET; + + if (!FIELD_GET(PS_TARGET, saved)) + return false; + if ((saved ^ woke) & control) + return true; + return !(saved & PS_AUTO_ENABLE) && + FIELD_GET(PS_ACTUAL, woke) != FIELD_GET(PS_TARGET, saved); +} + +/* + * Every domain back as it was at the commit, before any driver's resume reads its block: + * macOS's wake words first, in its order, whatever they read, if Linux had them on. Then + * any other domain the wake left otherwise, a second time for one whose parent came later. + */ +static void sleep_ps_rewake(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + const u32 *all = soc->ps_all.offset, *aon = soc->ps_aon_all.offset; + unsigned int n = soc->ps_all.count, naon = soc->ps_aon_all.count; + bool *redo = s->ps_late, *redo_aon = s->ps_aon_late; + bool *done = s->ps_done, *done_aon = s->ps_aon_done; + int i, j, pass; + + memset(done, 0, n * sizeof(*done)); + memset(done_aon, 0, naon * sizeof(*done_aon)); + memset(s->ps_late, 0, n * sizeof(*s->ps_late)); + memset(s->ps_aon_late, 0, naon * sizeof(*s->ps_aon_late)); + s->ps_rewoken = 0; + s->ps_rewake_timeouts = 0; + for (i = 0; i < n; i++) + s->ps_woke[i] = readl(s->ps + all[i]); + for (i = 0; i < naon; i++) + s->ps_aon_woke[i] = readl(s->ps_aon + aon[i]); + + for (j = 0; j < soc->ps_wake_order.count; j++) { + i = sleep_ps_find(all, n, soc->ps_wake_order.offset[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, s->ps_commit[i])) + continue; + done[i] = true; + s->ps_rewoken++; + if (sleep_ps_restore(s->ps + all[i], s->ps_commit[i])) + redo[i] = true; + } + for (j = 0; j < soc->ps_aon_wake_order.count; j++) { + i = sleep_ps_find(aon, naon, soc->ps_aon_wake_order.offset[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, s->ps_aon_commit[i])) + continue; + done_aon[i] = true; + s->ps_rewoken++; + if (sleep_ps_restore(s->ps_aon + aon[i], s->ps_aon_commit[i])) + redo_aon[i] = true; + } + + for (pass = 0; pass < PS_REWAKE_PASSES; pass++) { + for (i = 0; i < n; i++) { + if (!redo[i] && + (done[i] || !sleep_ps_dropped(s->ps_commit[i], s->ps_woke[i]))) + continue; + if (!done[i]) { + done[i] = true; + s->ps_rewoken++; + } + redo[i] = sleep_ps_restore(s->ps + all[i], s->ps_commit[i]); + } + for (i = 0; i < naon; i++) { + if (!redo_aon[i] && + (done_aon[i] || + !sleep_ps_dropped(s->ps_aon_commit[i], s->ps_aon_woke[i]))) + continue; + if (!done_aon[i]) { + done_aon[i] = true; + s->ps_rewoken++; + } + redo_aon[i] = sleep_ps_restore(s->ps_aon + aon[i], s->ps_aon_commit[i]); + } + } + + for (i = 0; i < n; i++) + s->ps_rewake_timeouts += redo[i]; + for (i = 0; i < naon; i++) + s->ps_rewake_timeouts += redo_aon[i]; +} + +static void sleep_ps_rewake_report(struct apple_sleep *s) +{ + const struct apple_sleep_words *all = &s->soc->ps_all, *aon = &s->soc->ps_aon_all; + int i; + + dev_dbg(s->dev, "power words: %u written back, %u not on in time\n", s->ps_rewoken, + s->ps_rewake_timeouts); + for (i = 0; i < all->count; i++) + if (s->ps_late[i]) + dev_err(s->dev, "power word %#x not on in time: %#x at the commit, %#x now\n", + all->offset[i], s->ps_commit[i], + readl(s->ps + all->offset[i])); + for (i = 0; i < aon->count; i++) + if (s->ps_aon_late[i]) + dev_err(s->dev, "power word aon %#x not on in time: %#x at the commit, %#x now\n", + aon->offset[i], s->ps_aon_commit[i], + readl(s->ps_aon + aon->offset[i])); +} + +/* As macOS writes the PMU: the reply's vector masked, then acknowledged, so none is left raised. */ +static void sleep_pmu_write(struct apple_sleep *s, u16 reg, u8 val) +{ + u32 status, enable; + + enable = readl(s->spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->spmi + SPMI_NOTIFY_ENABLE); + writel(((u32)reg << 16) | SPMI_CMD_WRITE, s->spmi + SPMI_CMD); + writel(val, s->spmi + SPMI_CMD); + if (!readl_poll_timeout_atomic(s->spmi + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) + readl(s->spmi + SPMI_REPLY); + writel(1, s->spmi + SPMI_NOTIFY_ACK); + writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); +} + +/* + * The DRAM-kept wake writes each of these words and gives its switch 171 us to report closed. + * A domain that went off with BUSY latched holds that poll until WD2 resets the SoC. The PMGR + * driver doesn't clear the latch in a system sleep, so do it here. Returns whether one stayed + * latched; the deep path then cuts their power instead. + */ +static bool apple_sleep_ps_unlatch(struct apple_sleep *s) +{ + const struct apple_sleep_words *words = &s->soc->ps_engine_polled; + bool stuck = false; + unsigned int i; + + s->ps_unlatched_count = 0; + for (i = 0; i < words->count && i < ARRAY_SIZE(s->ps_unlatched); i++) { + void __iomem *reg = s->ps + words->offset[i]; + u32 val = readl(reg), off; + + if (!(val & PS_BUSY) || (val & (PS_ACTUAL | PS_TARGET))) + continue; + /* A switch still finishing reads BUSY for a moment: only one that stays is a latch. */ + if (!readl_poll_timeout_atomic(reg, val, !(val & PS_BUSY), 1, PS_BUSY_SETTLE_US)) + continue; + s->ps_unlatched[s->ps_unlatched_count].offset = words->offset[i]; + s->ps_unlatched[s->ps_unlatched_count++].val = val; + off = val & ~(PS_AUTO_ENABLE | PS_FLAGS | PS_BUSY | PS_UNLATCH | PS_DEV_DISABLE); + writel(off | PS_UNLATCH, reg); + readl_poll_timeout_atomic(reg, val, !(val & PS_BUSY), 1, PS_OFF_TIMEOUT_US); + writel(off, reg); + if (readl(reg) & PS_BUSY) + stuck = true; + } + + return stuck; +} + +static void apple_sleep_match(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + int i, raised = 0, lowered = 0, failed = 0; + + writel(0, s->wdt + WDT_WD0_COUNT); + writel(0, s->wdt + WDT_WD1_COUNT); + sleep_pmu_write(s, PMU_BOOT_STAGE, PMU_BOOT_STAGE_SLEEP); + + s->spmi_saved[0] = readl(s->spmi + SPMI_NOTIFY_ENABLE); + s->spmi_saved[1] = readl(s->spmi_a0); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi + SPMI_NOTIFY_ENABLE); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi_a0); + + for (i = 0; i < soc->ps_macos_on.count; i++) { + u32 offset = soc->ps_macos_on.offset[i]; + + s->ps_on_saved[i] = readl(s->ps + offset); + /* State 4 reads as on, and macOS has these at f. */ + if (FIELD_GET(PS_ACTUAL, s->ps_on_saved[i]) == 0xf) + continue; + /* Measured at the commit with AUTO_ENABLE clear, unlike after a wake. */ + if (sleep_ps_raise_start(s, offset)) + failed++; + raised++; + } + for (i = 0; i < soc->ps_macos_off.count; i++) { + u32 offset = soc->ps_macos_off.offset[i]; + + s->ps_off_saved[i] = readl(s->ps + offset); + if (!sleep_ps_on(s, offset)) + continue; + sleep_ps_lower(s, offset); + lowered++; + } + for (i = 0; i < soc->ps_aon_macos_off.count; i++) { + void __iomem *reg = s->ps_aon + soc->ps_aon_macos_off.offset[i]; + u32 val = readl(reg); + + s->ps_aon_saved[i] = val; + if (!FIELD_GET(PS_ACTUAL, val)) + continue; + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); + lowered++; + } + for (i = 0; i < soc->ps_aon_macos_no_auto.count; i++) { + void __iomem *reg = s->ps_aon + soc->ps_aon_macos_no_auto.offset[i]; + u32 val = readl(reg); + + s->ps_aon_auto_saved[i] = val; + writel(val & ~(PS_AUTO_ENABLE | PS_FLAGS), reg); + } + s->matched = true; + + for (i = 0; i < soc->ps_macos_off.count; i++) { + u32 offset = soc->ps_macos_off.offset[i]; + + if (sleep_ps_wait_off(s, offset)) + dev_dbg(s->dev, "still on, macOS has it off: %x = %x\n", offset, + readl(s->ps + offset)); + } + dev_dbg(s->dev, "macOS's power words: raised %d, %d of them failed, lowered %d\n", + raised, failed, lowered); +} + +/* Back as they were, after a refusal past the match or a real wake. */ +static void apple_sleep_match_undo(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + int i; + + if (!s->matched) + return; + s->matched = false; + + for (i = soc->ps_aon_macos_no_auto.count - 1; i >= 0; i--) + writel(s->ps_aon_auto_saved[i] & ~PS_FLAGS, + s->ps_aon + soc->ps_aon_macos_no_auto.offset[i]); + for (i = soc->ps_aon_macos_off.count - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_aon_saved[i])) + writel(s->ps_aon_saved[i] & ~PS_FLAGS, + s->ps_aon + soc->ps_aon_macos_off.offset[i]); + for (i = soc->ps_macos_off.count - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_off_saved[i])) + sleep_ps_raise(s, soc->ps_macos_off.offset[i]); + for (i = soc->ps_macos_on.count - 1; i >= 0; i--) { + u32 val = s->ps_on_saved[i]; + + if (!FIELD_GET(PS_ACTUAL, val)) + sleep_ps_lower(s, soc->ps_macos_on.offset[i]); + else if (FIELD_GET(PS_ACTUAL, val) != 0xf) + writel(val & ~PS_FLAGS, s->ps + soc->ps_macos_on.offset[i]); + } + writel(s->spmi_saved[0], s->spmi + SPMI_NOTIFY_ENABLE); + writel(s->spmi_saved[1], s->spmi_a0); +} + +/* The safety net, as macOS arms it. */ +static void apple_sleep_arm_wd2(struct apple_sleep *s) +{ + writel(0, s->wdt + WDT_WD0_CTL); + apple_sleep_set_wd2(s, WDT_SLEEP_SECS); + writel(0, s->aon_misc + AON_AP_DEADLINE); +} + +static int sleep_perf_wait(void __iomem *reg, unsigned int state) +{ + u32 val; + + return readl_poll_timeout_atomic(reg, val, + (val & (DCS_BUSY | DCS_STATE)) == + FIELD_PREP(DCS_STATE, state), + 1, DCS_TIMEOUT_US); +} + +static int sleep_dcs_wait(struct apple_sleep *s, unsigned int state) +{ + return sleep_perf_wait(s->dcs, state); +} + +/* + * As macOS does at about -18 ms, on each channel in turn: the enables down a bit at a time, + * then each of four lanes strobed and cleared, two lanes on every channel before the + * next two. Each write is built from a fresh read, as the trace has it. + */ +static void apple_sleep_mcc_stop(struct apple_sleep *s) +{ + int ch, pass, n; + u32 val; + + for (ch = 0; ch < MCC_CHANNELS; ch++) { + val = readl(s->mcc[ch] + MCC_ENABLES); + for (n = 0; val && n < 32; n++) { + val &= val - 1; + writel(val, s->mcc[ch] + MCC_ENABLES); + val = readl(s->mcc[ch] + MCC_ENABLES); + } + } + + for (pass = 0; pass < 2; pass++) { + for (ch = 0; ch < MCC_CHANNELS; ch++) { + void __iomem *mcc = s->mcc[ch]; + + for (n = 2 * pass; n < 2 * pass + 2; n++) { + writel(readl(mcc + MCC_LANES_MASK), mcc + MCC_LANES_MASK); + writel(readl(mcc + MCC_LANES) | MCC_LANE_STROBE(n), + mcc + MCC_LANES); + writel(readl(mcc + MCC_LANES) & ~MCC_LANE(n), mcc + MCC_LANES); + } + } + } +} + +/* + * The engine's first step polls for DCS at this state and has no way to get it there. + * macOS has every voter at 0 by now, which is how it lands there, and steps it down itself + * on the path without the PMC. Both, here, in that order. Nothing can fail by now. + */ +static void apple_sleep_dcs_down(struct apple_sleep *s) +{ + unsigned int state; + int agent; + + for (agent = 0; agent < PMC_VOTER_AGENTS; agent++) + if (readl(s->pmc_voters + agent * PMC_VOTER_STRIDE)) + writel(0, s->pmc_voters + agent * PMC_VOTER_STRIDE); + if (!sleep_dcs_wait(s, DCS_SLEEP_STATE)) + return; + + state = FIELD_GET(DCS_STATE, readl(s->dcs)); + if (state > DCS_STEP_STATE) { + writel(DCS_STEP_STATE, s->dcs); + sleep_dcs_wait(s, DCS_STEP_STATE); + } + if (state > DCS_SLEEP_STATE) { + writel(DCS_SLEEP_STATE, s->dcs); + sleep_dcs_wait(s, DCS_SLEEP_STATE); + } +} + +/* The boot core's last WFI. An event that ends it is taken, and the WFI taken again. */ +static void __noreturn sleep_boot_wfi(struct apple_sleep *s) +{ + sleep_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + aic_sleep_ack(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +/* XNU polls these off just before its complex words. The sleep goes on whatever they read. */ +static void apple_sleep_wait_still_on(struct apple_sleep *s) +{ + const struct apple_sleep_words *words = &s->soc->ps_still_on; + int i; + + for (i = 0; i < words->count; i++) + sleep_ps_wait_off(s, words->offset[i]); +} + +/* + * The last mile, in macOS's order. It starts after every refusal: this can't be + * undone, and returning now would leave the clusters armed off. No printing, and + * under 33 ms from the complex words to the WFI (the engine's bound for the + * cores to go idle). A hang from here ends at the watchdog. + * + * What's left out on purpose: + * - The two writes of 0 to 0x302800000: reads 0, changes nothing. + * - The RTC alarm: macOS arms it after the commit. rtcwake has armed it long + * before this, with no SPMI traffic near the engine. + */ +static int apple_sleep_commit(unsigned long unused) +{ + struct apple_sleep *s = apple_sleep; + int plane, agent; + u32 val; + + /* + * The other cores stop first, as on macOS. Each answered the hold in .enter, and they + * stay coherent until the commit. + */ + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + s->cpus_stop_ret = spin_table_stop_parked(); + + apple_sleep_lower_io(s); + apple_sleep_arm_wd2(s); + apple_sleep_wait_still_on(s); + + /* Everything the wake polls is down by now, the words just lowered included. */ + if (apple_sleep_ps_unlatch(s)) { + s->deep_once = true; + sleep_slp_ddr_set_deep_once(true); + } + + /* The SoC comes down before memory does, as on macOS. */ + if (sleep_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE)) { + writel(SOC_SLEEP_STATE, s->clk_perf + CLK_PERF_SOC); + sleep_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE); + } + apple_sleep_dcs_down(s); + apple_sleep_mcc_stop(s); + + /* Gates, PTD, the first voter twice, gates: macOS's order, measured once. */ + for (agent = 0; agent < 2; agent++) { + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[1]); + } + writeq(0, s->ptd + PTD_A); + writeq(0, s->ptd + PTD_B); + for (agent = 0; agent < 2; agent++) { + readl(s->pmc_voters); + writel(0, s->pmc_voters); + readl(s->pmc_voters + 4); + } + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[1]); + + /* Already off since .prepare, as on macOS. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_commit_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, 5000); + writel(0, s->cluster + CLUSTER_E); + writel(0, s->cluster + CLUSTER_P); + + writel(readl(s->ps + s->soc->ps.sio_cpu) & ~PS_AUTO_ENABLE, s->ps + s->soc->ps.sio_cpu); + + /* The SoC refuses the first, on macOS too. */ + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_38); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_3C); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_34); + + writel(readl(s->cllt + CLLT_0) & ~CLLT_0_CLEAR, s->cllt + CLLT_0); + writel(readl(s->cllt + CLLT_1) & ~CLLT_1_CLEAR, s->cllt + CLLT_1); + writel(readl(s->cllt + CLLT_2) & ~CLLT_2_CLEAR, s->cllt + CLLT_2); + + /* + * Two runs put these writes after the CLLT's: 5 to 6 ms before the commit + * in one, and the CLLT at 5.75 ms in the other. No run has hooked both. + */ + sleep_dvfm_park(s); + + sleep_ps_save(s); + for (plane = 0; plane < ARRAY_SIZE(s->amcc); plane++) { + void __iomem *amcc = s->amcc[plane]; + + writel(readl(amcc + AMCC_WORD0_ENABLE) & ~AMCC_WORD0_DSID_AGE_OUT, + amcc + AMCC_WORD0_ENABLE); + writel(readl(amcc + AMCC_WORD1_ENABLE) & ~AMCC_WORD1_UNEXP_RT_HIT, + amcc + AMCC_WORD1_ENABLE); + } + + /* aura reads this with the MMU off. */ + s->resumes_before = READ_ONCE(s->mbox->resumes); + WRITE_ONCE(s->mbox->resume_pa, __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dsb(sy); + + writel(1, s->commit); + + /* As on macOS: a raised line would end the last WFI, and the engine waits for it. */ + aic_sleep_mask_all(); + + /* Each core's deep WFI comes after the commit, this one last. No SPMI from here on. */ + spin_table_release_stopped(); + sleep_boot_wfi(s); +} + +/* Back on the boot core through the reset vector, MMU on, in macOS's order. */ +static void apple_sleep_return(struct apple_sleep *s) +{ + u32 val; + int i; + + /* + * The reset leaves the timer control at its EL2 default, without the event stream + * or EL0's counter. The timer driver's CPU PM exit puts it back too, later. + */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + + /* This core takes the AIC's IRQs, and a reset may have changed that. */ + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + + spin_table_stop_finished(); + aic_sleep_restore(); + + writel(CLUSTER_ON, s->cluster + CLUSTER_E); + s->cluster_e_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_E, val, + val & CLUSTER_E_READY, 1, + CLUSTER_TIMEOUT_US); + writel(CLUSTER_ON, s->cluster + CLUSTER_P); + + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_wake[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_wake[1]); + + sleep_ps_rewake(s); + + /* Only C2S turns AP_TMM on. */ + s->deep_ran = sleep_ps_woke_on(s, s->soc->ps.ap_tmm); + + val = readl(s->ps + s->soc->ps.sep); + if (val & PS_DEV_DISABLE) + writel(val & ~PS_DEV_DISABLE, s->ps + s->soc->ps.sep); + + /* The complex comes back on, and macOS turns it off. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, CLUSTER_TIMEOUT_US); + + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[1]); + + sleep_dvfm_unpark(s); + + /* macOS stops it here. Keep a longer one over device resume, until .end. */ + apple_sleep_set_wd2(s, WDT_SUSPEND_SECS); + + /* + * What the commit lowered for a driver comes back before anything uses it, in + * macOS's order, measured about 14 ms after this point. With no driver, genpd's + * books decide, as they do for every other domain. + */ + if (s->pcie_ans_stopped) + s->pcie_ans_on_ret = apple_sleep_raise_apcie(s); + s->apcie_port_down = true; + s->pins_lost = s->pins_saved_valid; + + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + s->resumes_after = READ_ONCE(s->mbox->resumes); + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_status[i] = readl(s->wake + WAKE_STATUS(i)); + s->returned = true; +} + +static int apple_sleep_enter(suspend_state_t state) +{ + struct apple_sleep *s = apple_sleep; + int ret; + + ret = apple_sleep_refuse(s); + if (ret) + return ret; + ret = apple_sleep_cpus_hold(s); + /* As on macOS, the display goes off before the GPU. */ + if (!ret) + ret = sleep_display_off(s); + if (!ret) + ret = apple_sleep_coprocs_check(s); + /* Cover the platform's own preparation, after the PM core's early check. */ + if (!ret && pm_wakeup_pending()) + ret = -EBUSY; + if (!ret) + ret = apple_sleep_enclave_stop(s); + if (ret) { + dev_dbg(s->dev, "refused before any commit coprocessor stop: %d\n", ret); + spin_table_unhold_parked(); + apple_sleep_stray_restore(s); + return ret; + } + apple_sleep_coprocs_stop(s, true); + + sleep_wake_arm(s); + apple_sleep_match(s); + + /* Acknowledge a leftover done flag, as the DCS word has none on macOS. */ + if (readl(s->cluster_dcs) & CLUSTER_UP_DONE) + writel(readl(s->cluster_dcs), s->cluster_dcs); + apple_sleep_coprocs_stop(s, false); + /* Done by now unless no SMC driver is bound. */ + apple_sleep_usb_down(s); + s->deep_once = READ_ONCE(sleep_deep_requested); + if (s->deep_once) + sleep_slp_ddr_set_deep_once(true); + + s->cntkctl = read_sysreg(cntkctl_el1); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, apple_sleep_commit); + if (ret) + panic("Apple committed sleep returned without physical wake: %d", ret); + apple_sleep_return(s); + if (s->deep_once) + sleep_slp_ddr_set_deep_once(false); + + apple_sleep_match_undo(s); + apple_sleep_stray_restore(s); + sleep_wake_restore(s); + + return ret; +} + +/* From here to .end a hang resets the machine. Nobody has to be there to recover it. */ +static int apple_sleep_begin(suspend_state_t state) +{ + /* The last sleep's cut is its port controller's to take, by its .complete. */ + WRITE_ONCE(sleep_debug_link_cut, false); + /* The PM core also calls .begin for s2idle when no s2idle ops are installed. */ + if (state != PM_SUSPEND_MEM) + return 0; + + if (apple_sleep) { + apple_sleep->apcie_port_stopped = false; + apple_sleep->usb_down_ran = false; + apple_sleep->smc_woke_alone = false; + sleep_pins_save(apple_sleep); + } + apple_sleep_set_wd2(apple_sleep, WDT_SUSPEND_SECS); + pm_set_suspend_via_firmware(); + + return 0; +} + +static void apple_sleep_end(void) +{ + WRITE_ONCE(sleep_deep_requested, false); + if (pm_suspend_via_firmware()) + writel(0, apple_sleep->wdt + WDT_WD2_CTL); +} + +/* + * macOS turns this complex off at boot and again at the end of each wake, so at the commit + * its word is already 0 and the engine has no sequence to run for it. Turning it off at the + * commit instead puts that sequence, with its untimed wait, right before the CPU complexes'. + */ +static void apple_sleep_ave_off(struct apple_sleep *s) +{ + u32 val; + int ret; + + /* Target 0, and the leftover done flag from iBoot's up, as XNU's first write does. */ + writel(CLUSTER_UP_DONE, s->cluster + CLUSTER_AVE); + ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, 1, CLUSTER_TIMEOUT_US); + if (ret) + dev_warn(s->dev, "AVE complex didn't go off: %#x\n", val); +} + +static int apple_sleep_prepare(void) +{ + struct apple_sleep *s = apple_sleep; + int i; + + apple_sleep_ave_off(s); + + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + struct platform_device *pdev; + + s->display[i].bound = false; + if (!s->display[i].np) + continue; + pdev = of_find_device_by_node(s->display[i].np); + if (pdev) { + s->display[i].bound = device_is_bound(&pdev->dev); + put_device(&pdev->dev); + } + } + apple_sleep_gate_prepare(s); + + return 0; +} + +/* Only the SMC's sources woke the SoC. */ +static bool apple_sleep_smc_woke_alone(struct apple_sleep *s) +{ + const struct apple_sleep_wake_name *names = s->soc->wake_names; + u32 smc[WAKE_WORDS] = {}; + bool any = false; + int i; + + /* SMC.CPUWakeupAP and SMC.OutboxNotEmpty, by their names. */ + for (i = 0; i < s->soc->nwake_names; i++) { + u32 id = names[i].id; + + if (strstarts(names[i].name, "SMC.") && id / 32 < WAKE_WORDS) + smc[id / 32] |= BIT(id % 32); + } + for (i = 0; i < WAKE_WORDS; i++) { + u32 woke = s->wake_status[i] & s->wake_armed[i]; + + if (woke & ~smc[i]) + return false; + any |= !!woke; + } + + return any; +} + +static void apple_sleep_report(void) +{ + struct apple_sleep *s = apple_sleep; + const struct apple_sleep_wake_name *names = s->soc->wake_names; + int i; + + s->smc_woke_alone = false; + if (!s->returned) + return; + s->returned = false; + s->smc_woke_alone = apple_sleep_smc_woke_alone(s); + + dev_info(s->dev, "back from sleep, boot object resumes %llu -> %llu\n", + s->resumes_before, s->resumes_after); + for (i = 0; i < s->ps_unlatched_count; i++) + dev_warn(s->dev, "power word %#x was off with BUSY latched at the commit: %#x\n", + s->ps_unlatched[i].offset, s->ps_unlatched[i].val); + if (s->deep_once) + dev_info(s->dev, "took the deep path: a power word stayed latched or stuck\n"); + if (s->deep_ran) + dev_info(s->dev, "the deep path ran\n"); + for (i = 0; i < s->soc->nwake_names; i++) { + u32 id = names[i].id; + + if (s->wake_status[id / 32] & BIT(id % 32)) + dev_info(s->dev, "wake source: %s\n", names[i].name); + } + sleep_ps_rewake_report(s); + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (c->dev && c->stopped && c->stop_ret) + dev_err(s->dev, "%s stop reported %d; physical sleep completed\n", + dev_name(c->dev), c->stop_ret); + } + if (s->cpus_stop_ret) + dev_err(s->dev, "other CPUs did not stop: %d\n", s->cpus_stop_ret); + if (s->cluster_e_ret || s->cluster_ave_ret || s->cluster_ave_commit_ret) + dev_err(s->dev, "cluster power timed out: E %d AVE %d, AVE at the commit %d\n", + s->cluster_e_ret, s->cluster_ave_ret, s->cluster_ave_commit_ret); + if (s->ans_off_ret) + dev_err(s->dev, "ANS did not read off at the commit: %d\n", s->ans_off_ret); + if (s->usb_down_ret) + dev_err(s->dev, "debug link: the stop failed: %d\n", s->usb_down_ret); + if (s->pcie_ans_stopped && s->pcie_ans_on_ret) + dev_err(s->dev, "ANS's PCIe block did not come up after the wake: %d\n", + s->pcie_ans_on_ret); +} + +static const struct platform_suspend_ops apple_sleep_ops = { + .valid = suspend_valid_only_mem, + .begin = apple_sleep_begin, + .prepare = apple_sleep_prepare, + .enter = apple_sleep_enter, + .wake = apple_sleep_report, + .end = apple_sleep_end, +}; + +static int apple_sleep_read_wake_sources(struct apple_sleep *s) +{ + struct device_node *np = s->dev->of_node; + int count; + + count = of_property_read_variable_u32_array(np, "apple,wake-enable", s->wake_enable, + 1, WAKE_SOURCES_MAX); + if (count < 0) + return dev_err_probe(s->dev, count, "no wake source enabled\n"); + s->wake_enable_count = count; + + return 0; +} + +static int apple_sleep_map_displays(struct apple_sleep *s) +{ + int i; + + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + struct device_node *np; + int index; + + np = of_parse_phandle(s->dev->of_node, "apple,display-coprocessors", i); + if (!np) + return dev_err_probe(s->dev, -ENODEV, "no display coprocessor %d\n", i); + s->display[i].np = np; + + index = of_property_match_string(np, "reg-names", "coproc"); + if (index < 0) + return dev_err_probe(s->dev, index, "%pOF has no coproc registers\n", np); + s->display[i].asc = of_iomap(np, index); + if (!s->display[i].asc) + return dev_err_probe(s->dev, -ENOMEM, "cannot map %pOF\n", np); + } + + return 0; +} + +static void __iomem *apple_sleep_map(struct device *dev, const char *name) +{ + struct device_node *np = dev->of_node; + struct resource res; + void __iomem *base; + int index; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return IOMEM_ERR_PTR(index); + if (of_address_to_resource(np, index, &res)) + return IOMEM_ERR_PTR(-EINVAL); + + /* Shared with the drivers of these blocks, so not claimed. */ + base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!base) + return IOMEM_ERR_PTR(-ENOMEM); + + return base; +} + +/* The register blocks every sleep writes, by their reg-names. */ +static const struct { + const char *name; + size_t offset; +} apple_sleep_regs[] = { + { "ps", offsetof(struct apple_sleep, ps) }, + { "pwrgate", offsetof(struct apple_sleep, pwrgate) }, + { "cpu-start", offsetof(struct apple_sleep, cpu_start) }, + { "cllt", offsetof(struct apple_sleep, cllt) }, + { "cluster", offsetof(struct apple_sleep, cluster) }, + { "commit", offsetof(struct apple_sleep, commit) }, + { "wdt", offsetof(struct apple_sleep, wdt) }, + { "aon-misc", offsetof(struct apple_sleep, aon_misc) }, + { "wake", offsetof(struct apple_sleep, wake) }, + { "pmc-dcp", offsetof(struct apple_sleep, pmc_dcp) }, + { "spmi", offsetof(struct apple_sleep, spmi) }, + { "pmc-voters", offsetof(struct apple_sleep, pmc_voters) }, + { "dcs", offsetof(struct apple_sleep, dcs) }, + { "ps-aon", offsetof(struct apple_sleep, ps_aon) }, + { "spmi-a0", offsetof(struct apple_sleep, spmi_a0) }, + { "clk-mux", offsetof(struct apple_sleep, clk_mux) }, + { "mcc0", offsetof(struct apple_sleep, mcc[0]) }, + { "mcc1", offsetof(struct apple_sleep, mcc[1]) }, + { "mcc2", offsetof(struct apple_sleep, mcc[2]) }, + { "ptd", offsetof(struct apple_sleep, ptd) }, + { "clk-perf", offsetof(struct apple_sleep, clk_perf) }, + { "cluster-dcs", offsetof(struct apple_sleep, cluster_dcs) }, + { "dvfm-e", offsetof(struct apple_sleep, dvfm[0]) }, + { "dvfm-p", offsetof(struct apple_sleep, dvfm[1]) }, + { "amcc0", offsetof(struct apple_sleep, amcc[0]) }, + { "amcc1", offsetof(struct apple_sleep, amcc[1]) }, + { "amcc2", offsetof(struct apple_sleep, amcc[2]) }, + { "amcc3", offsetof(struct apple_sleep, amcc[3]) }, +}; + +static int apple_sleep_map_regs(struct apple_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_sleep_regs); i++) { + void __iomem **field = (void __iomem **)((u8 *)s + apple_sleep_regs[i].offset); + + *field = apple_sleep_map(s->dev, apple_sleep_regs[i].name); + if (IS_ERR(*field)) + return dev_err_probe(s->dev, PTR_ERR(*field), "no %s registers\n", + apple_sleep_regs[i].name); + } + + return 0; +} + +static int apple_sleep_map_mailbox(struct apple_sleep *s) +{ + struct device_node *chosen; + u64 pa; + int ret; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return -ENODEV; + ret = of_property_read_u64(chosen, "aurora,s2r-mailbox", &pa); + of_node_put(chosen); + if (ret) { + dev_info(s->dev, "no aurora,s2r-mailbox from the boot object: no deep sleep\n"); + return -ENODEV; + } + + s->mbox = devm_memremap(s->dev, pa, sizeof(*s->mbox), MEMREMAP_WB); + if (IS_ERR(s->mbox)) + return PTR_ERR(s->mbox); + + /* aura wrote it with the MMU off, so nothing of it can be in a cache. */ + if (s->mbox->magic != AURA_S2R_MAGIC) + return dev_err_probe(s->dev, -EINVAL, "bad mailbox magic %llx\n", + s->mbox->magic); + /* Probe can run on any CPU. Logical CPU 0 is always the boot CPU. */ + if (s->mbox->boot_mpidr != cpu_logical_map(0)) + return dev_err_probe(s->dev, -EINVAL, + "mailbox is for CPU %llx, not this boot CPU\n", + s->mbox->boot_mpidr); + + return 0; +} + +static struct apple_tunable *apple_sleep_apcie_tunable(struct apple_sleep *s, + struct device_node *np, + const char *name, const char *prop) +{ + int idx = of_property_match_string(np, "reg-names", name); + struct apple_tunable *t; + struct resource res; + + if (idx < 0 || of_address_to_resource(np, idx, &res)) + return NULL; + t = devm_apple_tunable_parse(s->dev, np, prop, &res); + + return IS_ERR(t) ? NULL : t; +} + +static void __iomem *apple_sleep_apcie_block(struct apple_sleep *s, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable) +{ + int idx = of_property_match_string(np, "reg-names", name); + + if (idx < 0) + return NULL; + if (prop) { + *tunable = apple_sleep_apcie_tunable(s, np, name, prop); + if (!*tunable) + return NULL; + } + + return of_iomap(np, idx); +} + +/* + * Its registers go with PCIe's node, which a test tree may have disabled, and a loader + * publishes the tunables there. Missing ones only refuse a real sleep with ANS stopped. + */ +static void apple_sleep_map_apcie(struct apple_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,apcie", 0); + struct device_node *engine; + + if (!np) + return; + s->apcie_axi = apple_sleep_apcie_block(s, np, "axi2af", "apple,tunable-axi2af", + &s->apcie_axi_tunable); + s->apcie_cio3pll = apple_sleep_apcie_block(s, np, "cio3pll", "apple,tunable-cio3pll", + &s->apcie_cio3pll_tunable); + s->apcie_clkgen = apple_sleep_apcie_block(s, np, "clkgen", "apple,tunable-clkgen", + &s->apcie_clkgen_tunable); + s->apcie_phy = apple_sleep_apcie_block(s, np, "phy-block", "apple,tunable-phy", + &s->apcie_phy_tunable); + s->apcie_rc = apple_sleep_apcie_block(s, np, "rc", NULL, NULL); + s->apcie_phy_ip = apple_sleep_apcie_block(s, np, "phy-ip", "apple,tunable-phy-ip-pll", + &s->apcie_phy_ip_pll); + s->apcie_phy_ip_lanes = apple_sleep_apcie_tunable(s, np, "phy-ip", + "apple,tunable-phy-ip-auspma"); + if (!s->apcie_axi || !s->apcie_cio3pll || !s->apcie_clkgen || !s->apcie_phy || + !s->apcie_rc || !s->apcie_phy_ip || !s->apcie_phy_ip_lanes) { + dev_info(s->dev, + "no PCIe tunables for ANS: suspend-to-RAM with it stopped will be refused\n"); + s->apcie_phy_tunable = NULL; + s->apcie_phy = NULL; + s->apcie_clkgen = NULL; + s->apcie_axi = NULL; + } + + s->apcie_port = apple_sleep_apcie_block(s, np, "port0", NULL, NULL); + s->apcie_port_phy = apple_sleep_apcie_block(s, np, "phy0", NULL, NULL); + engine = of_parse_phandle(np, "apple,piodma", 0); + if (engine) { + s->apcie_engine = apple_sleep_apcie_block(s, engine, "engine", NULL, NULL); + s->apcie_fabric = apple_sleep_apcie_block(s, engine, "fabric", NULL, NULL); + of_node_put(engine); + } + of_node_put(np); +} + +/* The radio port's PERST# on the AP pins @np, from the ports of the SoC's enabled PCIe host. */ +static void apple_sleep_find_radio_perst(struct apple_sleep *s, struct device_node *np) +{ + struct device_node *host, *port; + struct of_phandle_args args; + + if (!s->soc->radio_pcie) + return; + host = of_find_compatible_node(NULL, NULL, s->soc->radio_pcie); + if (!host) + return; + if (of_device_is_available(host)) { + for_each_available_child_of_node(host, port) { + if (of_parse_phandle_with_args(port, "reset-gpios", "#gpio-cells", 0, + &args)) + continue; + if (args.np == np && args.args_count >= 2 && args.args[0] < s->npins && + !test_bit(args.args[0], s->pins_skip)) { + s->radio_perst = args.args[0]; + s->radio_perst_low = args.args[1] & GPIO_ACTIVE_LOW; + } + of_node_put(args.np); + } + } + of_node_put(host); +} + +/* macOS leaves the skipped pins alone, and reading one may fault. */ +static void apple_sleep_map_pins(struct apple_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,pinctrl", 0); + u32 npins, pin; + int i, n; + + s->radio_perst = -1; + if (!np) + return; + if (!of_device_is_available(np) || of_property_read_u32(np, "apple,npins", &npins) || + !npins || npins > 512) + goto out; + s->pins_saved = devm_kcalloc(s->dev, npins, sizeof(*s->pins_saved), GFP_KERNEL); + s->pins_skip = devm_bitmap_zalloc(s->dev, npins, GFP_KERNEL); + if (!s->pins_saved || !s->pins_skip) + goto out; + n = of_property_count_u32_elems(s->dev->of_node, "apple,pinctrl-skip-pins"); + for (i = 0; i < n; i++) + if (!of_property_read_u32_index(s->dev->of_node, "apple,pinctrl-skip-pins", i, + &pin) && pin < npins) + set_bit(pin, s->pins_skip); + s->pin_irq_groups = of_irq_count(np); + s->npins = npins; + s->pins = of_iomap(np, 0); + if (s->pins) + apple_sleep_find_radio_perst(s, np); +out: + of_node_put(np); +} + +static void apple_sleep_map_debug_port(struct apple_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,debug-port", 0); + struct device_node *bus; + + if (!np) + return; + bus = of_get_parent(np); + if (bus && !of_property_read_u32_index(np, "reg", 0, &s->hpm_sid)) + s->hpm_spmi = of_iomap(bus, 0); + of_node_put(bus); + /* Kept, for as long as the driver: no unbind. */ + s->hpm_np = np; +} + +/* The books kept for each word of the SoC's lists */ +static int apple_sleep_alloc_books(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + unsigned int n = soc->ps_all.count, naon = soc->ps_aon_all.count; + struct device *dev = s->dev; + + /* One bit, and one gate, for each */ + if (soc->nstray > BITS_PER_LONG || soc->ngate_coprocs > ARRAY_SIZE(s->gate)) + return dev_err_probe(dev, -EINVAL, "the SoC's lists are too long\n"); + + s->ps_on_saved = devm_kcalloc(dev, soc->ps_macos_on.count, sizeof(u32), GFP_KERNEL); + s->ps_off_saved = devm_kcalloc(dev, soc->ps_macos_off.count, sizeof(u32), GFP_KERNEL); + s->ps_aon_saved = devm_kcalloc(dev, soc->ps_aon_macos_off.count, sizeof(u32), + GFP_KERNEL); + s->ps_aon_auto_saved = devm_kcalloc(dev, soc->ps_aon_macos_no_auto.count, sizeof(u32), + GFP_KERNEL); + s->ps_commit = devm_kcalloc(dev, n, sizeof(u32), GFP_KERNEL); + s->ps_woke = devm_kcalloc(dev, n, sizeof(u32), GFP_KERNEL); + s->ps_late = devm_kcalloc(dev, n, sizeof(bool), GFP_KERNEL); + s->ps_done = devm_kcalloc(dev, n, sizeof(bool), GFP_KERNEL); + s->ps_aon_commit = devm_kcalloc(dev, naon, sizeof(u32), GFP_KERNEL); + s->ps_aon_woke = devm_kcalloc(dev, naon, sizeof(u32), GFP_KERNEL); + s->ps_aon_late = devm_kcalloc(dev, naon, sizeof(bool), GFP_KERNEL); + s->ps_aon_done = devm_kcalloc(dev, naon, sizeof(bool), GFP_KERNEL); + if (!s->ps_on_saved || !s->ps_off_saved || !s->ps_aon_saved || !s->ps_aon_auto_saved || + !s->ps_commit || !s->ps_woke || !s->ps_late || !s->ps_done || !s->ps_aon_commit || + !s->ps_aon_woke || !s->ps_aon_late || !s->ps_aon_done) + return -ENOMEM; + + return 0; +} + +static int apple_sleep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_sleep *s; + int ret; + + /* Below EL2 a monitor owns all of this. */ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + s = devm_kzalloc(dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = dev; + s->soc = of_device_get_match_data(dev); + if (!s->soc) + return -ENODEV; + + /* Without the boot object's half, a wake would rerun its boot flow. */ + ret = apple_sleep_map_mailbox(s); + if (ret) + return ret; + + ret = apple_sleep_map_regs(s); + if (ret) + return ret; + ret = of_address_to_resource(dev->of_node, + of_property_match_string(dev->of_node, "reg-names", "ps"), + &s->ps_res); + if (ret) + return ret; + s->driver_down = devm_bitmap_zalloc(dev, resource_size(&s->ps_res) / 4, GFP_KERNEL); + if (!s->driver_down) + return -ENOMEM; + ret = apple_sleep_alloc_books(s); + if (ret) + return ret; + scoped_guard(spinlock_irqsave, &sleep_slp_ddr_lock) { + sleep_slp_ddr_word = s->wake + WAKE_SLP_DDR; + sleep_slp_ddr_write(); + } + + ret = apple_sleep_read_wake_sources(s); + if (ret) + return ret; + ret = apple_sleep_map_displays(s); + if (ret) + return ret; + apple_sleep_map_apcie(s); + apple_sleep_map_pins(s); + apple_sleep_map_debug_port(s); + + platform_set_drvdata(pdev, s); + + ret = spin_table_set_power_ops(&sleep_cpu_power_ops); + if (ret) + return dev_err_probe(dev, ret, "cannot control CPU power\n"); + + /* Last: the exported helpers take it as a probed driver, and a failure frees s. */ + apple_sleep = s; + /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ + suspend_set_ops(&apple_sleep_ops); + + dev_info(dev, "deep sleep %s: boot object resumes %llu\n", + mem_sleep_current == PM_SUSPEND_MEM ? "is the default" : + "available, not the default", READ_ONCE(s->mbox->resumes)); + + return 0; +} + +/* + * T8140 (MacBook Neo). Power words from the ADT's /arm-io/pmgr, measured from macOS's sleep + * under aura. CPU clusters: 0 is E, 1 is P, ADT /arm-io/pmgr reg[5], [7], [9] and [14], [16], + * [18]. + */ +#define T8140_PS_DISP_CPU 0x10000 +#define T8140_PS_DISP_FE 0x318 +#define T8140_PS_DISP_SYS 0x280 +#define T8140_PS_DISPEXT0_CPU 0x328 +#define T8140_PS_DISPEXT0_FE 0x320 +#define T8140_PS_DISPEXT0_SYS 0x288 +#define T8140_PS_SIO_CPU 0x2a8 +#define T8140_PS_SEP 0xc00 +#define T8140_PS_ATC0_USB_AON 0x3e0 +#define T8140_PS_ATC0_COMMON 0x3e8 +#define T8140_PS_ATC0_USB 0x3f0 +#define T8140_PS_AP_TMM 0x1a0 +#define T8140_PS_GPIO 0x120 +#define T8140_PS_ANS 0x278 +#define T8140_PS_APCIE_GP 0x258 +#define T8140_PS_APCIE_SYS_GP 0x2a0 +#define T8140_PS_APCIE_ST 0x310 +#define T8140_PS_APCIE_SYS_ST 0x3a0 +#define T8140_PS_APCIE_PHY_SW 0x3c0 + +/* + * Every power word in the ADT, read just before ANS's stop so a run shows the whole chip. + */ +static const u32 t8140_ps_all[] = { + 0x0, 0x8, 0x10, 0x18, 0x30, 0x38, 0x50, 0x58, 0x60, 0x100, 0x108, 0x110, 0x118, + 0x120, 0x128, 0x130, 0x138, 0x140, 0x148, 0x150, 0x158, 0x160, 0x168, 0x170, 0x178, + 0x180, 0x188, 0x190, 0x198, 0x1a0, 0x1a8, 0x1b0, 0x1b8, 0x1c0, 0x1c8, 0x1d0, 0x1d8, + 0x1e0, 0x1e8, 0x1f0, 0x1f8, 0x200, 0x208, 0x210, 0x218, 0x220, 0x228, 0x230, 0x238, + 0x240, 0x248, 0x250, 0x258, 0x260, 0x268, 0x270, 0x278, 0x280, 0x288, 0x290, 0x298, + 0x2a0, 0x2a8, 0x2b0, 0x2b8, 0x2c0, 0x2c8, 0x2d0, 0x2d8, 0x2e0, 0x2e8, 0x2f0, 0x2f8, + 0x300, 0x308, 0x310, 0x318, 0x320, 0x328, 0x330, 0x338, 0x340, 0x348, 0x350, 0x358, + 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x390, 0x398, 0x3a0, 0x3a8, 0x3b0, 0x3b8, + 0x3c0, 0x3c8, 0x3d0, 0x3d8, 0x3e0, 0x3e8, 0x3f0, 0xc00, 0x4000, 0x4008, 0x4010, + 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, 0x4050, 0x4058, 0x4060, + 0x8000, 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, + 0xc000, 0xc008, 0xc010, 0x10000, 0x10008, 0x10010, 0x10018, 0x10020, 0x10028, + 0x10030, 0x10038, 0x10040, +}; + +static const u32 t8140_ps_aon_all[] = { + 0x0, 0x58, 0x60, 0x68, 0x70, 0x78, 0x80, 0x88, 0x90, 0x98, 0xa0, 0xa8, 0xb0, + 0x4000, 0x4008, 0x4010, 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, + 0x4050, 0x4058, 0x4060, 0x4068, 0x4070, 0x4078, 0x4080, 0x4088, 0x4090, 0x4098, + 0x40a0, 0x40a8, 0x40b0, 0x40b8, 0x40c0, 0x40c8, 0x40d0, 0x40d8, 0x40e0, 0x40e8, + 0x40f0, 0x40f8, 0x4100, 0x4108, 0x4110, 0x4118, 0x4120, 0x4128, 0x4130, 0x4138, + 0x4140, 0x4148, 0x4150, 0x4158, 0x4160, 0x4168, 0x4170, 0x4178, 0x4180, 0x8000, + 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, 0x8050, + 0x8058, 0x8060, 0x8068, 0x8070, 0x8078, +}; + +/* What macOS has on at the commit and Linux leaves off, in its order: nothing uses them here. */ +static const u32 t8140_ps_macos_on[] = { + 0x2f0, 0x2f8, 0x358, 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x118, +}; + +/* + * The power words macOS's wake writes back on before any driver runs, in its order + * (logs/wakerec-20261105-154137Z). The engine leaves domains like GPIO dark, and genpd still + * has them on, so nothing else would. + */ +static const u32 t8140_ps_wake_order[] = { + 0xc00, 0x110, 0x118, 0x120, 0x168, 0x178, 0x2a8, 0x2f0, 0x2f8, 0x300, 0x358, 0x360, + 0x368, 0x370, 0x378, 0x380, 0x388, 0x3d0, 0x3d8, +}; + +static const u32 t8140_ps_aon_wake_order[] = { 0x78, 0x88, 0x8030, 0x8038 }; + +/* What macOS has off, highest first, as it lowers them. Domains with their own stop aren't here. */ +static const u32 t8140_ps_macos_off[] = { + 0x4060, 0x4058, 0x4050, 0x4048, 0x4040, 0x4038, 0x4030, 0x4028, 0x4020, 0x4018, + 0x4010, 0x4008, 0x4000, 0x398, 0x390, 0x350, 0x348, 0x340, 0x338, 0x330, + 0x308, 0x2e8, 0x2e0, 0x2d8, 0x2d0, 0x2c8, 0x2c0, 0x2b8, 0x2b0, 0x298, + 0x290, 0x270, 0x268, 0x250, 0x248, 0x240, 0x238, 0x230, 0x1a0, 0x190, + 0x188, 0x180, 0x170, 0x160, 0x158, 0x150, 0x148, 0x140, 0x108, +}; + +static const u32 t8140_ps_aon_macos_off[] = { + 0x8028, 0x4140, 0x40f8, 0x80, +}; + +/* The AOP's SPMI words: on, with AUTO_ENABLE off on macOS. */ +static const u32 t8140_ps_aon_macos_no_auto[] = { + 0x4078, 0x4080, +}; + +/* M2A_1B's power-gate polls, less the SEP, PMP and SIO switch, which macOS doesn't have off. */ +static const u32 t8140_ps_engine_polled[] = { + 0xc008, 0xc010, 0xc000, 0x290, 0x270, 0x8038, 0x8030, 0x8048, 0x8040, 0x8028, 0x248, + 0x8010, 0x8008, 0x8020, 0x8018, 0x8000, 0x240, 0x4038, 0x4030, 0x4028, 0x4040, 0x4060, + 0x4058, 0x4050, 0x4048, 0x4018, 0x4010, 0x4008, 0x4000, 0x268, 0x1f0, 0x298, 0x238, + 0x3b8, 0x3b0, 0x328, 0x288, 0x308, 0x3a0, 0x2a0, 0x278, +}; + +static const u32 t8140_ps_still_on[] = { + T8140_PS_ATC0_COMMON, T8140_PS_DISP_SYS, T8140_PS_DISPEXT0_SYS, T8140_PS_ANS, + 0x270 /* AVD_SYS */, 0x240 /* VENC0_SYS */, 0x248 /* VENC1_SYS */, 0x1f0 /* GFX */, + 0x230 /* JPG */, T8140_PS_APCIE_SYS_GP, +}; + +/* + * What XNU polls just before its complex words, and panics on if one is on, besides the + * display's, ANS's and ATC0's, which the commit lowers itself. + */ +static const struct apple_sleep_word t8140_stray_words[] = { + { "AVD_SYS", 0x270 }, { "VENC0_SYS", 0x240 }, { "VENC1_SYS", 0x248 }, + { "GFX", 0x1f0 }, { "JPG", 0x230 }, +}; + +/* + * A commit with the PMP and SIO still running reset the SoC 5.5 s later. The DCPs' drivers + * power their CPUs off at DPMS off. + */ +static const struct apple_sleep_gate_coproc t8140_gate_coprocs[] = { + { "/soc/dcp@412e00000", true, T8140_PS_DISP_CPU }, + { "/soc/dcp@416e00000", true, T8140_PS_DISPEXT0_CPU }, + { "/soc/pmp@300500000", true }, { "/soc/dma-controller@385e00000", true, T8140_PS_SIO_CPU }, + { "/soc/nvme@3cdcc0000" }, { "/soc/mtp@314600000" }, + { "/soc/aop@310c00000" }, { "/soc/smc@30c600000" }, + { "/soc/sep@282600000" }, { "/soc/gpu@482600000" }, +}; + +static const struct apple_sleep_word t8140_gate_off[] = { + { "ane_sys", 0x290 }, { "isp_sys", 0x268 }, { "prores", 0x250 }, + { "avd_sys", 0x270 }, { "jpg", 0x230 }, { "venc0_sys", 0x240 }, +}; + +static const struct apple_sleep_wake_name t8140_wake_names[] = { + { 0, "AOP.AICWakeupAP" }, { 5, "AOP.CPUWakeupAP" }, + { 42, "AOP.OutboxNotEmpty" }, { 62, "AOP.SPMI0.SW3" }, + { 66, "AOP.SPMI1.SW3" }, { 70, "ATC0_PMGR_USB_WAKEUP" }, + { 80, "MTP.DOCK.CHANNELS.AP0.IRQ" }, { 99, "MTP.OUTBOX.N.EMPTY_IRQ" }, + { 109, "NUB.APWakeupTime" }, { 110, "NUB.APWatchdog" }, + { 125, "NUB.GPIO.IRQ6" }, { 131, "NUB.SPMI0.SW3" }, + { 136, "NUB.SPMI1.SW3" }, { 140, "NUB.WakeupAP" }, + { 145, "SMC.CPUWakeupAP" }, { 165, "SMC.OutboxNotEmpty" }, + { 211, "AOP.TrustedOutboxNotEmpty" }, { 227, "AOP2.OutboxNotEmpty" }, + { 231, "AOP2.TrustedOutboxNotEmpty" }, { 236, "AOP2.CPUWakeupAP" }, +}; + +static const struct apple_sleep_soc t8140_sleep_soc = { + .ps = { + .disp = { + { T8140_PS_DISP_CPU, T8140_PS_DISP_FE, T8140_PS_DISP_SYS }, + { T8140_PS_DISPEXT0_CPU, T8140_PS_DISPEXT0_FE, T8140_PS_DISPEXT0_SYS }, + }, + .sio_cpu = T8140_PS_SIO_CPU, + .sep = T8140_PS_SEP, + .usb_aon = T8140_PS_ATC0_USB_AON, + .usb_common = T8140_PS_ATC0_COMMON, + .usb = T8140_PS_ATC0_USB, + .ap_tmm = T8140_PS_AP_TMM, + .gpio = T8140_PS_GPIO, + .ans = T8140_PS_ANS, + .apcie_gp = T8140_PS_APCIE_GP, + .apcie_sys_gp = T8140_PS_APCIE_SYS_GP, + .apcie_st = T8140_PS_APCIE_ST, + .apcie_sys_st = T8140_PS_APCIE_SYS_ST, + .apcie_phy_sw = T8140_PS_APCIE_PHY_SW, + }, + .ps_aon_debug_switch = 0xb0, + .pwrgate_wake = { 0x000, 0x040 }, + .pwrgate_cpm = { 0x060, 0x068 }, + .pwrgate_disp = { 0xc8c, 0xcd0 }, + .ps_all = SLEEP_WORDS(t8140_ps_all), + .ps_aon_all = SLEEP_WORDS(t8140_ps_aon_all), + .ps_macos_on = SLEEP_WORDS(t8140_ps_macos_on), + .ps_macos_off = SLEEP_WORDS(t8140_ps_macos_off), + .ps_aon_macos_off = SLEEP_WORDS(t8140_ps_aon_macos_off), + .ps_aon_macos_no_auto = SLEEP_WORDS(t8140_ps_aon_macos_no_auto), + .ps_wake_order = SLEEP_WORDS(t8140_ps_wake_order), + .ps_aon_wake_order = SLEEP_WORDS(t8140_ps_aon_wake_order), + .ps_still_on = SLEEP_WORDS(t8140_ps_still_on), + .ps_engine_polled = SLEEP_WORDS(t8140_ps_engine_polled), + .stray = t8140_stray_words, + .nstray = ARRAY_SIZE(t8140_stray_words), + .gate_off = t8140_gate_off, + .ngate_off = ARRAY_SIZE(t8140_gate_off), + .gate_coprocs = t8140_gate_coprocs, + .ngate_coprocs = ARRAY_SIZE(t8140_gate_coprocs), + .wake_names = t8140_wake_names, + .nwake_names = ARRAY_SIZE(t8140_wake_names), + /* 21 sources. Measured: macOS reads 0x001fef77 there. */ + .wake_last_word = GENMASK(20, 0), + .radio_pcie = "apple,t8140-pcie", +}; + +/* + * Only SoCs whose sleep was measured and tested. On every other the driver doesn't bind, and + * s2idle stays the only sleep. + */ +static const struct of_device_id apple_sleep_of_match[] = { + { .compatible = "apple,t8140-sleep", .data = &t8140_sleep_soc }, + { } +}; + +static struct platform_driver apple_sleep_driver = { + .driver = { + .name = "apple-sleep", + .of_match_table = apple_sleep_of_match, + .suppress_bind_attrs = true, + }, + .probe = apple_sleep_probe, +}; +builtin_platform_driver(apple_sleep_driver); + +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP_KUNIT_TEST) +#include "sleep-test.c" +#endif diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 58ada41ed96ec..b855d2bc82c83 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -12,7 +12,9 @@ */ #include +#include #include +#include #include #include #include @@ -23,8 +25,10 @@ #include #include #include +#include #include #include +#include /* SPMI Controller Registers */ #define SPMI_STATUS_REG 0 @@ -96,8 +100,11 @@ struct apple_spmi { struct irq_domain *irqd; raw_spinlock_t irq_mask_lock; DECLARE_BITMAP(irq_mask_cache, SPMI_MAX_PERIPHERAL_IRQS); + /* The sources a driver uses, enabled or not. */ + DECLARE_BITMAP(irq_used, SPMI_MAX_PERIPHERAL_IRQS); int irq; bool notify_irq; + bool notify_parked; bool prev_fail; }; @@ -339,8 +346,11 @@ static struct irq_chip apple_spmi_irq_chip = { static int apple_spmi_irq_domain_map(struct irq_domain *irqd, unsigned int irq, irq_hw_number_t hw) { + struct apple_spmi *spmi = irqd->host_data; + irq_domain_set_info(irqd, irq, hw, &apple_spmi_irq_chip, irqd->host_data, handle_edge_irq, NULL, NULL); + set_bit(hw, spmi->irq_used); return 0; } @@ -394,7 +404,9 @@ static void apple_spmi_irq_domain_free(struct irq_domain *irqd, unsigned int vir for (i = 0; i < nr_irqs; i++) { struct irq_data *d = irq_domain_get_irq_data(irqd, virq + i); + struct apple_spmi *spmi = irqd->host_data; + clear_bit(d->hwirq, spmi->irq_used); irq_set_handler(virq + i, NULL); irq_domain_reset_irq_data(d); } @@ -411,19 +423,44 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) struct apple_spmi *spmi = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); u32 notify = spmi->hw->num_irqs; - bool handled = false; + bool handled = false, held = false, raw_pending = false; unsigned long val, offset, bit; chained_irq_enter(chip, desc); val = readl(spmi->regs + SPMI_IRQ_ACK_BASE + (notify / 32) * 4); + raw_pending |= !!val; if (val & BIT(notify % 32)) { apple_spmi_irq_ack_raw(spmi, notify); complete(&spmi->fifo_rx); handled = true; } + /* Other queues' responses, such as the sleep's own commands at the debug link's cut. */ + if (val & ~BIT(notify % 32)) { + writel(val & ~BIT(notify % 32), spmi->regs + SPMI_IRQ_ACK_BASE + (notify / 32) * 4); + handled = true; + } for (offset = 0; offset < notify / 8; offset += sizeof(val)) { + unsigned long stray; + val = readq(spmi->regs + SPMI_IRQ_ACK_BASE + offset); + raw_pending |= !!val; + /* + * After a real sleep, sources no driver uses come up pending (on the Neo the + * USB-PD controllers' even lines, latched by the debug link's cut), and the + * interrupt fires for them with nothing of ours to run. Clear them, as probe does. + * A used source that is masked, as all are until the resume enables them, stays + * pending for its unmask: it isn't a bad interrupt either. + */ + stray = val & ~spmi->irq_used[offset / sizeof(val)]; + if (stray) { + writel(lower_32_bits(stray), spmi->regs + SPMI_IRQ_ACK_BASE + offset); + writel(upper_32_bits(stray), spmi->regs + SPMI_IRQ_ACK_BASE + offset + 4); + handled = true; + } + if (val & spmi->irq_used[offset / sizeof(val)] & + ~spmi->irq_mask_cache[offset / sizeof(val)]) + held = true; /** * because of other masters in the bus, we're going to get a multitude of * interrupts we're not interested in. irq_resolve_mapping isn't very @@ -436,8 +473,19 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) handled = true; } } - if (!handled) + /* + * Native dispatch completes normally when every sampled bank is empty. + * Keep the diagnostic for unclassified nonempty samples, including a + * mask transition; an empty parent needs no invented controller ACK. + */ + if (!handled && !held && raw_pending) handle_bad_irq(desc); + /* + * The acks are posted writes: let them reach the controller before the parent is + * unmasked, or it fires again for what was just cleared. + */ + if (handled) + readl(spmi->regs + SPMI_IRQ_ACK_BASE); chained_irq_exit(chip, desc); } @@ -485,6 +533,63 @@ static int apple_spmi_init_irq(struct platform_device *pdev, return 0; } +/* + * Every command asks for an interrupt on its response, on the vector this + * driver waits on. The system's wake logic watches this queue's interrupt line + * for the PMU's RTC alarm, so that vector has to be quiet and clear before a + * firmware-driven suspend, or the commit may read our own traffic as a wake. + * macOS clears the same enable bit at about the same point (measured under aura). + * Any command sent from here on polls for its response. + */ +static int apple_spmi_suspend_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + unsigned long flags; + + if (!spmi->notify_irq || !pm_suspend_via_firmware()) + return 0; + + guard(mutex)(&spmi->fifo_lock); + spmi->notify_irq = false; + spmi->notify_parked = true; + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + apple_spmi_irq_mask_raw(spmi, spmi->hw->num_irqs); + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + + return 0; +} + +static int apple_spmi_resume_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + u32 words[SPMI_MAX_PERIPHERAL_IRQS / 32]; + unsigned long flags; + size_t i; + + if (!spmi->notify_parked) + return 0; + + guard(mutex)(&spmi->fifo_lock); + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + /* Whether the enables survive a real sleep has not been measured. */ + if (pm_resume_via_firmware()) { + bitmap_to_arr32(words, spmi->irq_mask_cache, spmi->hw->num_irqs); + for (i = 0; i < spmi->hw->num_irqs / 32; i++) + writel(words[i], spmi->regs + SPMI_IRQ_MASK_BASE + i * 4); + } + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); + apple_spmi_irq_unmask_raw(spmi, spmi->hw->num_irqs); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + spmi->notify_parked = false; + spmi->notify_irq = true; + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(apple_spmi_pm_ops, apple_spmi_suspend_noirq, + apple_spmi_resume_noirq); + static int apple_spmi_probe(struct platform_device *pdev) { struct apple_spmi *spmi; @@ -547,6 +652,7 @@ static struct platform_driver apple_spmi_driver = { .driver = { .name = "apple-spmi", .of_match_table = apple_spmi_match_table, + .pm = pm_sleep_ptr(&apple_spmi_pm_ops), }, }; module_platform_driver(apple_spmi_driver); diff --git a/drivers/tty/serial/apple_dockchannel.h b/drivers/tty/serial/apple_dockchannel.h index 97fcccc6a5a57..4fcecee3b81c5 100644 --- a/drivers/tty/serial/apple_dockchannel.h +++ b/drivers/tty/serial/apple_dockchannel.h @@ -24,13 +24,20 @@ */ #define DOCKCHANNEL_NEVER_WAIT "apple,tx-never-wait" -/* Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. */ -static inline void dockchannel_wait_putc(void __iomem *data, unsigned char c) +/* + * Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. With the + * debug host gone the FIFO stays full: after one character timed out, the next + * are tried once each (*@no_reader) until there's room again, so printing on the + * way into and out of a sleep doesn't hold anything up. + */ +static inline void dockchannel_wait_putc(void __iomem *data, bool *no_reader, + unsigned char c) { - unsigned int retries = DOCKCHANNEL_TX_RETRIES; + unsigned int retries = *no_reader ? 1 : DOCKCHANNEL_TX_RETRIES; while (readl(data + DOCKCHANNEL_TX_FREE) == 0 && --retries) cpu_relax(); + *no_reader = !retries; if (!retries) return; diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 6c427189d34be..5004f17501e47 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -16,13 +16,15 @@ static struct dockchannel_tx dockchannel_early_tx; static bool dockchannel_early_never_wait; +/* The last character timed out: see dockchannel_wait_putc(). */ +static bool dockchannel_early_no_reader; static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) { if (dockchannel_early_never_wait) dockchannel_console_putc(port->membase, &dockchannel_early_tx, c); else - dockchannel_wait_putc(port->membase, c); + dockchannel_wait_putc(port->membase, &dockchannel_early_no_reader, c); } /* The DT's DockChannel UART node asks for DOCKCHANNEL_NEVER_WAIT (flat DT). */ diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index d4255cb949848..7150cf1cca350 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -109,6 +109,9 @@ static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) dcuart_irq_update(dcuart, bits, false); } +/* The last console character timed out: see dockchannel_wait_putc(). */ +static bool dcuart_no_reader; + static void dcuart_putc(struct uart_port *port, unsigned char c) { struct dcuart_port *dcuart = container_of(port, struct dcuart_port, port); @@ -116,7 +119,7 @@ static void dcuart_putc(struct uart_port *port, unsigned char c) if (dcuart->never_wait) dockchannel_console_putc(port->membase, &dcuart->console_tx, c); else - dockchannel_wait_putc(port->membase, c); + dockchannel_wait_putc(port->membase, &dcuart_no_reader, c); } static void dcuart_write(struct console *co, const char *s, diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 8e8a5f59b319b..0ea6a9a729bc5 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -2280,7 +2280,9 @@ int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg) usb_set_device_state(rhdev, USB_STATE_SUSPENDED); hcd->state = HC_STATE_SUSPENDED; - if (!PMSG_IS_AUTO(msg)) + /* A suspend after a raced autoresume finds them released already. */ + if (!PMSG_IS_AUTO(msg) && + !test_and_set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); @@ -2321,11 +2323,12 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) return 0; } - if (!PMSG_IS_AUTO(msg)) { + if (!PMSG_IS_AUTO(msg) && test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { status = usb_phy_roothub_resume(hcd->self.sysdev, hcd->phy_roothub); if (status) return status; + clear_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); } if (!hcd->driver->bus_resume) @@ -2368,7 +2371,11 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) } } else { hcd->state = old_state; - usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + /* Only release what this resume took: an autoresume took nothing. */ + if (!PMSG_IS_AUTO(msg)) { + usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); + } dev_dbg(&rhdev->dev, "bus %s fail, err %d\n", "resume", status); if (status != -ESHUTDOWN) @@ -3084,8 +3091,16 @@ void usb_remove_hcd(struct usb_hcd *hcd) usb_deregister_bus(&hcd->self); hcd_buffer_destroy(hcd); - usb_phy_roothub_power_off(hcd->phy_roothub); - usb_phy_roothub_exit(hcd->phy_roothub); + /* + * A root hub whose resume failed still has its PHYs released by the suspend, + * powered off and, without wakeup, exited too. Don't release them twice. + */ + if (!test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { + usb_phy_roothub_power_off(hcd->phy_roothub); + usb_phy_roothub_exit(hcd->phy_roothub); + } else if (device_may_wakeup(hcd->self.sysdev)) { + usb_phy_roothub_exit(hcd->phy_roothub); + } usb_put_invalidate_rhdev(hcd); hcd->flags = 0; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 30b5c5fc4d071..7821ccb4686e5 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1002,10 +1002,13 @@ struct dwc3_scratchpad_array { * need to be passed on to glue layer * @pre_set_role: Notify glue of role switch notifications * @pre_run_stop: Notify run stop enable/disable information to glue + * @pre_host_resume: Before xHCI touches the core at a system resume; an error keeps it away, + * and a positive value says the core lost its state and was set up again */ struct dwc3_glue_ops { void (*pre_set_role)(struct dwc3 *dwc, enum usb_role role); void (*pre_run_stop)(struct dwc3 *dwc, bool is_on); + int (*pre_host_resume)(struct dwc3 *dwc); }; /** diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index 7a04604ddd4a1..69abb0f32653f 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -38,10 +38,19 @@ static void dwc3_apple_non_role_test(struct kunit *test) } } +/* A role that arrives during a sleep decides what comes back after it. */ +static void dwc3_apple_role_during_sleep_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST), DWC3_APPLE_HOST); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE), DWC3_APPLE_DEVICE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE), DWC3_APPLE_NO_CABLE); +} + static struct kunit_case dwc3_apple_state_test_cases[] = { KUNIT_CASE(dwc3_apple_host_left_to_xhci_test), KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), KUNIT_CASE(dwc3_apple_non_role_test), + KUNIT_CASE(dwc3_apple_role_during_sleep_test), {} }; diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index 42c13d900428e..bb8d827fdec4c 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -9,6 +9,7 @@ #include #include +#include /* See the comment at the top of dwc3-apple.c for how these states are used. */ enum dwc3_apple_state { @@ -47,4 +48,22 @@ static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target } } +/** + * dwc3_apple_state_for_role() - state a role asks for while the core is stopped for sleep + * @role: the role asked for + * + * Return: the state to bring the controller up in after the sleep. + */ +static inline enum dwc3_apple_state dwc3_apple_state_for_role(enum usb_role role) +{ + switch (role) { + case USB_ROLE_HOST: + return DWC3_APPLE_HOST; + case USB_ROLE_DEVICE: + return DWC3_APPLE_DEVICE; + default: + return DWC3_APPLE_NO_CABLE; + } +} + #endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f66370e6b5f88..c3b02c3602d0a 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -9,16 +9,23 @@ */ #include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include +#include +#include +#include #include #include "dwc3-apple-state.h" @@ -91,6 +98,15 @@ * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device * may wake the system, as a halted xHC could not signal the wakeup. * + * A deep sleep takes the controller's power domain away. Behind a fixed hub on T8140 the host + * sleeps in place, as macOS's does: xHCI suspends and resumes as usual, and the PHY takes the + * core through hibernation in between, so the hub and its devices stay. The core comes back + * without its registers and is set up again in place. With its always-on side kept, xHCI + * restores its saved state and the hub resumes from U3, with no reset; without it xHCI starts + * over, and the devices reset-resume. Anywhere else, or when the PHY can't, the glue stops + * everything before the freezer runs, the way a cable disconnect does, and brings it back + * once the system has resumed. See dwc3_apple_pm_notify(). + * * Runtime PM (T6040 and T8132, see dwc3_apple_core_probe()): the glue holds a reference from * the reset deassert to the end of the shutdown, so with no cable the controller can * runtime-suspend and its power domain drop, its DARTs (which save and restore their state) and, @@ -128,6 +144,7 @@ * @rpm_held: The glue holds a runtime PM reference for the running core * @pre_core_rpm: Runtime PM is the glue's own until the first core bring-up * @dt_dis_u3_susphy: The device tree keeps USB3 PHY suspend off itself + * @sleep_ddr: The host votes for DRAM in a sleep while it runs (apple,sleep-ddr) * @drd_regs: The controller's optional blocks outside dwc3-core (drd-cfg, drd-bulk, drd-ausbc), * mapped for their tunables * @tunables: Controller tunables from the device tree, see dwc3_apple_tunable_list @@ -144,6 +161,8 @@ struct dwc3_apple { struct phy *usb3_phy; struct usb_role_switch *role_sw; bool fixed_hub; + /* The root port the fixed hub sits on, its node's reg. */ + u32 hub_port; bool host_only; struct mutex lock; @@ -159,6 +178,21 @@ struct dwc3_apple { bool rpm_held; bool pre_core_rpm; bool dt_dis_u3_susphy; + bool sleep_ddr; + struct notifier_block pm_nb; + /* Kept through a deep sleep, with the PHY's hibernation, and the GUID it had. */ + bool sleep_in_place; + u32 sleep_guid; + /* The core lost its registers in that sleep and was set up again in place. */ + bool sleep_reinit; + /* Looks for the fixed hub once a host restarted after a sleep has had time to find it. */ + struct delayed_work hub_check; + /* The core didn't come back from a sleep or a power cycle: nothing may touch it. */ + bool sleep_dead; + /* Stopped for a deep sleep: the state to come back to, and the wakeup setting. */ + bool sleep_stopped; + bool sleep_wakeup; + enum dwc3_apple_state sleep_state; void __iomem *drd_regs[DWC3_APPLE_NR_DRD_REGS]; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; @@ -210,6 +244,27 @@ static inline void dwc3_apple_mask(struct dwc3_apple *appledwc, u32 offset, u32 dwc3_apple_writel(appledwc, offset, reg); } +/* + * macOS's T8140 controller reset sets GCTL's hibernation enable and a power-down scale + * of 0xd. Without the enable the core's power management accepts a + * D0/D3 request but saves and restores nothing, so the PHY's hibernation for a sleep + * needs it. + */ +#define DWC3_APPLE_T8140_PWRDNSCALE 0xd + +static void dwc3_apple_setup_hibernation(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 reg; + + if (!of_device_is_compatible(appledwc->dev->of_node, "apple,t8140-dwc3")) + return; + reg = dwc3_readl(dwc, DWC3_GCTL); + reg &= ~DWC3_GCTL_PWRDNSCALE_MASK; + reg |= DWC3_GCTL_PWRDNSCALE(DWC3_APPLE_T8140_PWRDNSCALE) | DWC3_GCTL_GBLHIBERNATIONEN; + dwc3_writel(dwc, DWC3_GCTL, reg); +} + static void dwc3_apple_setup_cio(struct dwc3_apple *appledwc) { dwc3_apple_writel(appledwc, APPLE_DWC3_CIO_LFPS_OFFSET, APPLE_DWC3_CIO_LFPS_OFFSET_VALUE); @@ -361,6 +416,105 @@ static void dwc3_apple_set_ptrcap(struct dwc3_apple *appledwc, u32 mode) dwc3_set_prtcap(&appledwc->dwc, mode, false); } +/* Written to the core's GUID scratch register before a sleep in place, read back after it. */ +#define DWC3_APPLE_SLEEP_GUID 0x534c5045 + +/* macOS waits up to 100 ms for this after the core's D0, and only logs a timeout. */ +#define DWC3_APPLE_CFG_STATUS 0x2c +#define DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY BIT(30) +#define DWC3_APPLE_PIPE_PHY_READY_US 100000 + +/* + * The core is powered but its registers are at their reset values. Set it up again the way + * dwc3_apple_core_start() and dwc3_apple_set_role() do, in place: the PHYs and the xHCI + * device stay, so xHCI can restore its saved state, or start over. Runs in the resume, + * while nothing else touches the core. + */ +static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + int i, ret; + + ret = dwc3_core_init(dwc); + if (ret) + return ret; + /* dwc3_core_init() took PHY references the core holds already. */ + for (i = 0; i < dwc->num_usb3_ports; i++) { + phy_power_off(dwc->usb3_generic_phy[i]); + phy_exit(dwc->usb3_generic_phy[i]); + } + for (i = 0; i < dwc->num_usb2_ports; i++) { + phy_power_off(dwc->usb2_generic_phy[i]); + phy_exit(dwc->usb2_generic_phy[i]); + } + + dwc3_apple_setup_cio(appledwc); + dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); + dwc3_apple_set_ptrcap(appledwc, DWC3_GCTL_PRTCAP_HOST); + dwc->dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; + dwc3_enable_susphy(dwc, true); + /* A PHY that came back in a USB3 mode routes the PIPE to it now, before xHCI starts. */ + ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_HOST); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed after the sleep, USB2 only, err=%d\n", + ret); + + return 0; +} + +/* + * Runs before xHCI touches the core after a sleep in place. A core that may be unpowered must + * be left alone, as an access could be an SError: dwc3_apple_sleep_kept() deals with it once + * the system is back. A core that came back without its registers is set up again. xHCI then + * restores its saved state, as macOS's does, unless the always-on side lost power too: then + * it's told it lost its state (a positive return). A sleep refused before the hibernation + * left the core as it was. + */ +static int dwc3_apple_pre_host_resume(struct dwc3 *dwc) +{ + struct dwc3_apple *appledwc = to_dwc3_apple(dwc); + bool lost; + u32 val; + int ret; + + if (!READ_ONCE(appledwc->sleep_in_place)) + return 0; + + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + lost = ret == APPLE_ATCPHY_SLEEP_POWER_LOST; + if (ret == -EBUSY || lost) + ret = 0; + if (!ret && appledwc->drd_regs[0] && /* drd-cfg */ + readl_poll_timeout(appledwc->drd_regs[0] + DWC3_APPLE_CFG_STATUS, val, + val & DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY, 100, + DWC3_APPLE_PIPE_PHY_READY_US)) + dev_warn(appledwc->dev, "PIPE PHY not ready after the sleep: %#x\n", val); + if (!ret && dwc3_readl(dwc, DWC3_GUID) != DWC3_APPLE_SLEEP_GUID) { + ret = dwc3_apple_core_reinit(appledwc); + if (!ret && !lost) { + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core lost its registers in the sleep: set up again, xHCI restores its state\n"); + return 0; + } + if (!ret) { + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core and its always-on side lost power in the sleep: set up again, xHCI starts over\n"); + return 1; + } + } + if (ret) + dev_err(appledwc->dev, "not resuming xHCI, the core didn't come back: %d\n", ret); + + return ret; +} + +static const struct dwc3_glue_ops dwc3_apple_glue_ops = { + .pre_host_resume = dwc3_apple_pre_host_resume, +}; + static bool dwc3_apple_host_before_core_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { @@ -378,6 +532,7 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc, enum dwc3_apple_st WARN_ON_ONCE(appledwc->state != DWC3_APPLE_PROBE_PENDING); appledwc->dwc.dev = appledwc->dev; + appledwc->dwc.glue_ops = &dwc3_apple_glue_ops; probe_data.dwc = &appledwc->dwc; probe_data.res = appledwc->mmio_resource; probe_data.ignore_clocks_and_resets = true; @@ -622,6 +777,7 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st dwc3_apple_host_before_core_init(appledwc, state)) dwc3_apple_setup_cio(appledwc); dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); appledwc->core_active = true; appledwc->usb2_power_off_pending = BIT(appledwc->dwc.num_usb2_ports) - 1; appledwc->usb3_power_off_pending = BIT(appledwc->dwc.num_usb3_ports) - 1; @@ -762,6 +918,9 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st } appledwc->state = state; + /* macOS votes USB_SLP_DDR from here until the host's Off, through every sleep. */ + if (state == DWC3_APPLE_HOST && appledwc->sleep_ddr) + apple_sleep_usb_slp_ddr(true); /* * A connected cable leaves the Apple DWC3 core and its PHY as one live, * stateful unit. The glue driver tears that unit down explicitly on a @@ -784,6 +943,152 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st return ret; } +static int dwc3_apple_exit(struct dwc3_apple *appledwc); + +/* A hub that hasn't enumerated by then didn't come back. */ +#define DWC3_APPLE_HUB_CHECK_MS 5000 + +/* Any way a fixed-hub host came out of a sleep, its hub must be there a little later. */ +static void dwc3_apple_check_hub_later(struct dwc3_apple *appledwc) +{ + if (appledwc->fixed_hub) + mod_delayed_work(system_freezable_wq, &appledwc->hub_check, + msecs_to_jiffies(DWC3_APPLE_HUB_CHECK_MS)); +} + +/* + * Bring a controller up again after a sleep that stopped it or lost it. A fixed-hub host + * starts as at probe, where the core has no role yet, so dwc3_core_soft_reset() runs too: + * rebinding the driver brought the J700's hub back where a restart without it didn't. + */ +static int dwc3_apple_start_after_sleep(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +{ + int ret; + + lockdep_assert_held(&appledwc->lock); + if (appledwc->fixed_hub) + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, state); + if (ret) { + dev_err(appledwc->dev, "Failed to start after sleep, err=%d\n", ret); + return ret; + } + dwc3_apple_check_hub_later(appledwc); + return 0; +} + +/* Whether the fixed hub is on its USB2 root port. Takes the xHCI device's lock. */ +static bool dwc3_apple_hub_present(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root; + struct usb_hcd *hcd; + bool present = true; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return true; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + present = usb_hub_find_child(root, appledwc->hub_port); + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return present; +} + +/* Whether @udev or a device below it can wake the system. Takes the hubs' locks. */ +static bool dwc3_apple_can_wake(struct usb_device *udev) +{ + struct usb_device *child; + bool wake = device_may_wakeup(&udev->dev); + int port1; + + if (wake || !udev->maxchild) + return wake; + + usb_lock_device(udev); + usb_hub_for_each_child(udev, port1, child) { + wake = dwc3_apple_can_wake(child); + if (wake) + break; + } + usb_unlock_device(udev); + + return wake; +} + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The hub's own setting doesn't count. Takes the xHCI device's lock and the hubs'. + */ +static unsigned long dwc3_apple_hub_wake_ports(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root, *hub, *child; + unsigned long ports = 0; + struct usb_hcd *hcd; + int port1; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return 0; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + hub = usb_hub_find_child(root, appledwc->hub_port); + if (hub) { + usb_lock_device(hub); + usb_hub_for_each_child(hub, port1, child) { + if (port1 < BITS_PER_LONG && dwc3_apple_can_wake(child)) + ports |= BIT(port1); + } + usb_unlock_device(hub); + } + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return ports; +} + +/* + * Once after a sleep: if the hub isn't there, stop the host and start it again, with the + * hub's link down for a moment first, as unbinding and binding the driver does. This also + * catches an xHCI that didn't come back after the core was set up again. + */ +static void dwc3_apple_hub_check(struct work_struct *work) +{ + struct dwc3_apple *appledwc = container_of(to_delayed_work(work), struct dwc3_apple, + hub_check); + int ret; + + guard(mutex)(&appledwc->lock); + if (appledwc->shutting_down || appledwc->state != DWC3_APPLE_HOST || + dwc3_apple_hub_present(appledwc)) + return; + + dev_warn(appledwc->dev, "hub not back after the sleep: starting the host again\n"); + ret = dwc3_apple_exit(appledwc); + if (ret) { + dev_err(appledwc->dev, "Failed to stop the host again, err=%d\n", ret); + return; + } + msleep(500); + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); + if (ret) + dev_err(appledwc->dev, "Failed to start the host again, err=%d\n", ret); +} + static int dwc3_apple_exit(struct dwc3_apple *appledwc) { lockdep_assert_held(&appledwc->lock); @@ -801,6 +1106,8 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) appledwc->dwc.dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; dwc3_host_exit(&appledwc->dwc); appledwc->state = DWC3_APPLE_NO_CABLE; + if (appledwc->sleep_ddr) + apple_sleep_usb_slp_ddr(false); break; } @@ -821,6 +1128,12 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role ro if (appledwc->fixed_hub) return 0; + /* Stopped for a sleep: this is what comes back after it. */ + if (appledwc->sleep_stopped) { + appledwc->sleep_state = dwc3_apple_state_for_role(role); + return 0; + } + /* * Stay off instead of starting device mode (see dwc3_apple_host_only()). Reporting success * keeps the port controller from retrying: get() then reports USB_ROLE_NONE. @@ -875,7 +1188,7 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) guard(mutex)(&appledwc->lock); - switch (appledwc->state) { + switch (appledwc->sleep_stopped ? appledwc->sleep_state : appledwc->state) { case DWC3_APPLE_HOST: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: @@ -905,11 +1218,209 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } -static bool dwc3_apple_has_fixed_hub(struct device *dev) +/* + * A core that may be unpowered gets one more power cycle, as macOS's Off and cold start, and + * the host starts over once it's back. Until then nothing may touch it, and every sleep is + * refused, as xHCI's suspend would touch it. Returns whether it's back. + */ +static bool dwc3_apple_recover(struct dwc3_apple *appledwc) +{ + int ret; + + lockdep_assert_held(&appledwc->lock); + ret = apple_atcphy_sleep_recover(appledwc->usb2_phy); + if (ret < 0) { + if (!appledwc->sleep_dead) + dev_err(appledwc->dev, + "USB host unpowered after the sleep (%d): USB off, sleeps refused until it's back or a reboot\n", + ret); + appledwc->sleep_dead = true; + return false; + } + if (appledwc->sleep_dead) + dev_info(appledwc->dev, "USB host back after a power cycle: restarting it\n"); + appledwc->sleep_dead = false; + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after the power cycle, err=%d\n", ret); + else + dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + + return true; +} + +/* + * After a sleep in place: the PHY says whether the core came back to D0, and the GUID register + * whether it kept its registers. A core set up again at the resume kept its host and devices, + * which reset-resumed. Otherwise restart the host as after a stop, and the devices enumerate + * again. A core that may be unpowered gets another power cycle first. Returns whether the + * host was kept. + */ +static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 guid = 0; + int ret; + + appledwc->sleep_in_place = false; + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + if (ret == -ENODEV) { + dwc3_apple_recover(appledwc); + return false; + } + if (appledwc->sleep_reinit) { + appledwc->sleep_reinit = false; + dev_dbg(appledwc->dev, "host kept through the sleep, set up again\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + if (!ret || ret == -EBUSY || ret == APPLE_ATCPHY_SLEEP_POWER_LOST) { + guid = dwc3_readl(dwc, DWC3_GUID); + if (guid == DWC3_APPLE_SLEEP_GUID) { + dwc3_writel(dwc, DWC3_GUID, appledwc->sleep_guid); + dev_dbg(appledwc->dev, "host kept through the sleep\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + ret = -EIO; + } + + dev_warn(appledwc->dev, "host lost its state in the sleep (%d, GUID %#x): restarting it\n", + ret, guid); + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after sleep, err=%d\n", ret); + else + dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + + return false; +} + +/* + * Runs before the freezer, when the sleep's real target isn't known yet, so this goes by the + * state "mem" means now. Writing "freeze" with deep selected costs a USB restart and nothing else. + * A stop that fails refuses the sleep. PM_POST_SUSPEND follows an abandoned sleep as well. + */ +static int dwc3_apple_pm_notify_locked(struct dwc3_apple *appledwc, unsigned long action, + bool in_place, bool *finish, bool *kept) +{ + struct device *dev = appledwc->dev; + int ret; + + lockdep_assert_held(&appledwc->lock); + + switch (action) { + case PM_SUSPEND_PREPARE: + /* Any sleep: xHCI's suspend would touch a core that may be unpowered. */ + if (appledwc->sleep_dead && !appledwc->shutting_down && + !dwc3_apple_recover(appledwc)) { + dev_err(dev, "refusing the sleep: the USB host is still unpowered\n"); + return notifier_from_errno(-EIO); + } + if (pm_suspend_default_s2idle() || appledwc->shutting_down) { + *finish = in_place; + *kept = true; + break; + } + /* xHCI suspends and resumes as usual, and the wakeup setting stays. */ + if (in_place) { + appledwc->sleep_guid = dwc3_readl(&appledwc->dwc, DWC3_GUID); + dwc3_writel(&appledwc->dwc, DWC3_GUID, DWC3_APPLE_SLEEP_GUID); + appledwc->sleep_in_place = true; + /* For the port controllers' USB wake. Changes after this go unseen. */ + apple_sleep_usb_set_wake_ports(dwc3_apple_hub_wake_ports(appledwc)); + break; + } + appledwc->sleep_state = appledwc->state; + ret = dwc3_apple_exit(appledwc); + if (ret) + return notifier_from_errno(ret); + /* A wakeup path would keep the domain on, and it's going away. */ + appledwc->sleep_wakeup = device_may_wakeup(dev); + device_set_wakeup_enable(dev, false); + appledwc->sleep_stopped = true; + break; + case PM_POST_SUSPEND: + if (appledwc->sleep_in_place) { + apple_sleep_usb_set_wake_ports(0); + *kept = dwc3_apple_sleep_kept(appledwc); + *finish = true; + break; + } + if (!appledwc->sleep_stopped) + break; + appledwc->sleep_stopped = false; + device_set_wakeup_enable(dev, appledwc->sleep_wakeup); + if (appledwc->shutting_down || + (appledwc->sleep_state != DWC3_APPLE_HOST && + appledwc->sleep_state != DWC3_APPLE_DEVICE)) + break; + dwc3_apple_start_after_sleep(appledwc, appledwc->sleep_state); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +/* + * The PHY's sleep prepare and finish may take the SuperSpeed port down and up, under the + * controller's device lock, so they run outside the glue's lock. + */ +static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct dwc3_apple *appledwc = container_of(nb, struct dwc3_apple, pm_nb); + bool in_place = false, finish = false, kept = false; + int ret; + + /* + * The last wake's SuperSpeed restore ends before this sleep looks at the PHY. The hub + * check takes the glue's lock, and has nothing to look at until the sleep is over. + */ + if (action == PM_SUSPEND_PREPARE) { + apple_atcphy_sleep_finish_flush(appledwc->usb2_phy); + cancel_delayed_work_sync(&appledwc->hub_check); + } + if (action == PM_SUSPEND_PREPARE && !pm_suspend_default_s2idle()) { + scoped_guard(mutex, &appledwc->lock) + in_place = appledwc->fixed_hub && appledwc->state == DWC3_APPLE_HOST && + !appledwc->shutting_down && !appledwc->sleep_dead; + in_place = in_place && !apple_atcphy_sleep_prepare(appledwc->usb2_phy); + } + + scoped_guard(mutex, &appledwc->lock) + ret = dwc3_apple_pm_notify_locked(appledwc, action, in_place, &finish, &kept); + + /* + * After a wake, user space doesn't wait for SuperSpeed: it comes back in the background, + * from the PHY's work, which an unbind of this device doesn't wait for under its lock. + */ + if (finish && action == PM_POST_SUSPEND) + apple_atcphy_sleep_finish_async(appledwc->usb2_phy, kept); + else if (finish) + apple_atcphy_sleep_finish(appledwc->usb2_phy, kept); + + return ret; +} + +static void dwc3_apple_pm_notifier_remove(void *data) +{ + struct dwc3_apple *appledwc = data; + + unregister_pm_notifier(&appledwc->pm_nb); +} + +/* A fixed hub's node gives the root port it sits on in its reg, 1 if it doesn't. */ +static bool dwc3_apple_has_fixed_hub(struct device *dev, u32 *port) { struct device_node *hub __free(device_node) = of_get_available_child_by_name(dev->of_node, "hub"); + *port = 1; + if (hub) + of_property_read_u32(hub, "reg", port); + return hub; } @@ -924,9 +1435,10 @@ static int dwc3_apple_probe(struct platform_device *pdev) return -ENOMEM; appledwc->dev = &pdev->dev; - appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev); + appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev, &appledwc->hub_port); appledwc->host_only = dwc3_apple_host_only(dev); appledwc->dt_dis_u3_susphy = device_property_read_bool(dev, "snps,dis_u3_susphy_quirk"); + appledwc->sleep_ddr = device_property_read_bool(dev, "apple,sleep-ddr"); /* The PHY driver and the DARTs follow runtime PM on these SoCs. */ appledwc->idle_rpm = of_device_is_compatible(dev->of_node, "apple,t6040-dwc3") || of_device_is_compatible(dev->of_node, "apple,t8132-dwc3"); @@ -995,6 +1507,16 @@ static int dwc3_apple_probe(struct platform_device *pdev) * details. */ appledwc->state = DWC3_APPLE_PROBE_PENDING; + ret = devm_delayed_work_autocancel(dev, &appledwc->hub_check, dwc3_apple_hub_check); + if (ret) + return ret; + appledwc->pm_nb.notifier_call = dwc3_apple_pm_notify; + ret = register_pm_notifier(&appledwc->pm_nb); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dwc3_apple_pm_notifier_remove, appledwc); + if (ret) + return ret; /* * Until the first cable there is no core to keep powered: let the controller * runtime-suspend, and its DARTs with it, so that its power domain can drop. @@ -1044,6 +1566,7 @@ static void dwc3_apple_remove(struct platform_device *pdev) guard(mutex)(&appledwc->lock); + appledwc->shutting_down = true; usb_role_switch_unregister(appledwc->role_sw); /* diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 89e1c0c492dbe..652942d3f6f4f 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -79,8 +79,20 @@ static void dwc3_xhci_plat_start(struct usb_hcd *hcd) dwc3_enable_susphy(dwc, true); } +static int dwc3_xhci_plat_resume_quirk(struct usb_hcd *hcd) +{ + struct platform_device *pdev = to_platform_device(hcd->self.controller); + struct dwc3 *dwc = dev_get_drvdata(pdev->dev.parent); + + if (!usb_hcd_is_primary_hcd(hcd) || !dwc->glue_ops || !dwc->glue_ops->pre_host_resume) + return 0; + + return dwc->glue_ops->pre_host_resume(dwc); +} + static const struct xhci_plat_priv dwc3_xhci_plat_quirk = { .plat_start = dwc3_xhci_plat_start, + .resume_quirk = dwc3_xhci_plat_resume_quirk, }; static void dwc3_host_fill_xhci_irq_res(struct dwc3 *dwc, diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index dd1e3ef800e31..5ded59ac0f54d 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -636,6 +636,7 @@ static int xhci_plat_resume_common(struct device *dev, bool power_lost) struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_plat_priv *priv = hcd_to_xhci_priv(hcd); struct xhci_hcd *xhci = hcd_to_xhci(hcd); + bool lost; int ret; if (!device_may_wakeup(dev) && (xhci->quirks & XHCI_SUSPEND_RESUME_CLKS)) { @@ -650,11 +651,18 @@ static int xhci_plat_resume_common(struct device *dev, bool power_lost) } } + /* A positive result: the controller lost its state, which the quirk set up again. */ ret = xhci_priv_resume_quirk(hcd); - if (ret) + if (ret < 0) goto disable_clks; - ret = xhci_resume(xhci, power_lost || priv->power_lost, false); + lost = power_lost || priv->power_lost || ret > 0; + ret = xhci_resume(xhci, lost, false); + /* As after a restore error: a controller the quirk readied starts over if it can't. */ + if (ret && !lost && priv->resume_quirk) { + dev_warn(dev, "xHC state not restored: %d, starting over\n", ret); + ret = xhci_resume(xhci, true, false); + } if (ret) goto disable_clks; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index fed6ea4097662..c49b175e61715 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -1591,9 +1593,183 @@ static void cd321x_prepare(struct tps6598x *tps) cd321x_pm_prepare(&cd321x->pm); cd321x->state_valid = false; } - /* Gate scheduling first; neither worker may be drained under its mutex. */ + /* Gate scheduling first; no worker may be drained under its mutex. */ cancel_delayed_work_sync(&cd321x->resume_work); cancel_delayed_work_sync(&cd321x->update_work); + cancel_delayed_work_sync(&cd321x->rearm_work); + + /* As macOS does a second or so before the commit, measured on the same controller. */ + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) { + int ret; + + guard(mutex)(&tps->lock); + ret = tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S3); + if (ret) + dev_warn(tps->dev, "didn't take system power state S3: %d\n", ret); + else + dev_dbg(tps->dev, "system power state S3\n"); + } +} + +/* A try every 250 ms for 10 s */ +#define CD321X_REARM_TRIES 40 +#define CD321X_REARM_RETRY_MS 250 + +static int cd321x_debug_cmd(struct tps6598x *tps, const char *cmd, size_t len, const u8 *in) +{ + u8 out = 0; + int ret; + + ret = tps6598x_exec_cmd(tps, cmd, len, in, 1, &out); + return ret ?: (out & 0xf ? -EIO : 0); +} + +/* The firmware rejects a command for the mode it is switching to until it has switched. */ +static int cd321x_wait_dbma(struct tps6598x *tps, bool on) +{ + u32 mode; + int ret, err; + + err = read_poll_timeout(tps6598x_read32, ret, + ret || (!memcmp(&mode, modes[TPS_MODE_DBMA], 4) == on), + 1000, 100000, false, tps, TPS_REG_MODE, &mode); + return ret ?: err; +} + +/* + * The boot loader's sequence: unlock with the board's key, debug mode on, enable the debug + * VDM, debug mode off, lock. Debug mode is left and the lock set again after a failure too. + */ +static int cd321x_arm_debug_link(struct tps6598x *tps, const char *key, const char **step) +{ + static const u8 vdm[] = { 0x06, 0x46, 0x82, 0x01 }; + static const u8 no_key[4]; + u8 on = 1, off = 0; + int ret, err; + + *step = "unlock"; + ret = cd321x_debug_cmd(tps, "LOCK", sizeof(no_key), (const u8 *)key); + if (ret) + return ret; + + *step = "debug mode"; + ret = cd321x_debug_cmd(tps, "DBMa", 1, &on); + if (!ret) + ret = cd321x_wait_dbma(tps, true); + if (!ret) { + *step = "debug VDM"; + ret = cd321x_debug_cmd(tps, "DVEn", sizeof(vdm), vdm); + } + + err = cd321x_debug_cmd(tps, "DBMa", 1, &off); + if (!err) + err = cd321x_wait_dbma(tps, false); + if (err && !ret) { + *step = "leaving debug mode"; + ret = err; + } + err = cd321x_debug_cmd(tps, "LOCK", sizeof(no_key), no_key); + if (err && !ret) { + *step = "lock"; + ret = err; + } + return ret; +} + +/* + * A sleep cut this port's debug link. Right after the wake the controller can reject the + * sequence, so a failed try is repeated until the link is back. + */ +static void cd321x_rearm_debug_link(struct cd321x *cd321x) +{ + struct tps6598x *tps = &cd321x->tps; + const char *key, *step; + int ret; + + lockdep_assert_held(&tps->lock); + if (device_property_read_string(tps->dev, "apple,debug-lock-key", &key) || + strlen(key) != 4) { + cd321x->rearm_pending = false; + return; + } + + ret = cd321x_arm_debug_link(tps, key, &step); + cd321x->rearm_tries++; + if (!ret) { + if (cd321x->rearm_tries > 1) + dev_info(tps->dev, "debug link rearmed, try %u\n", cd321x->rearm_tries); + cd321x->rearm_pending = false; + return; + } + if (cd321x->rearm_tries < CD321X_REARM_TRIES) { + mod_delayed_work(system_freezable_wq, &cd321x->rearm_work, + msecs_to_jiffies(CD321X_REARM_RETRY_MS)); + return; + } + dev_warn(tps->dev, "debug link not rearmed after %u tries: %s: %d\n", + cd321x->rearm_tries, step, ret); + cd321x->rearm_pending = false; +} + +/* The cut's cold reset, and how it waits for the contract to come back, see apple_sleep_usb_down(). */ +#define CD321X_REARM_RESET_AFTER 4 +#define CD321X_REARM_RESETS 3 +#define CD321X_RESET_MS 1000 +#define CD321X_CONTRACT_MS 5000 + +static bool cd321x_has_contract(struct tps6598x *tps) +{ + u16 pwr; + + return !tps6598x_read16(tps, TPS_REG_POWER_STATUS, &pwr) && + TPS_POWER_STATUS_CONNECTION(pwr) && + TPS_POWER_STATUS_PWROPMODE(pwr) == TYPEC_PWR_MODE_PD; +} + +/* + * The debug VDM needs a PD contract. After a cut whose sleep was refused, the partner was + * found to have dropped it, both ends at default USB power, and no try could rearm the link. + * Do what the cut does when the contract doesn't come back: reset the controller again and + * wait for it. The controller then needs setting up as after the cut. + */ +static void cd321x_rearm_recover_contract(struct cd321x *cd321x) +{ + struct tps6598x *tps = &cd321x->tps; + int ms, ret; + + lockdep_assert_held(&tps->lock); + cd321x->rearm_resets++; + dev_warn(tps->dev, "debug link: no contract with the partner, reset %u\n", + cd321x->rearm_resets); + ret = tps6598x_exec_cmd_tmo(tps, "Gaid", 0, NULL, 0, NULL, 2000, 0); + if (ret) + dev_dbg(tps->dev, "reset command: %d\n", ret); + msleep(CD321X_RESET_MS); + for (ms = 0; ms < CD321X_CONTRACT_MS && !cd321x_has_contract(tps); ms += 50) + msleep(50); + if (ms < CD321X_CONTRACT_MS) + dev_info(tps->dev, "debug link: contract back %d ms after reset %u\n", + CD321X_RESET_MS + ms, cd321x->rearm_resets); + cd321x->pm.setup_pending = true; + cd321x->reset_by_cut = true; + cd321x->state_valid = false; + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + cd321x->rearm_tries = 0; +} + +static void cd321x_rearm_work(struct work_struct *work) +{ + struct cd321x *cd321x = container_of(to_delayed_work(work), struct cd321x, rearm_work); + struct tps6598x *tps = &cd321x->tps; + + guard(mutex)(&tps->lock); + if (!cd321x->rearm_pending || cd321x->pm.phase == CD321X_PM_PREPARED || + cd321x->pm.phase == CD321X_PM_REMOVED) + return; + if (cd321x->rearm_tries >= CD321X_REARM_RESET_AFTER && + cd321x->rearm_resets < CD321X_REARM_RESETS && !cd321x_has_contract(tps)) + cd321x_rearm_recover_contract(cd321x); + cd321x_rearm_debug_link(cd321x); } static void cd321x_complete(struct tps6598x *tps) @@ -1601,6 +1777,27 @@ static void cd321x_complete(struct tps6598x *tps) struct cd321x *cd321x = container_of(tps, struct cd321x, tps); guard(mutex)(&tps->lock); + /* + * The cut was a cold reset of this controller's firmware: its interrupt mask and power + * state are back at their defaults, so set them up again whether or not an event comes. + */ + if (apple_sleep_debug_link_cut_peek(tps->dev)) { + cd321x->pm.setup_pending = true; + cd321x->reset_by_cut = true; + } + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + /* + * A link still down from an earlier sleep isn't cut again, so it stays pending. A dark + * wake sleeps again at once: the link stays down for it, and the full wake rearms it. + */ + if (apple_sleep_debug_link_was_cut(tps->dev)) + cd321x->rearm_pending = true; + if (cd321x->rearm_pending && !pm_dark_wake()) { + cd321x->rearm_tries = 0; + cd321x->rearm_resets = 0; + cd321x_rearm_debug_link(cd321x); + } if (cd321x_pm_resume(&cd321x->pm)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } @@ -1670,8 +1867,11 @@ static int cd321x_check_setup(struct cd321x *cd321x, u64 event) dev_err(tps->dev, "controller setup recovery exhausted: %d\n", ret); return ret; } - if (ret) + if (ret && cd321x->reset_by_cut) + dev_dbg(tps->dev, "set up again after the debug link's cut\n"); + else if (ret) dev_warn(tps->dev, "restored controller power state and interrupt mask\n"); + cd321x->reset_by_cut = false; /* Only a fresh snapshot may be applied after restoring the controller. */ cd321x->state_valid = false; cd321x->pm.phase = CD321X_PM_REVALIDATE; @@ -2222,6 +2422,7 @@ VISIBLE_IF_KUNIT void cd321x_stop_work(struct cd321x *cd321x) } cancel_delayed_work_sync(&cd321x->resume_work); cancel_delayed_work_sync(&cd321x->update_work); + cancel_delayed_work_sync(&cd321x->rearm_work); } EXPORT_SYMBOL_IF_KUNIT(cd321x_stop_work); @@ -2352,6 +2553,7 @@ VISIBLE_IF_KUNIT void cd321x_init_work(struct cd321x *cd321x) { INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); + INIT_DELAYED_WORK(&cd321x->rearm_work, cd321x_rearm_work); INIT_WORK(&cd321x->tbt_attach_work, cd321x_tbt_attach_work); atomic_set(&cd321x->link_event, 0); atomic_set(&cd321x->ready_event, 0); @@ -3233,6 +3435,87 @@ void tipd_remove(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_remove); +/* + * macOS tells the port controller of a USB root port with dock-remote-wake when that port + * sleeps, and the controller then watches the USB2 lines for a remote wakeup (static; what its + * firmware does with it isn't known). The J700's root port has none, but measured on a J700, + * a USB keyboard on a connector wakes it this way. The mode goes by the speed on the port: 0x09 + * for full and high speed. At the wake the controller hands the lines back before the port + * resumes. Only a firmware-driven sleep needs this, and a failure only logs. + */ +#define TIPD_USBW_HIGH_SPEED 0x09 +#define TIPD_USBW_HAND_BACK 0x80 +#define TIPD_USBW_OFF 0x00 + +static void tipd_usbw(struct tps6598x *tps, u8 mode) +{ + int ret; + + guard(mutex)(&tps->lock); + ret = tps6598x_exec_cmd(tps, "USBw", sizeof(mode), &mode, 0, NULL); + if (ret) + dev_warn(tps->dev, "USBw %#x failed: %d\n", mode, ret); + else + dev_dbg(tps->dev, "USBw %#x\n", mode); +} + +/* The fixed hub's port that the connector's USB2 pair goes to, from the graph, or 0. */ +static u32 tipd_hub_port(struct tps6598x *tps) +{ + struct fwnode_handle *con __free(fwnode_handle) = + device_get_named_child_node(tps->dev, "connector"); + struct fwnode_handle *ep __free(fwnode_handle) = + fwnode_graph_get_endpoint_by_id(con, 0, 0, 0); + struct fwnode_handle *port __free(fwnode_handle) = fwnode_graph_get_remote_port(ep); + u32 reg = 0; + + fwnode_property_read_u32(port, "reg", ®); + return reg; +} + +void tipd_usb_wake(struct tps6598x *tps, bool asleep) +{ + bool armed; + + if (!pm_suspend_via_firmware() || + !device_property_read_bool(tps->dev, "apple,usb-remote-wake")) + return; + if (asleep) { + u32 port = tipd_hub_port(tps); + bool wake = apple_sleep_usb_port_can_wake(port); + + /* + * Only for a device on this connector that can wake. Armed without one, the + * controller reported a USB2 plug at the sleep's entry and the sleep came straight + * back, on an empty port and on one left with a passive adapter (measured on a + * J700). A device plugged in during the sleep still wakes through the SMC's plug + * event. + */ + if (wake) + dev_dbg(tps->dev, "a device on hub port %u can wake: arming USB wake\n", + port); + else + dev_dbg(tps->dev, "nothing on hub port %u can wake: USB wake not armed\n", + port); + scoped_guard(mutex, &tps->lock) + tps->usbw_armed = wake; + if (wake) + tipd_usbw(tps, TIPD_USBW_HIGH_SPEED); + return; + } + /* Out of S3 first: .complete would only come once the port has resumed. */ + scoped_guard(mutex, &tps->lock) { + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + armed = tps->usbw_armed; + tps->usbw_armed = false; + } + if (!armed) + return; + tipd_usbw(tps, TIPD_USBW_HAND_BACK); + tipd_usbw(tps, TIPD_USBW_OFF); +} +EXPORT_SYMBOL_GPL(tipd_usb_wake); + int tipd_suspend(struct tps6598x *tps) { if (tps->wakeup) { diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 30d472c3d83a4..bbe2ef971b35f 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -855,8 +855,11 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) } else { err = sn201202x_wake(sn); } - if (!err) + if (!err) { + /* Registered before xHCI, so this is before the root port resumes. */ + tipd_usb_wake(tps, false); err = tipd_resume(tps); + } /* The resume revalidation acknowledges events latched while asleep. */ if (tps->irq) enable_irq(tps->irq); @@ -906,6 +909,8 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) else dev_warn(dev, "port state unreadable, no wakeup for an attach\n"); } + /* Registered before xHCI, so its root port has suspended by now. */ + tipd_usb_wake(tps, true); reinit_completion(&sn->sleep_completion); err = spmi_command_sleep(sn->sdev); if (err) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 20b6b73436a29..a5512eea4205f 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -395,6 +395,8 @@ struct tps6598x { struct tipd_debugfs *debugfs; struct typec_altmode *partner_altmode_dp; + /* USB remote wake armed on this port for the sleep (USBw). */ + bool usbw_armed; }; /* TPS_REG_USB4_STATUS */ @@ -466,12 +468,20 @@ struct cd321x { atomic_t ready_event; atomic_t restore_event; bool display_route_active; + /* The sleep's debug link cut reset this controller: its setup is expected. */ + bool reset_by_cut; + /* The debug link is still to be rearmed after a cut, and the tries so far */ + bool rearm_pending; + unsigned int rearm_tries; + /* Resets taken since the cut for a contract the partner dropped, see cd321x_rearm_work() */ + unsigned int rearm_resets; bool dp_sink_reported; bool dp_irq_hpd; struct cd321x_status update_status; struct delayed_work update_work; struct delayed_work resume_work; + struct delayed_work rearm_work; struct cd321x_pm_state pm; struct usb_pd_identity cur_partner_identity; bool cur_partner_is_pd; @@ -522,6 +532,7 @@ struct cd321x_sleep_state { }; int cd321x_sleep_events(struct tps6598x *tps, struct cd321x_sleep_state *st); +void tipd_usb_wake(struct tps6598x *tps, bool asleep); int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, size_t in_len, const u8 *in_data, size_t out_len, u8 *out_data, diff --git a/include/linux/irqchip/irq-apple-aic.h b/include/linux/irqchip/irq-apple-aic.h new file mode 100644 index 0000000000000..3c3781322a52b --- /dev/null +++ b/include/linux/irqchip/irq-apple-aic.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_IRQCHIP_IRQ_APPLE_AIC_H +#define _LINUX_IRQCHIP_IRQ_APPLE_AIC_H + +#include + +#ifdef CONFIG_APPLE_AIC +void aic_sleep_mask_all(void); +void aic_sleep_restore(void); +u32 aic_sleep_ack(void); +#else +static inline void aic_sleep_mask_all(void) { } +static inline void aic_sleep_restore(void) { } +static inline u32 aic_sleep_ack(void) { return 0; } +#endif + +#endif diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index 604c54b0b3274..32a33410b4d2f 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -112,6 +112,17 @@ struct apple_smc { struct apple_rtkit *rtk; + /* Between the AP power notices of a firmware-driven suspend. */ + bool ap_notified; + /* Told of a coming sleep, and not yet of its end. */ + bool sleep_warned; + /* To be told at the noirq suspend, after the debug link's cut. */ + bool sleep_warn_late; + /* In a noirq phase: replies are polled for. */ + bool polled; + /* MBSW at the last wake: what the SMC had woken the AP for. */ + u64 wake_record; + struct completion init_done; enum apple_smc_boot_stage boot_stage; diff --git a/include/linux/phy/apple-atc.h b/include/linux/phy/apple-atc.h new file mode 100644 index 0000000000000..7c2e32a3c13fe --- /dev/null +++ b/include/linux/phy/apple-atc.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What the Apple Type-C PHY offers the USB controller it serves, beyond the + * generic PHY operations. + */ +#ifndef _LINUX_PHY_APPLE_ATC_H_ +#define _LINUX_PHY_APPLE_ATC_H_ + +#include +#include + +struct phy; + +/* In D0 after the sleep, but its always-on side lost power, so xHCI's saved state is gone. */ +#define APPLE_ATCPHY_SLEEP_POWER_LOST 1 + +#if IS_REACHABLE(CONFIG_PHY_APPLE_ATC) +/* + * For a firmware-driven system sleep, before any device suspends and without the + * controller's locks held. 0 means the PHY takes the controller through hibernation after + * xHCI has saved its state, then its DARTs and power domains down, and brings all of it + * back before xHCI resumes, so the host can stay up; with USB3 its PIPE is parked for it. + * Otherwise the controller has to be stopped for the sleep. + */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy); + +/* + * After the sleep: 0 if the controller is in D0 and its always-on side kept power, or + * APPLE_ATCPHY_SLEEP_POWER_LOST. -ENODEV if it may be unpowered, so that its registers must + * not be touched. Any other error: it's in D0, but the PHY isn't set up for it. + */ +int apple_atcphy_sleep_result(struct phy *usb2_phy); + +/* + * For a controller that may be unpowered after a sleep: power cycles it, as macOS's Off + * and cold start do. APPLE_ATCPHY_SLEEP_POWER_LOST once it's back, reset, or -ENODEV. + */ +int apple_atcphy_sleep_recover(struct phy *usb2_phy); + +/* + * Ends a sleep that prepare took on, with the SuperSpeed side back as the mux wants it. + * Without @kept the host was restarted; only a failure of the PHY's own then makes the + * controller stop for later sleeps. + */ +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept); + +/* The same from the PHY's work, for after a wake; the flush waits for it. */ +void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept); +void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy); +#else +static inline int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline int apple_atcphy_sleep_recover(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ +} + +static inline void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept) +{ +} + +static inline void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy) +{ +} +#endif + +#endif diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index e594a74e02573..f6f27b4e0e152 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -156,6 +156,12 @@ int apple_dart_restore_tunnel_state(struct device *dev); int apple_dart_radio_check_idle(struct device *dev); /* A firmware-owned buffer's address for a host copy; Linux's own mappings refused. */ int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, phys_addr_t *phys); +/* + * A DART clocked by its host's port: after a real sleep it stays gated until the host has + * set the port up and calls apple_dart_host_resume(). + */ +int apple_dart_set_host_resumes(struct device *dev); +int apple_dart_host_resume(struct device *dev); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } @@ -167,6 +173,68 @@ static inline int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool { return -EOPNOTSUPP; } +static inline int apple_dart_set_host_resumes(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_host_resume(struct device *dev) { return -EOPNOTSUPP; } +#endif + +/* For a consumer whose firmware gates itself around a firmware-driven suspend. */ +#if IS_REACHABLE(CONFIG_APPLE_DART) +bool apple_dart_sleep_listed(struct device *dev); +int apple_dart_sleep_prepare(struct device *dev); +void apple_dart_sleep_restore(struct device *dev); +/* A display DART's streams around a display power cycle, see the DART driver. */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush); +/* A GAPF slice released before its coprocessor's CPU domain comes up after a wake. */ +int apple_dart_gapf_ready(struct device *dev); +/* A display DART whose domain goes off at runtime while others still map into it. */ +void apple_dart_power_dark(struct device *dev, bool dark); +/* A DART's state kept across its consumer's own power cycle, outside a system sleep. */ +int apple_dart_state_save(struct device *dev); +int apple_dart_state_restore(struct device *dev); +void apple_dart_sleep_restore_lost(struct device *dev, bool lost); +#else +static inline bool apple_dart_sleep_listed(struct device *dev) +{ + return false; +} + +static inline int apple_dart_sleep_prepare(struct device *dev) +{ + return -ENODEV; +} + +static inline void apple_dart_sleep_restore(struct device *dev) +{ +} + +static inline int apple_dart_display_streams(struct device *dev, u32 streams, bool on, + bool flush) +{ + return -ENODEV; +} + +static inline int apple_dart_gapf_ready(struct device *dev) +{ + return 0; +} + +static inline void apple_dart_power_dark(struct device *dev, bool dark) +{ +} + +static inline int apple_dart_state_save(struct device *dev) +{ + return -ENODEV; +} + +static inline int apple_dart_state_restore(struct device *dev) +{ + return -ENODEV; +} + +static inline void apple_dart_sleep_restore_lost(struct device *dev, bool lost) +{ +} #endif /* diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index d7fc9dd4134f1..dd6ae8fef53db 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -100,8 +100,15 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); +/* For a caller with interrupts off, where apple_mbox_send()'s own deadline can't pass. */ +bool apple_mbox_send_ready(struct apple_mbox *mbox); int apple_mbox_restore(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* Narrow no-post-callback-MMIO receive; see provider's supported transport guard. */ +bool apple_mbox_poll_one_supported(struct apple_mbox *mbox); +int apple_mbox_poll_one(struct apple_mbox *mbox); +/* Bit 0 of the receive control word, for firmware that powers itself down. */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled); /* * Leave received messages in the hardware FIFO, without an interrupt, until * apple_mbox_release_rx(). For consumers that must not wake the system. @@ -123,6 +130,14 @@ int apple_mbox_ascwrap_v6_get_lifecycle( int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); +/* + * For a CPU that a system sleep's last step stops with interrupts off: its control word, for + * polling its status, and the stop's register writes alone. Once the sleep is over, + * apple_mbox_ascwrap_v6_cpu_stopped() settles the books without touching the registers. + */ +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox); +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, u32 *control, u32 *status); diff --git a/include/linux/soc/apple/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h index 3b2971b8b1db4..3a496a2040989 100644 --- a/include/linux/soc/apple/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -5,16 +5,30 @@ #include #include +struct module; struct pci_dev; #if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) /* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); +/* A function that a sleep left unusable asks for both to be restarted from cold. */ +void apple_piodma_radio_recover(void); +/* What the host calls before it removes Wi-Fi for that. A failure keeps it bound. */ +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner); +#define apple_piodma_radio_set_stop(stop) __apple_piodma_radio_set_stop(stop, THIS_MODULE) #else static inline int apple_piodma_radio_check(struct pci_dev *pdev) { return -ENODEV; } + +static inline void apple_piodma_radio_recover(void) +{ +} + +static inline void apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev)) +{ +} #endif #endif /* _LINUX_SOC_APPLE_PCI_PIODMA_H */ diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 3a942b7990fa6..9cc33b48e334a 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -27,6 +27,11 @@ int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); +void apple_pmp_report_sleep(struct device *pmp, bool at_commit); +void apple_pmp_report_wake(struct device *pmp, bool lost_power); +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable); +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned); +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry); /* * Set or clear one bit of the PMP power request word of the report bound to @report_np, under the * report's lock, with the acknowledge wait (see pmp-report.c). @@ -43,6 +48,31 @@ void apple_pmp_agx_activity_end(void); void apple_pmp_agx_activity_release(void); void apple_pmp_agx_activity_resume(void); #else +static inline int apple_pmp_report_entry_request(const struct device_node *entry, + bool enable) +{ + return 0; +} + +static inline int apple_pmp_report_entry_commit_owned(const struct device_node *entry, + bool owned) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + return 0; +} + +static inline void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ +} + +static inline void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ +} + static inline int apple_pmp_report_ane_available(const struct device_node *np) { return -EOPNOTSUPP; diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 8b329bf5d4dd5..8663ca195febe 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -5,6 +5,7 @@ #include #include struct device; +struct dev_pm_ops; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); @@ -25,8 +26,27 @@ static inline int apple_pmp_t6030_gpu_ready(void) } #endif /* Registration is private to the PMP Rust driver and its C bridge. */ +bool apple_pmp_keep_bound(void); int apple_pmp_register(struct device *dev, const void *data); void apple_pmp_unregister(const void *data); int apple_pmp_send_power_command(const void *data, u8 command, u16 device_id, u32 enabled); +extern const struct dev_pm_ops apple_pmp_pm_ops; +int apple_pmp_sleep(const void *data); +int apple_pmp_wake(const void *data); + +/* + * What the sleep's last step needs to idle and stop the PMP with interrupts off: its RTKit and + * CPU control, and where the SRAM it keeps goes (no range: sram NULL). Filled in suspend_late. + */ +struct apple_rtkit; +struct apple_pmp_commit_stop { + struct apple_rtkit *rtk; + void __iomem *control; + void __iomem *sram; + u32 *words; + size_t count; +}; + +int apple_pmp_sleep_prepare(const void *data, struct apple_pmp_commit_stop *stop); #endif diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index ac646240f8587..44dcb386efd76 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -215,6 +215,9 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* apple_rtkit_boot(), calling @before_ap between the IOP's report and the AP going on. */ +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data); + /* * Mark an instance created with apple_rtkit_init() as running without a * handshake, see apple_rtkit_init_adopted(). @@ -259,6 +262,92 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk); */ int apple_rtkit_idle(struct apple_rtkit *rtk); +/* + * Turn the mailbox off for a co-processor that has acknowledged idle, before its + * run bit and power state go. Only apple_rtkit_warm_restart() follows this. + */ +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk); + +/* + * The same with interrupts off and one CPU up, for a co-processor that doesn't come + * back from idle without losing power, so that its idle has to follow every refusal. + * @us is about how long the two acknowledgements took. Only + * apple_rtkit_warm_restart() follows this, once the sleep has reached the hardware. + */ +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us); + +/* + * Only the mailbox's receive bit, for the moment before that co-processor's run bit + * goes. Safe with interrupts off, and apple_rtkit_warm_restart() sets it again. + */ +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk); + +/* + * Prepare for a firmware being restarted with its buffers kept. + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk); + +/* + * For a firmware whose last endpoint map is answered, on a warm restart, with + * only the endpoints in @last_mask rather than an echo of all of them. + */ +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask); + +/* + * Tell a co-processor that keeps running across a system sleep about the AP's + * power state, without changing its own: idle before the sleep and on after. + */ +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on); + +/* The same, for a noirq phase: device interrupts are off, so it polls. */ +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on); + +/* + * Stop a co-processor for a system sleep that may remove its power: idle, wait + * for the status that shows it has finished, then clear the run bit. On an error + * the run bit is still set and the firmware still runs. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Bring back a co-processor taken down for a system sleep with + * apple_rtkit_sleep_stop() or apple_rtkit_idle(), from a process context with its + * mailbox's supplier resumed. + * + * Where the sleep reached the hardware (pm_resume_via_firmware()) the firmware + * restarts and keeps its buffers and endpoints: this restarts the receive path, sets + * the run bit, echoes the endpoint map and takes the IOP state it reports unasked, + * then sets the AP state. Where it didn't, one that idled gets -EAGAIN: an idled + * firmware doesn't wake with its power kept, so it stays down, and comes back the + * same way after the next sleep that does reach the hardware. One whose idle + * failed still runs, and gets 0 once its AP state is on again. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * A start as after a real sleep, for a co-processor its driver power-cycled without one: + * the warm restart, the run bit set, then RTKit's handshake. + */ +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * The first half of apple_rtkit_sleep_return() after a real sleep: the restart and the run + * bit, with no wait. apple_rtkit_boot() then waits for the firmware to report on, so other + * firmware can be started in between. -EOPNOTSUPP after a sleep that kept the power. + */ +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Echo each OSLOG channel-0 counter the firmware sends, as macOS does. On by default on + * T8140, where a firmware with one outstanding takes no AP power notice. + */ +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo); + /* * Checks if RTKit is running and ready to handle messages. */ @@ -301,6 +390,21 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * A port that only answers the co-processor, polled, with no interrupts. Map answers + * its HELLO and endpoint map, and on brings the AP side of the port up. Handshake is + * both. Idle takes the AP side down. Restart prepares for a co-processor that was + * stopped. + */ +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_restart(struct apple_rtkit *rtk); + +/* apple_rtkit_idle(), calling @after_ap between the AP's idle and the IOP's. */ +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data); + /* * Stop taking interrupts for messages from the co-processor, which then * stay queued in the mailbox until apple_rtkit_resume_rx(). diff --git a/include/linux/soc/apple/sart.h b/include/linux/soc/apple/sart.h index f5d0db5022831..53186e759e55d 100644 --- a/include/linux/soc/apple/sart.h +++ b/include/linux/soc/apple/sart.h @@ -64,4 +64,10 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, size_t size); +/* + * After a firmware-driven sleep, writes the entries in use before it back again and reads + * each back. Returns how many didn't hold, or -EOPNOTSUPP. + */ +int apple_sart_replay(struct apple_sart *sart); + #endif /* _LINUX_SOC_APPLE_SART_H_ */ diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h new file mode 100644 index 0000000000000..6ce00ba6001d9 --- /dev/null +++ b/include/linux/soc/apple/sleep.h @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What drivers tell the code that takes an Apple SoC into a firmware-driven + * suspend. + */ +#ifndef _LINUX_SOC_APPLE_SLEEP_H_ +#define _LINUX_SOC_APPLE_SLEEP_H_ + +#include +#include +#include +#include + +struct device; +struct device_node; + +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) +/* + * A coprocessor that can't be stopped until a sleep is certain, and can't be + * restarted without one. While one is registered and not ready, a suspend is + * refused. Once ready, the last step before the commit runs @last_stop, with + * interrupts off and every refusal behind it, then clears its run bit. With + * @pcie_ans, that also lowers the PCIe and ANS domains, which nothing else may do + * under it. @last_stop can't refuse: a nonzero return means it failed, and the sleep + * goes on. + */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data); + +/* + * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power + * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. + */ +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data); +void apple_sleep_coproc_unregister(struct device *dev); + +/* + * During a suspend: a driver left something the DRAM-kept wake would stall on, so this + * sleep takes the deep path. False if no sleep driver can do that. + */ +bool apple_sleep_request_deep_once(void); + +/* + * For a registered coprocessor: the last of its stop that may still refuse the sleep, run + * with interrupts off before any coprocessor's @last_stop. A negative return refuses, and the + * coprocessor must still be able to carry on. + */ +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)); + +/* + * The enclave's irreversible sleep request. @stop runs with interrupts off, + * after every refusal and before other irreversible coprocessor stops. Nonzero + * may refuse ONLY if no request was sent. Once sent, protocol failure is fatal; + * success must continue through physical sleep/wake, even for a newly pending wake. + * Unsupported/no-provider paths refuse before sending, never at device suspend. + */ +int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data); +void apple_sleep_enclave_unregister(struct device *dev); + +/* Its driver has quiesced it for that last step, or has taken it back. */ +void apple_sleep_coproc_set_ready(struct device *dev, bool ready); + +/* Whether the last step stopped it: after a refused sleep, false means it still runs. */ +bool apple_sleep_coproc_stopped(struct device *dev); + +/* + * Whether this sleep's commit stops the registered coprocessors. Then one that can't be + * registered refuses the sleep: stopped any earlier, a later refusal would strand it. + */ +bool apple_sleep_commit_stops_coprocs(void); + +/* + * The USB host sleeps in place for this sleep, as macOS's does: its always-on domain stays + * up through the commit. Its PHY sets it once the controller is in its hibernation state, + * and clears it at the wake. + */ +void apple_sleep_usb_set_asleep(bool asleep); + +/* + * The USB host is up, as macOS's USB_SLP_DDR vote has it: from the host's start to its stop, + * awake and asleep. It sets bit 0 of the wake block's word 0 for every sleep in between. + */ +void apple_sleep_usb_slp_ddr(bool up); + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The host that sleeps in place sets them before the freeze and clears them after. + */ +void apple_sleep_usb_set_wake_ports(unsigned long ports); + +/* + * For a driver that takes its own power domain down for a sleep and up again, as macOS's USB + * host does: @pd is the domain's power controller node. Lowering waits until it reads off. + * 0, -ETIMEDOUT, -ENODEV without the sleep driver, or -EINVAL for another node, which leave + * the domain alone and which apple_sleep_domain_check() reports beforehand. + */ +int apple_sleep_domain_set(struct device_node *pd, bool on); +int apple_sleep_domain_check(struct device_node *pd); + +/* For the PMGR's driver at the wake: a driver took @pd down and brings it up itself. */ +bool apple_sleep_domain_is_down(struct device_node *pd); + +/* For the port controller wired to the fixed hub's @port, as it arms USB wake. */ +bool apple_sleep_usb_port_can_wake(unsigned int port); + +/* The last sleep ended the debug link on @dev, whose driver may rearm it. */ +bool apple_sleep_debug_link_was_cut(struct device *dev); + +/* The same, without taking it: the port controller's own setup after its cold reset. */ +bool apple_sleep_debug_link_cut_peek(struct device *dev); + +/* + * End the debug link now, if this sleep hasn't: true if this call did. For the SMC's driver, + * ahead of its AP idle notice. It takes over a second, and doesn't sleep. + */ +bool apple_sleep_debug_link_cut(void); + +/* This sleep will end the debug link: for the SMC's driver, which warns the SMC after it. */ +bool apple_sleep_debug_link_up(void); + +/* + * From the SMC's driver, in its noirq resume: the SMC woke the AP for the lid alone, and the + * lid reads closed. If nothing else woke the SoC, the PM core goes back to sleep once every + * device has resumed, as macOS does from a dark wake. + */ +void apple_sleep_lid_closed_wake(void); + +/* + * From the SMC's driver, in its noirq resume: the SMC woke the AP for the port alone. If + * this sleep cut the debug link and nothing else woke the SoC, it's a dark wake. + */ +void apple_sleep_debug_link_cut_wake(void); + +/* The radio's host has its port back after a real sleep, so the next commit may stop it. */ +void apple_sleep_radio_port_up(void); + +/* From RTKit: this coprocessor's firmware crashed. */ +void apple_sleep_coproc_crashed(struct device *dev); + +/* After a deep sleep, put back the pins it reset. ANS's driver calls it before the run bit. */ +void apple_sleep_restore_pins(void); + +/* + * After a refused sleep, for a firmware that idled on a domain that can't gate alone: the + * sleep engine's PMGR reset of its own domains, its DARTs kept. The caller starts it after. + */ +int apple_sleep_engine_reset(struct device *dev); +#else +static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, + bool pcie_ans, int (*last_stop)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_coproc_register_first(struct device *dev, + int (*last_stop)(void *data), void *data) +{ + return -EOPNOTSUPP; +} + +static inline bool apple_sleep_request_deep_once(void) +{ + return false; +} + +static inline void apple_sleep_coproc_unregister(struct device *dev) +{ +} + +static inline int apple_sleep_coproc_set_last_check(struct device *dev, + int (*last_check)(void *data)) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_enclave_unregister(struct device *dev) +{ +} + +static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ +} + +static inline bool apple_sleep_coproc_stopped(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_commit_stops_coprocs(void) +{ + return false; +} + +static inline void apple_sleep_usb_set_asleep(bool asleep) +{ +} + +static inline void apple_sleep_usb_slp_ddr(bool up) +{ +} + +static inline void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ +} + +static inline int apple_sleep_domain_set(struct device_node *pd, bool on) +{ + return -ENODEV; +} + +static inline int apple_sleep_domain_check(struct device_node *pd) +{ + return -ENODEV; +} + +static inline bool apple_sleep_domain_is_down(struct device_node *pd) +{ + return false; +} + +static inline bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return false; +} + +static inline bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut(void) +{ + return false; +} + +static inline bool apple_sleep_debug_link_up(void) +{ + return false; +} + +static inline void apple_sleep_lid_closed_wake(void) +{ +} + +static inline void apple_sleep_debug_link_cut_wake(void) +{ +} + +static inline void apple_sleep_radio_port_up(void) +{ +} + +static inline void apple_sleep_coproc_crashed(struct device *dev) +{ +} + +static inline void apple_sleep_restore_pins(void) +{ +} + +static inline int apple_sleep_engine_reset(struct device *dev) +{ + return -EOPNOTSUPP; +} +#endif + +#endif diff --git a/include/linux/suspend.h b/include/linux/suspend.h index b02876f1ae38a..365437ce2cc02 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -159,6 +159,7 @@ extern unsigned int pm_suspend_global_flags; #define PM_SUSPEND_FLAG_FW_SUSPEND BIT(0) #define PM_SUSPEND_FLAG_FW_RESUME BIT(1) #define PM_SUSPEND_FLAG_NO_PLATFORM BIT(2) +#define PM_SUSPEND_FLAG_DARK_WAKE BIT(3) static inline void pm_suspend_clear_flags(void) { @@ -180,6 +181,11 @@ static inline void pm_set_suspend_no_platform(void) pm_suspend_global_flags |= PM_SUSPEND_FLAG_NO_PLATFORM; } +static inline void pm_set_dark_wake(void) +{ + pm_suspend_global_flags |= PM_SUSPEND_FLAG_DARK_WAKE; +} + /** * pm_suspend_via_firmware - Check if platform firmware will suspend the system. * @@ -237,6 +243,27 @@ static inline bool pm_suspend_no_platform(void) return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_NO_PLATFORM); } +/** + * pm_dark_wake - Check if the system woke up for nothing user space has to see. + * + * To be called during system-wide power management transitions from sleep + * states. + * + * Return 'true' if the platform found that the event that woke up the system + * needs nothing from user space (for example, a laptop lid closing while the + * system slept). Every device then resumes as usual, and the system goes back + * to the same sleep state with user space still frozen. A device should not do + * what the user would notice, such as lighting a display, during such a wake. + * + * Only a wake from the sleep state can be dark. A device that had to leave out + * part of its resume finishes it in a later resume that is not dark, or in + * ->complete() if the next suspend fails before suspending it. + */ +static inline bool pm_dark_wake(void) +{ + return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_DARK_WAKE); +} + /* Suspend-to-idle state machnine. */ enum s2idle_states { S2IDLE_STATE_NONE, /* Not suspended/suspending. */ @@ -287,6 +314,8 @@ static inline void pm_set_resume_via_firmware(void) {} static inline bool pm_suspend_via_firmware(void) { return false; } static inline bool pm_resume_via_firmware(void) { return false; } static inline bool pm_suspend_no_platform(void) { return false; } +static inline void pm_set_dark_wake(void) {} +static inline bool pm_dark_wake(void) { return false; } static inline bool pm_suspend_default_s2idle(void) { return false; } static inline void suspend_set_ops(const struct platform_suspend_ops *ops) {} diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 604b9dda94317..b8bf424370bc7 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -112,6 +112,7 @@ struct usb_hcd { #define HCD_FLAG_DEAD 6 /* controller has died? */ #define HCD_FLAG_INTF_AUTHORIZED 7 /* authorize interfaces? */ #define HCD_FLAG_DEFER_RH_REGISTER 8 /* Defer roothub registration */ +#define HCD_FLAG_PHY_SUSPENDED 9 /* root hub PHYs released by a suspend */ /* The flags can be tested using these macros; they are likely to * be slightly faster than test_bit(). diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 57c44268698f7..34253036bbbd9 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -532,6 +532,9 @@ int suspend_devices_and_enter(suspend_state_t state) do { error = suspend_enter(state, &wakeup); } while (!error && !wakeup && platform_suspend_again(state)); + /* Only a wake from the sleep state can be dark. */ + if (error) + pm_suspend_global_flags &= ~PM_SUSPEND_FLAG_DARK_WAKE; Resume_devices: suspend_test_start(); @@ -616,6 +619,15 @@ static int enter_state(suspend_state_t state) trace_suspend_resume(TPS("suspend_enter"), state, false); pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); error = suspend_devices_and_enter(state); + /* + * Every device has resumed, but nothing woke the system that user space + * needs to see: go back to sleep with user space still frozen. + */ + while (!error && pm_dark_wake()) { + pr_info("dark wake: suspending again (%s)\n", mem_sleep_labels[state]); + pm_suspend_clear_flags(); + error = suspend_devices_and_enter(state); + } Finish: events_check_enabled = false; diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f763d5541472..389c67d5740ed 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -102,6 +102,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "suspend.c" #include "srcu.c" #include "sync.c" #include "task.c" diff --git a/rust/helpers/suspend.c b/rust/helpers/suspend.c new file mode 100644 index 0000000000000..a616907ecc0b5 --- /dev/null +++ b/rust/helpers/suspend.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_pm_suspend_via_firmware(void) +{ + return pm_suspend_via_firmware(); +} + +__rust_helper bool rust_helper_pm_resume_via_firmware(void) +{ + return pm_resume_via_firmware(); +} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 1f845937899db..7bce2fe29e841 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -80,6 +80,7 @@ unsafe impl driver::RegistrationOps for Adapter { (*pdrv.get()).driver.of_match_table = of_table; (*pdrv.get()).driver.acpi_match_table = acpi_table; (*pdrv.get()).driver.suppress_bind_attrs = T::suppress_bind_attrs(); + (*pdrv.get()).driver.pm = T::pm_ops().map_or(core::ptr::null(), core::ptr::from_ref); } // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. @@ -230,6 +231,11 @@ pub trait Driver { false } + /// The driver's power management callbacks, if it has any. + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + None + } + /// Platform driver probe. /// /// Called when a new platform device is added or discovered. diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index c4f86930e3c65..e0c4be586801d 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -91,11 +91,34 @@ impl Mailbox { unsafe { device::Device::from_raw((*self.mbox).dev) } } + /// The C mailbox, for code that drives it with interrupts off. + pub fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + /// Sends the specified message pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { // SAFETY: Calling the c function, `mbox` is a valid pointer to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) } + + /// Sets bit 0 of the receive control word, for firmware that powers itself down. + pub fn set_recv_enabled(&self, enabled: bool) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_set_recv_enabled(self.mbox, enabled) }; + } + + /// Stops taking receive interrupts, so that no handler reads the mailbox. + pub fn hold_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_hold_rx(self.mbox) }; + } + + /// Undoes [`Self::hold_rx`]. + pub fn release_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_release_rx(self.mbox) }; + } } impl Drop for Mailbox { diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 1524127cd475a..c34f187f4fb10 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -373,6 +373,26 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) } + /// Asks the coprocessor to go idle, AP then IOP state, and waits for both acknowledgments. + pub fn idle(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_idle(self.rtk) }) + } + + /// Turns the mailbox off for a coprocessor that has acknowledged idle, before its run bit and + /// power state go. Only [`Self::warm_restart`] follows this. + pub fn mailbox_off(self: Pin<&mut Self>) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_mailbox_off(self.rtk) }; + } + + /// Prepares for HELLO from firmware restarting warm, which keeps its buffers and rings. The + /// caller then sets the run bit and calls [`Self::boot`]. + pub fn warm_restart(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_warm_restart(self.rtk) }) + } + /// Starts a non-system endpoint. pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { // SAFETY: `rtk` is valid per the type invariant. @@ -399,6 +419,12 @@ impl RtKit { unsafe { bindings::apple_rtkit_is_crashed(self.rtk) } } + /// Echoes the OSLOG counters the firmware sends, or drops them. + pub fn set_oslog_echo(&self, echo: bool) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_set_oslog_echo(self.rtk, echo) }; + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/scripts/ci/apple-dwc3-atc-compile.sh b/scripts/ci/apple-dwc3-atc-compile.sh new file mode 100755 index 0000000000000..0d769c907f213 --- /dev/null +++ b/scripts/ci/apple-dwc3-atc-compile.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0 +# Compile the real Apple DWC3 consumer with ATC's unreachable-provider stubs. +set -euo pipefail + +if [[ $# != 1 || -z ${RUNNER_TEMP:-} ]]; then + echo "usage: RUNNER_TEMP= $0 " >&2 + exit 2 +fi +profile=$1 +case "$profile" in + builtin-no-atc) dwc3=y; atc=n ;; + modular-no-atc) dwc3=m; atc=n ;; + builtin-modular-atc) dwc3=y; atc=m ;; + *) echo "unknown DWC3/ATC profile: $profile" >&2; exit 2 ;; +esac + +source_dir=$(git rev-parse --show-toplevel) +cd "$source_dir" +work="$RUNNER_TEMP/dwc3-atc-$profile" +# Refuse reused output: a stale object must never satisfy this compile gate. +mkdir "$work" +output="$work/output" +artifacts="$work/artifacts" +mkdir "$output" "$artifacts" +fragment="$artifacts/requested.config" +compile_exit=NOT_RUN +stage=configuration + +finish() +{ + local rc=$? + trap - EXIT + { + printf 'last_stage=%s\ncompile_exit=%s\nscript_exit=%s\n' "$stage" "$compile_exit" "$rc" + if [[ -f $output/.config ]]; then + cp "$output/.config" "$artifacts/resolved.config" + sha256sum "$artifacts/resolved.config" + fi + } | tee -a "$artifacts/receipt.txt" + exit "$rc" +} +trap finish EXIT + +{ + printf 'profile=%s\nsource_commit=%s\nsource_tree=%s\n' "$profile" \ + "$(git rev-parse HEAD)" "$(git rev-parse HEAD^{tree})" + printf 'github_sha=%s\npr_head=%s\npr_base=%s\n' "${GITHUB_SHA:-unset}" \ + "${PR_HEAD_SHA:-unset}" "${PR_BASE_SHA:-unset}" + printf 'requested_USB_DWC3=%s\nrequested_USB_DWC3_APPLE=%s\nrequested_PHY_APPLE_ATC=%s\n' \ + "$dwc3" "$dwc3" "$atc" + clang --version + ld.lld --version +} | tee "$artifacts/receipt.txt" + +cat > "$fragment" <> "$fragment" +else + printf 'CONFIG_PHY_APPLE_ATC=%s\n' "$atc" >> "$fragment" +fi +sha256sum "$fragment" | tee -a "$artifacts/receipt.txt" + +export KCONFIG_CONFIG="$output/.config" +kmake=(make -C "$source_dir" O="$output" ARCH=arm64 LLVM=1) +"${kmake[@]}" defconfig 2>&1 | tee "$artifacts/defconfig.log" +scripts/kconfig/merge_config.sh -m -O "$output" "$output/.config" "$fragment" \ + 2>&1 | tee "$artifacts/merge-config.log" +"${kmake[@]}" olddefconfig 2>&1 | tee "$artifacts/olddefconfig.log" + +assert_config() +{ + local symbol=$1 expected=$2 actual + actual=$(scripts/config --file "$output/.config" --state "$symbol") + printf 'resolved_%s=%s\n' "$symbol" "$actual" | tee -a "$artifacts/receipt.txt" + if [[ $actual != "$expected" ]]; then + echo "$symbol resolved to $actual; expected $expected" >&2 + exit 1 + fi +} +# Assert the legal configuration after Kconfig resolution, before invoking CC. +assert_config USB_DWC3 "$dwc3" +assert_config USB_DWC3_APPLE "$dwc3" +assert_config PHY_APPLE_ATC "$atc" +for symbol in MODULES ARCH_APPLE OF PM USB_SUPPORT USB USB_GADGET USB_XHCI_HCD \ + TYPEC GENERIC_PHY USB_DWC3_DUAL_ROLE; do + assert_config "$symbol" y +done +cp "$output/.config" "$artifacts/resolved.config" +sha256sum "$artifacts/resolved.config" | tee -a "$artifacts/receipt.txt" + +target=drivers/usb/dwc3/dwc3-apple.o +command=("${kmake[@]}" -j2 V=1 "$target") +printf '%q ' "${command[@]}" | tee "$artifacts/compile-command.txt" +printf '\n' | tee -a "$artifacts/compile-command.txt" +stage=compile +set +e +"${command[@]}" 2>&1 | tee "$artifacts/compile.log" +pipeline_status=("${PIPESTATUS[@]}") +set -e +compile_exit=${pipeline_status[0]} +if [[ $compile_exit != 0 ]]; then + exit "$compile_exit" +fi +if [[ ${pipeline_status[1]} != 0 ]]; then + exit "${pipeline_status[1]}" +fi +stage=object-verification +test -s "$output/$target" +sha256sum "$output/$target" | tee -a "$artifacts/receipt.txt" +cp "$output/drivers/usb/dwc3/.dwc3-apple.o.cmd" "$artifacts/kbuild-command.txt" +stage=complete diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs index d6662ecbacf81..113b0ea6db869 100644 --- a/sound/soc/apple/aop_audio.rs +++ b/sound/soc/apple/aop_audio.rs @@ -1108,6 +1108,36 @@ kernel::of_device_table!( ] ); +/// A system sleep with a firmware behind it. The firmware keeps running, so what it runs for +/// the streams has to stop first. Other variants' firmware isn't traced around a sleep. +/// +/// # Safety +/// +/// Called by the PM core with a live device of this driver. +unsafe extern "C" fn aop_audio_suspend(dev: *mut bindings::device) -> i32 { + // SAFETY: the helper only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: the PM core passes a live device, and keeps unbind away for this call. + let dev = unsafe { device::Device::::from_raw(dev) }; + let driver = dev.get_drvdata::(); + if driver.is_null() { + return 0; + } + // SAFETY: this driver stored it at probe, and it stays until unbind. + if let AopAudio::T8140(driver) = unsafe { &*driver } { + driver.sleep_stop(); + } + 0 +} + +static AOP_AUDIO_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend: Some(aop_audio_suspend), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + /// The bound driver, as the variant set it up. enum AopAudio { /// Held for its `Drop`, which frees the card. @@ -1121,6 +1151,10 @@ impl platform::Driver for AopAudio { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_AUDIO_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/sound/soc/apple/aop_t8140.rs b/sound/soc/apple/aop_t8140.rs index aa97deda3b5c3..751cde0c4eb8e 100644 --- a/sound/soc/apple/aop_t8140.rs +++ b/sound/soc/apple/aop_t8140.rs @@ -62,6 +62,10 @@ use kernel::{ new_spinlock, platform, prelude::*, + regulator::{ + self, + Regulator, // + }, soc::apple::aop::{ from_fourcc, EPICService, @@ -492,6 +496,10 @@ struct SndSocT8140AopData { spkr_want_run: AtomicFlag, /// A microphone-array stream has been triggered and not yet stopped. hpai_want_run: AtomicFlag, + /// The array's rail (ADT function-input_power_enable), held while its + /// service runs. + #[pin] + hpai_supply: Mutex>>, /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. spkr_volume: Atomic, /// REG_SPKR_FULL_SCALE: the volume is below SPKR_CEILING_Q31, not the @@ -958,7 +966,11 @@ impl SndSocT8140AopData { if !self.hpai_want_run.load(Relaxed) || self.hpai.running.load(Relaxed) { return Ok(()); } - self.service_run(&self.hpai) + // As macOS does: the rail, then the stream. A failed start drops it again. + let supply = Regulator::::get(&self.dev, c"mic")?; + self.service_run(&self.hpai)?; + *self.hpai_supply.lock() = Some(supply); + Ok(()) } /// Called from sleeping close paths: a queued start must finish before @@ -972,7 +984,9 @@ impl SndSocT8140AopData { fn hpai_shutdown(&self) { self.flush_hpai_start(); - self.service_shutdown(&self.hpai, &self.hpai_domains()) + self.service_shutdown(&self.hpai, &self.hpai_domains()); + // As macOS does: the stream's stop, then the rail. + *self.hpai_supply.lock() = None; } // ---- internal speakers: the spkr service on LEAP TX0 (leap-ns ADMAC @@ -1037,6 +1051,35 @@ impl SndSocT8140AopData { Ok(()) } + /// `pw0 ` if running; the leaves stay with the stream. + fn service_stop(&self, svc: &Service) { + if svc.running.load(Relaxed) { + if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { + dev_err!(self.dev, "{}: pw0 failed: {:?}\n", svc.name, e); + } + svc.running.store(false, Relaxed); + } + } + + /// A system sleep: nothing the firmware runs may outlast the streams, whose DMA stops with + /// the sleep. Each one's next prepare or start asks for it again. + fn sleep_stop(&self) { + if self.powered.xchg(false, Relaxed) { + if let Err(e) = self.lpai_run_put() { + dev_err!(self.dev, "unable to return lpai to idle: {:?}\n", e); + } + } + self.flush_hpai_start(); + self.service_stop(&self.hpai); + *self.hpai_supply.lock() = None; + if let Err(e) = self.spkr_prepare() { + dev_err!(self.dev, "spkr: stop failed: {:?}\n", e); + } + self.service_stop(&self.tap); + self.service_stop(&self.cout); + self.service_stop(&self.cin); + } + /// Called from sleeping prepare/shutdown paths while ALSA serializes /// this stream: a queued start must finish before the mute and pw0. fn flush_spkr_start(&self) { @@ -1571,6 +1614,9 @@ fn pcm_open_dmaengine( if hw.formats == 0 { return EINVAL.to_errno(); } + // A system sleep stops the services, and only a prepare starts them all again. Offer + // neither resume nor pause: a transfer paused by the suspend would block the next start. + hw.info &= !(bindings::SNDRV_PCM_INFO_PAUSE | bindings::SNDRV_PCM_INFO_RESUME); // SAFETY: the substream's runtime exists for the op's duration; the // dmaengine open attaches `chan` to it on success. unsafe { @@ -2609,6 +2655,8 @@ impl AsocContext { pub(crate) struct SndSocT8140AopDriver { card: *mut bindings::snd_card, + /// The card's two microphone PCMs, null until created. + pcms: [*mut bindings::snd_pcm; 2], data: Arc, asoc: KBox, asoc_registered: bool, @@ -2664,6 +2712,7 @@ impl SndSocT8140AopDriver { // whatever was set up. let mut this = SndSocT8140AopDriver { card: ptr::null_mut(), + pcms: [ptr::null_mut(); 2], asoc: AsocContext::new(data.clone())?, data: data.clone(), asoc_registered: false, @@ -2777,6 +2826,7 @@ impl SndSocT8140AopDriver { (*pcm).private_free = Some(pcm_free_private); (*pcm).info_flags = 0; } + self.pcms[device as usize] = pcm; // SAFETY: as above. copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; if device == 0 { @@ -2797,6 +2847,21 @@ impl SndSocT8140AopDriver { Ok(()) } + /// A system sleep: stop the card's own streams, then whatever the firmware runs for them + /// and for the sound card's. Userspace prepares its streams again afterwards, since none + /// of them can resume. + pub(crate) fn sleep_stop(&self) { + for pcm in self.pcms.iter().copied().filter(|pcm| !pcm.is_null()) { + // SAFETY: the PCMs live as long as the card, and the driver core keeps + // unbind away from a system sleep. + let ret = unsafe { bindings::snd_pcm_suspend_all(pcm) }; + if ret < 0 { + dev_err!(self.data.dev, "unable to suspend a PCM: {}\n", ret); + } + } + self.data.sleep_stop(); + } + /// Undo probe: stop the reports, unregister the ASoC component and free /// the card (which closes every stream, releasing its channel and power /// references), then return lpai to idle. Runs from unbind, and again @@ -2889,6 +2954,9 @@ impl SndSocT8140AopDriver { // SAFETY: the pointer was checked above and stays valid with the child. let svc = unsafe { *service }; + // A capture takes the array's rail, which needs the SMC's GPIOs bound. + Regulator::::get(&dev, c"mic")?; + // The source ring: 512 KiB through our own IOMMU stream (dart-aop 9), // owned by the AOP driver because the firmware binds it once per boot. // SAFETY: no DMA is in flight on this device yet. @@ -2940,6 +3008,7 @@ impl SndSocT8140AopDriver { pd_rx0, spkr_want_run: AtomicFlag::new(false), hpai_want_run: AtomicFlag::new(false), + hpai_supply <- new_mutex!(None), spkr_volume: Atomic::new(SPKR_VOLUME_MAX), spkr_full_scale: AtomicFlag::new(false), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 18d73b50af861..9dfb329f46201 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -1467,10 +1468,41 @@ static const struct of_device_id apple_mca_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_mca_of_match); +/* + * hw_params is the only place a stream's cluster and serdes setup is written. A + * sleep that takes the registers would leave an open stream silent, and nothing + * here restores them yet. + */ +static int __maybe_unused apple_mca_suspend_late(struct device *dev) +{ + struct mca_data *mca = dev_get_drvdata(dev); + int i; + + if (!pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < mca->nclusters; i++) { + struct mca_cluster *cl = &mca->clusters[i]; + + if (cl->pd_link || cl->clocks_in_use[SNDRV_PCM_STREAM_PLAYBACK] || + cl->clocks_in_use[SNDRV_PCM_STREAM_CAPTURE]) { + dev_err(dev, "cluster %d is in use, refusing the sleep\n", i); + return -EBUSY; + } + } + + return 0; +} + +static const struct dev_pm_ops apple_mca_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_mca_suspend_late, NULL) +}; + static struct platform_driver apple_mca_driver = { .driver = { .name = "apple-mca", .of_match_table = apple_mca_of_match, + .pm = pm_sleep_ptr(&apple_mca_pm_ops), }, .probe = apple_mca_probe, .remove = apple_mca_remove, diff --git a/tools/asahi/aop-bootargs-tests.py b/tools/asahi/aop-bootargs-tests.py index d2e24ecb07eb2..fd2920a7a3353 100644 --- a/tools/asahi/aop-bootargs-tests.py +++ b/tools/asahi/aop-bootargs-tests.py @@ -90,9 +90,29 @@ def method(source): def configs(source): + """struct AopHwConfig and the HW_CFG_* constants, each with its doc comment. + + Other items between them (the sleep's PM callbacks) use kernel bindings the harness + doesn't have. + """ start = source.index('struct AopHwConfig {') end = source.index('kernel::of_device_table!', start) - return source[start:end] + out, pending, item, keep, depth = [], [], [], None, 0 + for line in source[start:end].splitlines(keepends=True): + if keep is None: + if not line.strip() or line.startswith(('//', '#[')): + pending.append(line) + continue + keep = line.startswith(('struct AopHwConfig', 'const HW_CFG_')) + item, pending, depth = pending + [line], [], 0 + else: + item.append(line) + depth += line.count('{') - line.count('}') + if depth == 0 and line.rstrip().endswith((';', '}')): + if keep: + out += item + keep = None + return ''.join(out + pending) def compile_run(rustc, p, binary, code): diff --git a/tools/asahi/apple-dart-dapf-tests.py b/tools/asahi/apple-dart-dapf-tests.py new file mode 100644 index 0000000000000..9a516d2feead8 --- /dev/null +++ b/tools/asahi/apple-dart-dapf-tests.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the actual DAPF parser's optional-description and rejection contract. + +The host dependencies model DT metadata, allocation and mapping counts only. +No filter register or firmware is accessed; this cannot qualify hardware replay. +""" +import argparse +from pathlib import Path +import re +import resource +import subprocess + + +def function(text): + match = re.search(r'^static int apple_dart_dapf_parse\(', text, re.M) + if not match: + raise RuntimeError('production DAPF parser not found') + start = text.index('{', match.start()) + end, depth = start + 1, 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[match.start():end] + + +PREFIX = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define IORESOURCE_MEM 0 +#define GFP_KERNEL 0 +#define IS_ERR(p) ((uintptr_t)(p) >= (uintptr_t)-4095) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +struct device_node { int unused; }; +struct device { struct device_node *of_node; }; +struct platform_device { struct device dev; }; +struct resource { size_t size; }; +struct apple_dart { unsigned dapf_count; void *dapf; u64 *dapf_ranges; u32 *dapf_r20; }; +static bool has_ranges, has_r20, force_replay, map_failure; +static int count, maps, allocations; +static struct resource region; +static struct resource *platform_get_resource_byname(struct platform_device *p, int type, const char *name) +{ return ®ion; } +static size_t resource_size(struct resource *r) { return r->size; } +static bool of_property_present(struct device_node *n, const char *name) +{ return !strcmp(name,"apple,dapf-ranges") ? has_ranges : has_r20; } +static bool of_property_read_bool(struct device_node *n, const char *name) +{ assert(!strcmp(name,"apple,restore-on-power-up")); return force_replay; } +static int of_property_count_u64_elems(struct device_node *n, const char *name) +{ assert(!strcmp(name,"apple,dapf-ranges")); return has_ranges ? count : -EINVAL; } +static int dev_err_probe(struct device *d, int error, const char *format, ...) { return error; } +static void *devm_platform_ioremap_resource_byname(struct platform_device *p, const char *name) +{ ++maps; return map_failure ? (void *)(intptr_t)-EIO : ®ion; } +static void *devm_kcalloc(struct device *d, size_t n, size_t size, int flags) +{ ++allocations; return calloc(n ? n : 1,size); } +static int of_property_read_u64_array(struct device_node *n, const char *name, u64 *values, int nr) +{ for (int i=0;i\n' + 'static inline bool pm_suspend_via_firmware(void) { return false; }\n' + 'static inline bool pm_resume_via_firmware(void) { return false; }\n') + # The bridge's sleep-commit stop: MMIO, RTKit and the sleep driver, reached only in a suspend. + write(tmp, 'linux/io.h', '#include "mock.h"\n' + 'static inline u32 readl_relaxed(const volatile void __iomem *a) { assert(!"no system sleep here"); return 0; }\n' + 'static inline void iounmap(volatile void __iomem *a) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/of_address.h', '#include "mock.h"\n' + 'static inline void __iomem *of_iomap(struct device_node *n, int i) { assert(!"no system sleep here"); return NULL; }\n') + write(tmp, 'linux/soc/apple/rtkit.h', '#include "mock.h"\n#include \n' + 'struct apple_rtkit;\n' + 'static inline int apple_rtkit_sleep_idle_polled(struct apple_rtkit *r, void __iomem *c,\n' + '\t\t\t\t\t\tunsigned int *us) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_rtkit_mailbox_recv_off(struct apple_rtkit *r) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/soc/apple/sleep.h', '#include "mock.h"\n#include \n' + 'static inline int apple_sleep_coproc_register(struct device *d, void __iomem *a, bool p,\n' + '\t\t\t\t\t int (*f)(void *), void *x) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_sleep_coproc_set_ready(struct device *d, bool r) { assert(!"no system sleep here"); }\n' + 'static inline bool apple_sleep_coproc_stopped(struct device *d) { assert(!"no system sleep here"); return false; }\n' + 'static inline bool apple_sleep_commit_stops_coprocs(void) { assert(!"no system sleep here"); return false; }\n') + write(tmp, 'linux/soc/apple/pmp-display.h', '#include \nbool apple_pmp_display_retained_board(void);\n') write(tmp, 'mock.h', '''#pragma once #include #include #include #include +#define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_err_probe(d, e, ...) (e) +#define dev_info(...) ((void)0) +#include +#define __iomem #include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; @@ -121,6 +153,7 @@ def main(): /* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} +static inline int of_property_match_string(const struct device_node*n,const char*p,const char*s){assert(!"no system sleep here");return -EINVAL;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" @@ -129,6 +162,12 @@ def main(): int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} +bool apple_pmp_display_retained_board(void){return false;} +void apple_pmp_report_sleep(struct device*d,bool at_commit){assert(!"no system sleep here");} +void apple_pmp_report_wake(struct device*d,bool real){assert(!"no system sleep here");} +int apple_pmp_sleep(const void*data){assert(!"no system sleep here");return -EIO;} +int apple_pmp_sleep_prepare(const void*data,struct apple_pmp_commit_stop*stop){assert(!"no system sleep here");return -EIO;} +int apple_pmp_wake(const void*data){assert(!"no system sleep here");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/isp-tests.py b/tools/asahi/isp-tests.py index d31176c125fab..e649d3d83cbd6 100644 --- a/tools/asahi/isp-tests.py +++ b/tools/asahi/isp-tests.py @@ -50,6 +50,9 @@ def main(): 'apple_isp_firmware_halt', 'isp_firmware_boot', 'apple_isp_firmware_boot', 'apple_isp_firmware_shutdown'], } + # #1445 sends through a per-channel helper; the latch guards it too. + if 'static int isp_cmd_send_chan(' in inputs['isp-cmd.c']: + groups['isp-cmd.c'] = ['isp_cmd_send_chan'] + groups['isp-cmd.c'] if args.suite == 'session': groups['isp-ipc.c'] += ['chan_rx_done', 'ipc_chan_handle'] groups['isp-fw.c'] += ['apple_isp_isr_thread'] @@ -66,7 +69,11 @@ def main(): guard = 'if (READ_ONCE(isp->ipc_error))\n\t\treturn READ_ONCE(isp->ipc_error);' if guard not in bodies['isp_cmd_send']: raise RuntimeError('overwrite negative control no longer matches its guard') - bodies['isp_cmd_send'] = bodies['isp_cmd_send'].replace(guard, '') + for name in ('isp_cmd_send', 'isp_cmd_send_chan'): + if name in bodies: + if guard not in bodies[name]: + raise RuntimeError('overwrite negative control no longer matches ' + name) + bodies[name] = bodies[name].replace(guard, '') elif args.control == 'stop': bodies['apple_isp_stop_capture'] = bodies['apple_isp_stop_capture'].replace( 'if (!isp_cmd_ch_stop(isp, 0)) /* TODO channel mask */', diff --git a/tools/asahi/isp/buffers.c b/tools/asahi/isp/buffers.c index d78873ffe27dc..d52cf0f7e47d8 100644 --- a/tools/asahi/isp/buffers.c +++ b/tools/asahi/isp/buffers.c @@ -63,6 +63,11 @@ static void set_bit(unsigned int bit, unsigned long *state) *state |= 1UL << bit; } +static void clear_bit(unsigned int bit, unsigned long *state) +{ + *state &= ~(1UL << bit); +} + #define dev_err(...) ((void)0) #define dev_err_ratelimited(...) ((void)0) #define dev_warn_ratelimited(...) ((void)0) @@ -301,6 +306,8 @@ static void apple_isp_stop_capture(struct apple_isp *isp); /* These fixtures exercise the non-P010/J613 path. P010 has kernel KUnit coverage. */ #define ISP_FW_DEAD 3 +/* The zero-initialized fixtures run the firmware (#1445's stop checks for it). */ +#define ISP_FW_RUNNING 0 #define __set_bit set_bit static bool isp_is_p010(struct apple_isp *isp) { return false; } static bool isp_supports_p010(struct apple_isp *isp) { return false; } diff --git a/tools/asahi/isp/session.c b/tools/asahi/isp/session.c index 08d24f9f124b7..0b9d563b5bfbe 100644 --- a/tools/asahi/isp/session.c +++ b/tools/asahi/isp/session.c @@ -55,6 +55,8 @@ struct isp_channel { struct hardware { bool resident_fw; + /* #1445: resident firmware that starts again from its own memory */ + bool fw_restart; unsigned int fw_idle_ms; }; @@ -70,6 +72,7 @@ struct apple_isp { struct isp_channel *ipc_stale; bool ipc_failed; const struct hardware *hw; + struct { bool started; } fw; void *dev, *wq, *cmd_virt; u64 cmd_iova; unsigned int current_ch; @@ -322,6 +325,8 @@ static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, sizeof(v)); static bool isp_is_p010(struct apple_isp *isp) { return false; } static int isp_ipc_resync(struct apple_isp *isp) { return 0; } static void isp_restore_fw_memory(struct apple_isp *isp) { } +/* #1445's post-restart query; these fixtures never restart (fw.started is clear). */ +static int isp_check_restarted(struct apple_isp *isp) { CHECK(false); return 0; } /* DRIVER FUNCTIONS */ int main(void) { @@ -355,6 +360,9 @@ int main(void) isp.ipc_error = -EIO; CHECK(ipc_chan_send(&isp, &ch, 1) == -EIO); CHECK(writes == 1 && cursors == 0); + /* A capture stops while it streams, with the firmware running. */ + int fw_state = isp.fw_state; + isp.fw_state = ISP_FW_RUNNING; for (unsigned int fail = 0; fail < 2; fail++) { stop_error = fail ? -ETIME : 0; stops = 0; @@ -363,6 +371,7 @@ int main(void) apple_isp_stop_capture(&isp); CHECK(stops == 1 && returns == !fail && watchdog_stops == 1); } + isp.fw_state = fw_state; for (fail_stage = 1; fail_stage <= 5; fail_stage++) { isp.ipc_error = -ETIME; stage_error = -ENOMEM; diff --git a/tools/asahi/j613-mtp-s2idle-tests.py b/tools/asahi/j613-mtp-s2idle-tests.py index 7aa413645303a..21d673862dac6 100644 --- a/tools/asahi/j613-mtp-s2idle-tests.py +++ b/tools/asahi/j613-mtp-s2idle-tests.py @@ -32,6 +32,11 @@ def fn(s,n): static int pm_runtime_force_resume(struct device *d){forced_resume++;return 8;} static bool of_machine_is_compatible(const char *s){return true;} static unsigned readl(void *p){return *(unsigned *)p;} +/* #1445: only a firmware-driven (T8140 S2R) sleep takes the noirq or AP-idle paths. */ +static bool pm_suspend_via_firmware(void){return false;} +static int apple_rtkit_helper_ap_idle(struct apple_rtkit_helper *h){assert(!"firmware-driven path in s2idle");return 0;} +/* Its AP-on notice is a no-op without an earlier idle notice. */ +static void apple_rtkit_helper_ap_on(struct apple_rtkit_helper *h){assert(h);} ''' body=fn(s,'apple_dart_system_suspend')+fn(s,'apple_dart_system_resume')+fn(h,'apple_rtkit_helper_suspend')+fn(h,'apple_rtkit_helper_resume') for forbidden in ['writel','hw_reset','invalidate_tlb','apple_rtkit_wake','apple_rtkit_boot','apple_rtkit_quiesce']: diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 13d1262f601ed..e72bebc02f3ab 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -126,6 +126,8 @@ def function(source, name): static void mt_awdl_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_data_complete(struct mt7932*m,const u8*p,size_t n){} static void mt_packet_receive(struct mt7932*m,const u8*p,size_t n){} +static void mt_hif_event(struct mt7932*m,const struct mt7932_event*e){} +static void mt_wow_reason_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_rf_fail_locked(struct mt7932*m,int e){m->rf_ready=false;m->link_failed=true;m->cal_state.error=e;m->cal_state.active=false;m->connect_error=e;complete_all(&m->assoc_done);complete_all(&m->cal_response);} CHANNEL LINK @@ -254,6 +256,9 @@ def main(): headers.append(header) cal=(root/DRIVER/"calibration.h").read_text() headers.append(cal[cal.index("struct mt7932_cal_completion {"):cal.index("struct mt7932_cal_segment {")]) + # The receive path's other events, by their production IDs: the firmware's HIF and WoW reports. + headers.append(re.search(r"^#define MT7932_CID_HIF_CTRL\b.*$", rx, re.M).group(0)) + headers.append(re.search(r"^#define MT7932_EID_WOW_REASON\b.*$", (root/DRIVER/"wow.h").read_text(), re.M).group(0)) parts["HEADERS"]="\n".join(headers) candidates={"production":parts} if args.old_source: diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py index 4f4d183021905..fe4a55aa16d79 100644 --- a/tools/asahi/mtp-pm-tests.py +++ b/tools/asahi/mtp-pm-tests.py @@ -46,12 +46,18 @@ def function(source, name): static bool schedule_delayed_work(struct delayed_work *w, unsigned long d) { w->queued++; w->pending = true; return true; } struct hid_device { struct magicmouse_sc *data; int id; }; struct dchid_iface { struct hid_device *hid; }; -struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; +struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended, host_asleep; }; struct device { struct dockchannel_hid *data; }; static struct dockchannel_hid *transport; static unsigned suspends[16], resumes[16], key_releases; static int suspend_error_at = -1, resume_error_at = -1; static void *dev_get_drvdata(struct device *d) { return d->data; } +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 +#define dev_warn(...) ((void)0) +/* Not a firmware-driven suspend: the host state is never set here. */ +static bool pm_suspend_via_firmware(void) { return false; } +static int dchid_host_state(struct dockchannel_hid *d, int state) { (void)d; (void)state; assert(0); return 0; } static void *hid_get_drvdata(struct hid_device *d) { return d->data; } static void magicmouse_force_key(struct magicmouse_sc *m, bool down) { assert(!down); key_releases++; diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index 1d0a5302af4ce..8cfe659498c49 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -80,7 +80,8 @@ def function(source, name): enum dwc3_apple_state {DWC3_APPLE_PROBE_PENDING,DWC3_APPLE_NO_CABLE,DWC3_APPLE_HOST,DWC3_APPLE_DEVICE,DWC3_APPLE_SUSPENDED}; #define DWC3_GCTL_PRTCAP_HOST 1 struct dwc3 {int num_usb3_ports,num_usb2_ports;int usb3_generic_phy[2],usb2_generic_phy[2],usb3_phy,usb2_phy;int current_dr_role;bool dis_u3_susphy_quirk;}; -struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending,dt_dis_u3_susphy;unsigned usb3_power_off_pending,usb2_power_off_pending;int usb2_phy;}; +struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending,dt_dis_u3_susphy,sleep_ddr;unsigned usb3_power_off_pending,usb2_power_off_pending;int usb2_phy;}; +static void apple_sleep_usb_slp_ddr(bool on){(void)on;} static int core_error,host_error,gadget_error,reset_error,role_error,start_role; static unsigned rpm_puts; /* A started core holds the glue runtime PM reference until core_stop drops it. */ diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 469cacc4b3082..062597ca80b56 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -25,6 +25,9 @@ 'control |= rtc->hw->control_event_bit;'), 'stowe-offset': ('static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = {', 'static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = {\n\t.offset_in_seconds = true,'), + 'baku-limit': ('.max_delay = U32_MAX,', '.max_delay = APPLE_PMU_RTC_MAX_DELAY,'), + 'set-no-round': ('(counter + BIT(APPLE_PMU_RTC_HZ_SHIFT - 1))', '(counter)'), + 'set-negative': ('if (offset < 0 || offset', 'if (offset'), } PRELUDE = r''' @@ -41,12 +44,14 @@ typedef uint64_t u64; typedef int64_t s64; typedef uint32_t __le32; +typedef uint64_t __le64; #define BIT(n) (1U << (n)) #define U32_MAX UINT32_MAX #define RTC_TIMESTAMP_END_9999 253402300799LL #define cpu_to_le32(x) (x) #define le32_to_cpu(x) (x) #define le64_to_cpu(x) (x) +#define cpu_to_le64(x) (x) #define IS_ERR(x) ((intptr_t)(x) < 0) #define PTR_ERR(x) ((int)(intptr_t)(x)) #define kfree free @@ -63,13 +68,15 @@ struct rtc_wkalrm { struct rtc_time time; bool enabled, pending; }; struct rtc_device { int events; }; struct device { void *data; bool wake; int wake_events; }; -struct nvmem_cell { u8 data[8]; size_t bytes; int error; }; +struct nvmem_cell { u8 data[8]; size_t bytes; int error, write_error; }; struct regmap { u8 bytes[65536]; unsigned ctrl, counter, compare, mask, event, bit, control_event; int ops, fail; }; static int disabled, pm_suspend_target_state = PM_SUSPEND_TO_IDLE; +static bool firmware_suspend; +static bool pm_suspend_via_firmware(void) { return firmware_suspend; } static inline __attribute__((unused)) s64 sign_extend64(u64 v, int bit) { return (s64)(v << (63 - bit)) >> (63 - bit); } static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } static void mutex_lock(struct mutex *m) { (void)m; } @@ -90,6 +97,10 @@ void *p = malloc(c->bytes); assert(p); memcpy(p, c->data, c->bytes); *bytes = c->bytes; return p; } +static int nvmem_cell_write(struct nvmem_cell *c, void *buf, size_t bytes) { + if (c->write_error) return c->write_error; + assert(bytes == c->bytes); memcpy(c->data, buf, bytes); return bytes; +} static bool io_error(struct regmap *m) { return ++m->ops == m->fail; } static int regmap_read(struct regmap *m, unsigned reg, unsigned *v) { if (io_error(m)) return -EIO; @@ -226,16 +237,59 @@ assert(!apple_pmu_rtc_validate_alarm(&rtc, 101)); assert(apple_pmu_rtc_validate_alarm(&rtc, 100) == -ETIME); assert(!apple_pmu_rtc_validate_alarm(&rtc, 3700)); - assert(apple_pmu_rtc_validate_alarm(&rtc, 3701) == -ERANGE); alarm.time.seconds = 1100; assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ETIME); alarm.time.seconds = 4701; - assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ERANGE); + if (a) { + /* One hour, where the alarm only ends a suspend-to-idle. */ + assert(apple_pmu_rtc_validate_alarm(&rtc, 3701) == -ERANGE); + assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ERANGE); + } else { + /* Baku: as far as the 32-bit comparator reaches. */ + assert(!apple_pmu_rtc_validate_alarm(&rtc, 3701)); + assert(!apple_pmu_rtc_validate_alarm(&rtc, U32_MAX)); + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && comparator() == 3701); + alarm.time.seconds = 86400 * 7 + 1000; + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && comparator() == 86400 * 7); + alarm.time.seconds = 1000 + (s64)U32_MAX + 1; + assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ERANGE); + assert(!apple_pmu_rtc_alarm_enable(&dev, 0)); + } /* Reject malformed offsets and propagate NVMEM errors. */ offset.bytes = a ? 8 : 6; assert(apple_pmu_rtc_offset(&rtc, &ticks) == -EINVAL); offset.error = -EIO; assert(apple_pmu_rtc_read_time(&dev, &time) == -EIO); + if (!a) { + /* Baku sets its whole-second offset, rounding the counter to the second. */ + reset(v); + time.seconds = 5000; + set_counter((100ULL << 16) + 32767); + assert(!apple_pmu_rtc_set_time(&dev, &time)); + assert(get_unaligned_le64(offset.data) == 4900); + assert(!apple_pmu_rtc_read_time(&dev, &time) && time.seconds == 5000); + time.seconds = 5000; + set_counter((100ULL << 16) + 32768); + assert(!apple_pmu_rtc_set_time(&dev, &time)); + assert(get_unaligned_le64(offset.data) == 4899); + /* Alarms follow the new offset. */ + alarm.time.seconds = 5030; + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && comparator() == 131); + assert(!apple_pmu_rtc_alarm_enable(&dev, 0)); + /* Before the counter's own time, and past what the offset may hold. */ + time.seconds = 99; + assert(apple_pmu_rtc_set_time(&dev, &time) == -ERANGE); + time.seconds = RTC_TIMESTAMP_END_9999; + assert(apple_pmu_rtc_set_time(&dev, &time) == -ERANGE); + assert(get_unaligned_le64(offset.data) == 4899); + time.seconds = 5000; + offset.write_error = -EIO; + assert(apple_pmu_rtc_set_time(&dev, &time) == -EIO); + offset.write_error = 0; + map.ops = 0; map.fail = 1; + assert(apple_pmu_rtc_set_time(&dev, &time) == -EIO); + assert(get_unaligned_le64(offset.data) == 4899); + } reset(v); set_offset(-32768); if (a) { @@ -275,11 +329,15 @@ reset(v); assert(!apple_pmu_rtc_prepare(&dev)); pm_suspend_target_state = 2; assert(apple_pmu_rtc_prepare(&dev) == -EBUSY); + /* A firmware-driven suspend programs the wake itself. */ + firmware_suspend = true; + assert(!apple_pmu_rtc_prepare(&dev)); + firmware_suspend = false; pm_suspend_target_state = PM_SUSPEND_TO_IDLE; } int main(void) { run_variant(BAKU); run_variant(ABBEY); run_variant(STOWE); run_variant(MAVERICK); - puts("PMU RTC: Baku, Abbey, Stowe and Maverick arithmetic, W1C transactions and fault injection passed"); + puts("PMU RTC: Baku, Abbey, Stowe and Maverick arithmetic, W1C transactions, Baku clock setting and fault injection passed"); return 0; } ''' diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index e8e7166399a0e..9df97f65c1591 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -37,6 +37,7 @@ def function(source, name): #define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) #define APPLE_RTKIT_MAX_ENDPOINTS 256 #define WRITE_ONCE(x,v) ((x)=(v)) +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) #define spin_lock_irqsave(l,f) ((void)(l), (f)=0) #define spin_unlock_irqrestore(l,f) ((void)(l), (void)(f)) struct apple_rtkit_shmem { size_t size; }; @@ -49,6 +50,9 @@ def function(source, name): struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer, pmp_aux_buffer; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; + bool oslog_echo; + u64 oslog_owed; + int boot_result; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4], started[4]; char *syslog_msg_buffer; @@ -69,6 +73,7 @@ def function(source, name): static void set_bit(unsigned n, unsigned long *b) { b[n/64] |= 1UL << (n%64); } static void bitmap_zero(unsigned long *b, unsigned n) { memset(b,0,n/8); } static void reinit_completion(int *c) { *c=0; } +static void complete_all(int *c) { *c=1; } static void apple_rtkit_rx(void) {} static void apple_rtkit_detach_rx(struct apple_rtkit *r) { ++detachments; r->shutting_down=true; } static void flush_workqueue(void *w) { (void)w; ++flushes; } diff --git a/tools/asahi/rtkit-retained-tests.py b/tools/asahi/rtkit-retained-tests.py index b79a9cd542bd2..f4e81d99f681d 100755 --- a/tools/asahi/rtkit-retained-tests.py +++ b/tools/asahi/rtkit-retained-tests.py @@ -66,18 +66,20 @@ def constants(source): { (void)rtk; sent++; sent_type = type; sent_data = data; return 0; } -/* A signal is pending for every interruptible wait. */ +/* A fatal signal is pending for every killable wait. */ #define ERESTARTSYS 512 #define READ_ONCE(x) (x) +enum { PM_SUSPEND_ON, PM_SUSPEND_TO_IDLE, PM_SUSPEND_STANDBY, PM_SUSPEND_MEM }; +static int pm_suspend_target_state; struct completion { int unused; }; -static unsigned waited_uninterruptible, waited_interruptible; +static unsigned waited_uninterruptible, waited_killable; static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long t) { (void)c; waited_uninterruptible++; return t; } -static long wait_for_completion_interruptible_timeout(struct completion *c, unsigned long t) +static long wait_for_completion_killable_timeout(struct completion *c, unsigned long t) { - (void)c; (void)t; waited_interruptible++; return -ERESTARTSYS; + (void)c; (void)t; waited_killable++; return -ERESTARTSYS; } ''' @@ -156,14 +158,23 @@ def constants(source): assert(sent == 0 && r.iop_pwr_ack_completion == 1); assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_QUIESCED); - /* A retained wake waits through a signal; other waits stay interruptible. */ + /* + * A retained wake waits through a signal, and so does a wait in a system + * sleep; any other wait ends only for a fatal one, not the freezer's. + */ struct completion c; r = fresh(); assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); - assert(waited_uninterruptible == 1 && waited_interruptible == 0); + assert(waited_uninterruptible == 1 && waited_killable == 0); r.retained_wake = false; assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); - assert(waited_uninterruptible == 1 && waited_interruptible == 1); + assert(waited_uninterruptible == 1 && waited_killable == 1); + pm_suspend_target_state = PM_SUSPEND_MEM; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); + assert(waited_uninterruptible == 2 && waited_killable == 1); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); + assert(waited_uninterruptible == 2 && waited_killable == 2); puts("PASS retained endpoint map replies, IOP acknowledgments and waits"); return 0; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 976d7b27117c1..c12950590b509 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -78,6 +78,8 @@ def mutations(flag, names, base, expect=KILL): 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-m4-pmu-tests.py': [([], PASS)] + mutations('--mutation', ('m1-layout', 'cycles-anywhere', 'expose-8-9', 'flags-in-event'), []), + 'apple-dart-dapf-tests.py': [(out(), PASS)] + + mutations('--mutation', ('missing-guard', 'ignore-replay', 'ignore-orphan'), out()), 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'asahi-stats-raw-tests.py': [([], PASS)], @@ -167,13 +169,22 @@ def mutations(flag, names, base, expect=KILL): + [(out('--output') + ['--old-usb'], KILL), (out('--output') + ['--old-regulatory'], KILL)], 'pciec-policy-tests.py': [(out(), PASS)], 'pmgr-bus-tests.py': [([], PASS)], + 'sep-suspend-errors-tests.py': [(out(), PASS)] + + mutations('--mutation', ('unsupported-as-success', 'read-as-deadline', 'no-capability-check', + 'no-transport-check', 'lost-accept', 'nonphysical-wake', + 'wake-before-arm', 'bulk-error-as-success', 'omit-pending-wake', + 'omit-before-send', 'leak-prepared', 'sent-error-return', + 'accept-unanswered-gate', 'ignore-result', 'ignore-gate-timeout', + 'finish-without-physical', 'poll-after-callback', 'poll-unsupported', + 'stale-observation', 'drain-two', 'post-send-wake-refusal'), out()), 'pmp-agx-vote-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'pmp-j613-floor-tests.py': [([], PASS)] + mutations('--mutant', ['no-hold', 'no-rate', 'no-streak', 'bad-word', 'foreign-ok', 'no-names', 'no-sum', 'no-board', 'no-default', 'no-disable'], []), 'pmp-j613-hwmon-tests.py': [([], PASS)], 'pmu-rtc-tests.py': [(out(), PASS)] + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', 'offset-sign', 'event-bit', 'wrong-control', - 'maverick-event', 'stowe-offset'), out()), + 'maverick-event', 'stowe-offset', 'baku-limit', + 'set-no-round', 'set-negative'), out()), 'perf-mmap-alias-lifecycle-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-crashlog-tests.py': [(out(), PASS)] diff --git a/tools/asahi/sep-suspend-errors-tests.py b/tools/asahi/sep-suspend-errors-tests.py new file mode 100644 index 0000000000000..2ae69ec2c32fc --- /dev/null +++ b/tools/asahi/sep-suspend-errors-tests.py @@ -0,0 +1,736 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual SEP commit/error paths with host-only dependency stubs. + +Extract current C stop/finish/prepare/noirq, provider poll-one and registration, +Rust wrappers/enter/leave/exit/kick, and the kernel polling macros verbatim. +C fatal paths are observed wholly within C, never by unwinding a Rust frame. +Virtual time/MMIO only test AP control decisions, not firmware timing or safety. +No target access or kernel build occurs. +""" +import argparse +from pathlib import Path +import re +import resource +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def function(source, name): + match = re.search(r'^[ \t]*(?:(?:pub(?:\([^)]*\))? )?fn|(?:static )?(?:int|bool|void)(?: __maybe_unused)?) ' + re.escape(name) + r'\([^;]*?\)\s*(?:->[^\{]+)?\{', + source, re.M) + if not match: + raise AssertionError(f'missing function: {name}') + start = source.index('{', match.start()) + end, depth = start + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +def macro(source, name): + match = re.search(r'^#define ' + re.escape(name) + r'\([^\n]*\n(?:.*\\\n)*.*', + source, re.M) + if not match: + raise AssertionError(f'missing macro: {name}') + return match.group() + + +def replace_once(source, before, after): + # A stale mutation is a broken harness, never a killed behavioral control. + if source.count(before) != 1: + raise AssertionError(f'mutation requires one exact match: {before!r}') + return source.replace(before, after, 1) + + +# Mutations apply to an extracted real function, not to a mock decision model. +# All must compile and fail a C assertion or a Rust test assertion in remote CI. +MUTATIONS = { + 'unsupported-as-success': ('leave_for_commit', 'Err(Error::from_errno(rc))', 'Ok(())'), + 'read-as-deadline': ('sep_pm_wait_gated', 'if (ret)\n\t\treturn ret;\n\n\treturn !!poll_ret;', + 'if (ret)\n\t\treturn 1;\n\n\treturn !!poll_ret;'), + 'no-capability-check': ('sep_pm_commit_prepare', + ' || !IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) ||\n\t !is_kernel_in_hyp_mode()', ''), + 'no-transport-check': ('sep_pm_commit_prepare', + 'if (!apple_mbox_poll_one_supported(mbox))', 'if (false)'), + 'lost-accept': ('sleep_enter', 'pm::SleepGate::Lost => return Err(EIO),', + 'pm::SleepGate::Lost => return Ok(()),'), + 'nonphysical-wake': ('sleep_exit', 'let real = unsafe { bindings::pm_resume_via_firmware() };', + 'let real = true;'), + 'wake-before-arm': ('sleep_kick', + 'self.control.lock().wake_reset();\n self.sleep_release_rx();', + 'self.sleep_release_rx();\n self.control.lock().wake_reset();'), + 'bulk-error-as-success': ('sep_pm_commit_stop', 'if (ret < 0)\n\t\tgoto out;', + 'if (ret < 0) { ret = 0; goto out; }'), + 'omit-pending-wake': ('sep_pm_commit_stop', 'if (pm_wakeup_pending())', 'if (false)'), + 'omit-before-send': ('sep_pm_commit_stop', 'if (xchg(&sleep_refuse_before_send, false))', + 'if (false)'), + 'leak-prepared': ('sep_pm_commit_stop', 'if (prepared)\n\t\t\tapple_dart_sleep_restore(c->dart);', + 'if (false)\n\t\t\tapple_dart_sleep_restore(c->dart);'), + 'sent-error-return': ('sep_pm_commit_stop', + 'panic("SEP SLEEP failed after send: ret %d reply_valid %d result %#x state %#x",\n\t\t ret, c->answered, c->reply_result, c->state);', + 'return ret;'), + 'accept-unanswered-gate': ('sep_pm_commit_stop', 'if (ret >= 0 && !c->answered)', + 'if (ret >= 0 && !c->answered && !(c->state & SEP_PS_DEV_DISABLE))'), + 'ignore-result': ('sep_pm_commit_stop', 'if (c->answered && c->reply_result)', + 'if (false && c->answered && c->reply_result)'), + 'ignore-gate-timeout': ('sep_pm_commit_stop', 'if (c->gate_ret)', 'if (false && c->gate_ret)'), + 'finish-without-physical': ('sep_pm_commit_finish', 'if (c->sent && !pm_resume_via_firmware())', + 'if (false)'), + 'poll-after-callback': ('apple_mbox_poll_one', 'mbox->rx(mbox, msg, mbox->cookie);', + 'mbox->rx(mbox, msg, mbox->cookie);\n\t\tcontrol = readl_relaxed(mbox->regs + mbox->hw->i2a_control);'), + 'poll-unsupported': ('apple_mbox_poll_one', 'if (!apple_mbox_poll_one_supported(mbox))', + 'if (false)'), + 'stale-observation': ('sep_pm_commit_prepare', 'c->answered = false;', '/* stale answered */'), + 'drain-two': ('apple_mbox_poll_one', 'ret = 1;', + 'msg = apple_mbox_read_msg(mbox);\n\t\tmbox->rx(mbox, msg, mbox->cookie);\n\t\tret = 2;'), + 'post-send-wake-refusal': ('sep_pm_commit_stop', 'c->sent = true;', + 'c->sent = true;\n\tif (pm_wakeup_pending()) return -EBUSY;'), +} + +C_PREFIX = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint32_t u32; typedef uint64_t u64; +typedef int64_t s64; typedef long long ktime_t; +#define BIT(n) (1u << (n)) +#define SEP_PS_DEV_DISABLE BIT(10) +#define USEC_PER_MSEC 1000 +#define NSEC_PER_USEC 1000 +#define READ_ONCE(x) (x) +#define IS_ENABLED(x) (config_enabled) +#define CONFIG_APPLE_SOC_SLEEP 1 +#define __maybe_unused +#define __iomem +#define unlikely(x) (x) +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_info(...) ((void)0) +#define might_sleep_if(x) ((void)0) +#define barrier() ((void)0) +#define cpu_relax() ((void)0) +#define guard(kind) (void) +#define xchg(p, v) ({ __typeof__(*(p)) old = *(p); *(p) = (v); old; }) +struct device_node { struct device_node *parent; }; +struct device { struct device_node *of_node; }; +struct resource { unsigned long start; }; +struct regmap { int unused; }; +struct apple_mbox_msg { u64 msg0; u32 msg1; }; +struct apple_mbox_hw { + bool is_ascwrap_v8, counted_fifo, has_irq_controls; unsigned i2a_control; +}; +struct apple_mbox { + void *regs; struct apple_mbox_hw *hw; bool removing; int rx_lock; + void (*rx)(struct apple_mbox *, struct apple_mbox_msg, void *); void *cookie; +}; +static struct device_node parent, node = { &parent }; +static struct device device = { &node }, dart; +static struct regmap regmap; +static struct apple_mbox_hw hw; +static struct apple_mbox mbox; +static int apple_sleep_coprocs_lock; +static void *apple_sleep; +static struct { + struct device *dev; int (*stop)(void *); void *data; +} apple_sleep_enclave; +static bool config_enabled, hyp, sep_pm_suspending, dart_present, dart_ready, map_ok; +static bool firmware, real_wake, pending_wake, late_wake, sleep_refuse_before_send, tx_ready; +static int domain_error, property_error, address_error, read_error, read_error_after; +static int gate_after, reads, put_refs, maps, unmaps, lookups, prepares, restores; +static int prepare_error, send_error, bulk_error, one_error, sends, closes; +static int mailbox_reads, message_reads, forwarded, lock_depth; +static int resource_bytes, close_gate, early_gate; +static bool message_in_lock, matched_read; +static struct apple_mbox_msg queue[4]; +static unsigned queued, consumed; +static ktime_t now; +static bool (*sep_pm_told)(void); +static bool yes(void) { return true; } +static bool no(void) { return false; } +static bool pm_suspend_via_firmware(void) { return firmware; } +static bool pm_resume_via_firmware(void) { return real_wake; } +static bool is_kernel_in_hyp_mode(void) { return hyp; } +static bool pm_wakeup_pending(void) { return pending_wake; } +static int sep_pm_domain(void *p, struct regmap **m, u32 *o) { + lookups++; *m = ®map; *o = 0; return domain_error; +} +static int regmap_read(struct regmap *m, u32 o, u32 *v) { + ++reads; if (read_error && reads >= read_error_after) return read_error; + *v = gate_after && reads >= gate_after ? SEP_PS_DEV_DISABLE : 0; return 0; +} +static int regmap_write(struct regmap *m, u32 o, u32 v) { return 0; } +static ktime_t ktime_get(void) { return now; } +static ktime_t ktime_add_us(ktime_t t, u64 us) { return t + us; } +static int ktime_compare(ktime_t a, ktime_t b) { return (a > b) - (a < b); } +static void usleep_range(unsigned long a, unsigned long b) { now += b; } +static void udelay(unsigned long us) { now += us; } +static struct device_node *of_parse_phandle(struct device_node *n, const char *p, int i) { + lookups++; return domain_error ? NULL : &node; +} +static int of_property_read_u32(struct device_node *n, const char *p, u32 *v) { + *v = 0; return property_error; +} +static int of_address_to_resource(struct device_node *n, int i, struct resource *r) { + r->start = 0; return address_error; +} +static void of_node_put(struct device_node *n) {} +static unsigned resource_size(struct resource *r) { return resource_bytes; } +static struct device *sep_pm_dart(struct device *d) { return dart_present ? &dart : NULL; } +static void *dev_get_drvdata(struct device *d) { return dart_ready ? d : NULL; } +static void *ioremap(unsigned long start, unsigned size) { ++maps; return map_ok ? ®map : NULL; } +static void iounmap(void *p) { ++unmaps; } +static void put_device(struct device *d) { ++put_refs; } +static int apple_dart_sleep_prepare(struct device *d) { ++prepares; return prepare_error; } +static void apple_dart_sleep_restore(struct device *d) { ++restores; } +static void lock_rx(void) { + assert(lock_depth == 0); lock_depth++; message_in_lock = false; +} +static void unlock_rx(void) { assert(lock_depth == 1); lock_depth--; message_in_lock = false; } +#define spin_lock_irqsave(lock, flags) do { (flags) = 0; lock_rx(); } while (0) +#define spin_unlock_irqrestore(lock, flags) do { (void)(flags); unlock_rx(); } while (0) +static void foreign_rx(struct apple_mbox *m, struct apple_mbox_msg msg, void *cookie) { + assert(cookie == &device); ++forwarded; +} +static u32 readl_relaxed(void *p) { + // A second status/ACK/read after delivering a message is forbidden. + assert(!message_in_lock && !matched_read); ++mailbox_reads; + return queued == consumed; +} +static bool apple_mbox_rx_empty(struct apple_mbox *m, u32 control) { return control != 0; } +static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *m) { + assert(!message_in_lock && !matched_read); assert(consumed < queued); + message_in_lock = true; ++message_reads; + struct apple_mbox_msg msg = queue[consumed++]; + if ((msg.msg0 & 0xffffff) == 0x1234) matched_read = true; + return msg; +} +static int apple_mbox_poll(struct apple_mbox *m) { return bulk_error; } +static bool apple_mbox_send_ready(struct apple_mbox *m) { return tx_ready; } +static int apple_mbox_send(struct apple_mbox *m, struct apple_mbox_msg msg, bool atomic) { + assert(atomic); assert(prepares == 1); + // Production apple_mbox_send has only pre-write negative exits. + if (send_error) return send_error; + ++sends; + if (late_wake) pending_wake = true; + if (one_error == -ENODEV) m->removing = true; + if (one_error == -EOPNOTSUPP) m->hw->is_ascwrap_v8 = true; + return 0; +} +static void apple_mbox_set_recv_enabled(struct apple_mbox *m, bool enabled) { + assert(!enabled); assert(matched_read); ++closes; +} +static u32 readl(void *p) { + // PMGR is the sole permitted query after the matched reply/receive close. + return early_gate || (closes && close_gate) ? SEP_PS_DEV_DISABLE : 0; +} +static jmp_buf fatal_env; +static bool fatal_armed; +static char fatal_message[256]; +static _Noreturn void panic_observe(const char *fmt, ...) { + va_list ap; va_start(ap, fmt); vsnprintf(fatal_message, sizeof(fatal_message), fmt, ap); va_end(ap); + assert(fatal_armed && "unexpected C panic (must never unwind across Rust)"); + longjmp(fatal_env, 1); +} +#define panic panic_observe +''' + +# Appended after the actual functions/struct: only dependency state and drivers. +C_CONTROLS = r''' +void fixture(void) { + memset(&sep_pm_commit, 0, sizeof(sep_pm_commit)); + memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); + memset(&hw, 0, sizeof(hw)); + mbox = (struct apple_mbox){ .regs = &hw, .hw = &hw, .rx = foreign_rx, .cookie = &device }; + config_enabled = hyp = sep_pm_suspending = dart_present = dart_ready = map_ok = true; + firmware = tx_ready = true; apple_sleep = &device; + real_wake = pending_wake = late_wake = sleep_refuse_before_send = false; + domain_error = property_error = address_error = read_error = gate_after = reads = 0; + put_refs = maps = unmaps = lookups = prepares = restores = sends = closes = 0; + prepare_error = send_error = bulk_error = one_error = 0; + mailbox_reads = message_reads = forwarded = lock_depth = 0; + resource_bytes = 4; close_gate = 1; early_gate = 0; read_error_after = 1; + queued = consumed = 0; message_in_lock = matched_read = false; now = 0; + fatal_armed = false; fatal_message[0] = 0; sep_pm_told = yes; +} +void parameter(int which, int value) { + switch (which) { + case 0: config_enabled = value; break; case 1: hyp = value; break; + case 2: sep_pm_suspending = value; break; case 3: dart_present = value; break; + case 4: map_ok = value; break; case 5: domain_error = value; break; + case 6: read_error = value; break; case 7: apple_sleep = value ? &device : NULL; break; + case 8: gate_after = value; break; case 9: sep_pm_commit.registered = value; break; + case 10: read_error_after = value; break; case 11: dart_ready = value; break; + case 12: real_wake = value; break; case 13: hw.is_ascwrap_v8 = value; break; + case 14: hw.counted_fifo = value; break; case 15: hw.has_irq_controls = value; break; + case 16: property_error = value; break; case 17: address_error = value; break; + case 18: resource_bytes = value; break; + case 19: apple_sleep_enclave.dev = value ? &device : NULL; break; + case 20: sep_pm_commit.sent = value; break; case 21: firmware = value; break; + case 22: sep_pm_told = value ? yes : NULL; break; + } +} +int counter(int which) { + switch (which) { + case 0: return put_refs; case 1: return maps; case 2: return unmaps; + case 3: return sep_pm_commit.registered; case 4: return lookups; + case 5: return reads; case 6: return sends; case 7: return mailbox_reads; + default: assert(0); return -1; + } +} +void *device_pointer(void) { return &device; } +void *mbox_pointer(void) { return &mbox; } +int noirq(void) { return sep_pm_suspend_noirq(&device); } + +#ifdef CONTROL_MAIN +static int stage(void) { + return sep_pm_commit_prepare(&device, &mbox, 0x55, 0, 0x1234, 0xffffff); +} +static void reply(unsigned result) { + assert(queued < 4); queue[queued++] = (struct apple_mbox_msg){0x1234 | ((u64)result << 24), 0}; +} +static void balanced(void) { + assert(lock_depth == 0); assert(mbox.rx == foreign_rx && mbox.cookie == &device); +} +static void expect_stop_fatal(const char *reason) { + // This setjmp frame, production callback and panic observer are all C. + fatal_armed = true; + if (setjmp(fatal_env) == 0) { + (void)sep_pm_commit_stop(&sep_pm_commit); + assert(0 && "sent failure must panic, never return"); + } + fatal_armed = false; + assert(strstr(fatal_message, reason)); balanced(); + assert(sep_pm_commit.sent && sends == 1 && restores == 0); + assert(sep_pm_commit.registered && put_refs == 0 && unmaps == 0); +} +static void test_prepare_and_cleanup(void) { + // Unsupported RAM guards do no mailbox MMIO, mapping or sending. + for (int p = 0; p < 3; ++p) { + fixture(); parameter(p, 0); assert(stage() == -EOPNOTSUPP); + assert(!maps && !lookups && !sends && !mailbox_reads); + } + for (int p = 13; p <= 15; ++p) { + fixture(); parameter(p, 1); assert(stage() == -EOPNOTSUPP); + assert(!maps && !lookups && !sends && !mailbox_reads); + } + // Offset/resource/property/ref/registration failures retain useful negatives. + const int cases[][5] = { + {5,-ENOENT,-ENOENT,0,0}, {16,-EIO,-EIO,0,0}, {17,-EINVAL,-EINVAL,0,0}, + {18,3,-EINVAL,0,0}, {3,0,-ENODEV,0,0}, {11,0,-ENODEV,1,0}, + {4,0,-ENOMEM,1,0}, {7,0,-EOPNOTSUPP,1,1}, {19,1,-EBUSY,1,1}, + {9,1,-EBUSY,0,0}, + }; + for (unsigned i = 0; i < sizeof(cases)/sizeof(cases[0]); ++i) { + fixture(); parameter(cases[i][0], cases[i][1]); + assert(stage() == cases[i][2]); assert(put_refs == cases[i][3]); assert(unmaps == cases[i][4]); + assert(!sends && !mailbox_reads); + } + fixture(); sep_pm_commit.sent = sep_pm_commit.answered = sep_pm_commit.gate_valid = true; + sep_pm_commit.reply_result = 0; sep_pm_commit.answer_us = 90; sep_pm_commit.state = 42; + assert(stage() == 0); assert(!sep_pm_commit.sent && !sep_pm_commit.answered); + assert(!sep_pm_commit.gate_valid && sep_pm_commit.gate_ret == -EINVAL); + assert(sep_pm_commit.reply_result == 0xff && !sep_pm_commit.answer_us && !sep_pm_commit.state); + assert(!sends && !mailbox_reads); assert(sep_pm_commit_finish(&dart) == -ENOENT); + assert(sep_pm_commit.registered); assert(sep_pm_commit_finish(&device) == 0); + assert(put_refs == 1 && unmaps == 1 && !apple_sleep_enclave.dev); + assert(sep_pm_commit_finish(&device) == -ENOENT); +} +static void test_unsent(void) { + for (int scenario = 0; scenario < 7; ++scenario) { + fixture(); assert(stage() == 0); reply(0); int expected; + switch (scenario) { + case 0: bulk_error = -EIO; expected = -EIO; break; + case 1: bulk_error = -ETIMEDOUT; expected = -ETIMEDOUT; break; + case 2: tx_ready = false; expected = -ETIMEDOUT; break; + case 3: prepare_error = -EIO; expected = -EIO; break; + case 4: pending_wake = true; expected = -EBUSY; break; + case 5: sleep_refuse_before_send = true; expected = -ECANCELED; break; + default: send_error = -ENODEV; expected = -ENODEV; break; + } + assert(sep_pm_commit_stop(&sep_pm_commit) == expected); balanced(); + assert(!sends && !sep_pm_commit.sent && !closes && !message_reads); + assert(!sep_pm_commit.answered && !sep_pm_commit.gate_valid); + assert(restores == (scenario >= 4)); assert(prepares == (scenario >= 3)); + if (scenario == 5) assert(!sleep_refuse_before_send); + assert(sep_pm_commit_finish(&device) == 0); + assert(put_refs == 1 && unmaps == 1 && !apple_sleep_enclave.dev); + } +} +static void test_sent(void) { + fixture(); assert(stage() == 0); late_wake = true; bulk_error = 3; + queue[queued++] = (struct apple_mbox_msg){0x7777, 0}; reply(0); + assert(sep_pm_commit_stop(&sep_pm_commit) == 0); balanced(); + assert(sends == 1 && closes == 1 && forwarded == 1 && message_reads == 2); + assert(pending_wake); // A wake after the send decision cannot cancel this stop. + assert(sep_pm_commit.answered && !sep_pm_commit.reply_result && sep_pm_commit.gate_valid); + assert(!sep_pm_commit.gate_ret && !restores); real_wake = true; + assert(sep_pm_commit_finish(&device) == 1); assert(put_refs == 1 && unmaps == 1); + + fixture(); assert(stage() == 0); one_error = -ENODEV; expect_stop_fatal("ret -19 "); assert(!closes); + fixture(); assert(stage() == 0); one_error = -EOPNOTSUPP; + expect_stop_fatal("ret -95 "); assert(!closes); + fixture(); assert(stage() == 0); reply(7); expect_stop_fatal("ret -121 "); + assert(sep_pm_commit.answered && sep_pm_commit.reply_result == 7 && !closes); + fixture(); assert(stage() == 0); early_gate = 1; + expect_stop_fatal("ret -112 "); assert(!sep_pm_commit.answered && !closes && !mailbox_reads); + fixture(); assert(stage() == 0); expect_stop_fatal("ret -110 "); + assert(!sep_pm_commit.answered && !closes); + fixture(); assert(stage() == 0); reply(0); close_gate = 0; + expect_stop_fatal("did not gate after send: -110"); + assert(closes == 1 && sep_pm_commit.gate_valid && sep_pm_commit.gate_ret == -ETIMEDOUT); + + fixture(); assert(stage() == 0); reply(0); assert(sep_pm_commit_stop(&sep_pm_commit) == 0); + fatal_armed = true; + if (setjmp(fatal_env) == 0) { + (void)sep_pm_commit_finish(&device); + assert(0 && "sent finish requires physical wake"); + } + fatal_armed = false; assert(strstr(fatal_message, "without a physical sleep/wake")); + assert(sep_pm_commit.registered && !put_refs && !unmaps && apple_sleep_enclave.dev); +} +static void test_provider(void) { + for (int p = 13; p <= 15; ++p) { + fixture(); parameter(p, 1); assert(!apple_mbox_poll_one_supported(&mbox)); + assert(apple_mbox_poll_one(&mbox) == -EOPNOTSUPP); + assert(!mailbox_reads && !message_reads && !forwarded); balanced(); + } + fixture(); mbox.removing = true; assert(apple_mbox_poll_one(&mbox) == -ENODEV); + assert(!mailbox_reads); balanced(); + fixture(); mbox.rx = NULL; assert(apple_mbox_poll_one(&mbox) == 0); assert(!mailbox_reads); + fixture(); assert(apple_mbox_poll_one(&mbox) == 0); + assert(mailbox_reads == 1 && !message_reads && !forwarded); balanced(); + fixture(); queue[queued++] = (struct apple_mbox_msg){0x7777, 0}; + queue[queued++] = (struct apple_mbox_msg){0x8888, 0}; + assert(apple_mbox_poll_one(&mbox) == 1); + assert(consumed == 1 && forwarded == 1 && mailbox_reads == 1 && message_reads == 1); balanced(); + assert(apple_mbox_poll_one(&mbox) == 1); + assert(consumed == 2 && forwarded == 2 && mailbox_reads == 2 && message_reads == 2); balanced(); +} +static void test_poll_contract(void) { + fixture(); assert(sep_pm_wait_gated(&device, 5) == 1); int n = reads; assert(n > 1); + fixture(); gate_after = n; assert(sep_pm_wait_gated(&device, 5) == 0 && reads == n); + fixture(); gate_after = n + 1; assert(sep_pm_wait_gated(&device, 5) == 1 && reads == n); + for (int e = 0; e < 2; ++e) { + int err = e ? -ETIMEDOUT : -EIO; + fixture(); read_error = err; assert(sep_pm_wait_gated(&device, 5) == err); + fixture(); read_error = err; read_error_after = n; + assert(sep_pm_wait_gated(&device, 5) == err && reads == n); + fixture(); read_error = err; assert(noirq() == err); + } + fixture(); domain_error = -ENOENT; assert(sep_pm_wait_gated(&device, 5) == -ENOENT); + assert(noirq() == 0); + fixture(); domain_error = -EIO; assert(noirq() == -EIO); + fixture(); assert(noirq() == -ETIMEDOUT); + fixture(); gate_after = 1; assert(noirq() == 0); + fixture(); firmware = false; assert(noirq() == 0 && !lookups); + fixture(); sep_pm_told = NULL; assert(noirq() == 0 && !lookups); + fixture(); sep_pm_told = no; assert(noirq() == 0 && !lookups); +} +int main(void) { + test_prepare_and_cleanup(); test_unsent(); test_provider(); test_sent(); test_poll_contract(); + puts("SEP C commit controls passed"); return 0; +} +#endif +''' + +RUST_PREFIX = r''' +#![allow(dead_code, unused_imports, unused_variables)] +use std::cell::{Cell, RefCell, RefMut}; +use std::ffi::{c_int, c_void}; +use std::sync::atomic::{AtomicU32, Ordering, Ordering::Relaxed}; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] struct Error(i32); +impl Error { fn from_errno(n: i32) -> Self { assert!(n < 0); Self(n) } } +type Result = std::result::Result; +const EBUSY: Error = Error(-16); const ENODEV: Error = Error(-19); +const ENOMEM: Error = Error(-12); const EIO: Error = Error(-5); +const ENOENT: Error = Error(-2); const EINVAL: Error = Error(-22); +const ETIMEDOUT: Error = Error(-110); const EOPNOTSUPP: Error = Error(-95); +macro_rules! dev_err { ($($a:tt)*) => {{}}; } +macro_rules! dev_warn { ($($a:tt)*) => {{}}; } +macro_rules! dev_dbg { ($($a:tt)*) => {{}}; } +macro_rules! dev_info { ($($a:tt)*) => {{}}; } +macro_rules! dev_err_ratelimited { ($($a:tt)*) => {{}}; } +macro_rules! dev_warn_ratelimited { ($($a:tt)*) => {{}}; } +macro_rules! dev_info_ratelimited { ($($a:tt)*) => {{}}; } +extern "C" { + fn fixture(); fn parameter(which: i32, value: i32); fn counter(which: i32) -> i32; + fn device_pointer() -> *mut c_void; fn mbox_pointer() -> *mut c_void; +} +thread_local! { + static REAL: Cell = const { Cell::new(false) }; + static EVENTS: RefCell> = const { RefCell::new(Vec::new()) }; + static RING: Cell> = const { Cell::new(Ok(())) }; +} +fn event(n: u8) { EVENTS.with(|v| v.borrow_mut().push(n)); } +mod bindings { pub unsafe fn pm_resume_via_firmware() -> bool { super::REAL.with(|v| v.get()) } } +mod kernel { + pub mod device { + pub struct Device; + impl Device { pub fn as_raw(&self) -> *mut std::ffi::c_void { unsafe { crate::device_pointer() } } } + } + pub mod bindings { #[allow(non_camel_case_types)] pub enum apple_mbox {} } + pub mod soc { pub mod apple { pub mod mailbox { + #[derive(Clone, Copy)] pub struct Message { pub msg0: u64, pub msg1: u32 } + } } } + pub mod time { + pub struct Delta; impl Delta { pub fn from_millis(_: i64) -> Self { Self } } + pub mod delay { pub fn fsleep(_: super::Delta) {} } + } +} +mod time { pub fn msecs_to_jiffies(ms: u32) -> u64 { ms.into() } } +mod proto { + pub const EP_CONTROL: u32 = 0; pub const CONTROL_TIMEOUT_MS: u32 = 5000; + pub const TAG_SLEEP: u32 = 0; pub const SLEEP_REPLY: u64 = 0x1234; + pub const SLEEP_REPLY_MASK: u64 = 0xffffff; + pub const WAKE_KICK: crate::kernel::soc::apple::mailbox::Message = + crate::kernel::soc::apple::mailbox::Message { msg0: 0x99, msg1: 0 }; + pub fn op_sleep() {} + pub fn encode_control(_: &(), _: u32) -> crate::kernel::soc::apple::mailbox::Message { + crate::kernel::soc::apple::mailbox::Message { msg0: 0x55, msg1: 0 } + } +} +mod control { + pub fn wait_idle bool, D: FnMut(u32)>(_: u32, _: u32, mut f: F, _: D) -> bool { !f() } +} +struct Lock(RefCell); +impl Lock { fn lock(&self) -> RefMut<'_, T> { self.0.borrow_mut() } } +struct Flag(Cell); +impl Flag { fn xchg(&self, v: bool, _: Ordering) -> bool { self.0.replace(v) } } +struct Control { busy: bool, wake_ok: bool } +impl Control { + fn requests_in_flight(&self) -> bool { self.busy } + fn wake_reset(&mut self) { event(1); } + fn wake_complete(&self) -> bool { self.wake_ok } +} +enum CondVarTimeoutResult { Woken { jiffies: u64 }, Signal { jiffies: u64 }, Timeout } +struct Wq; +impl Wq { + fn wait_interruptible_timeout(&self, _: &mut RefMut<'_, Control>, _: u64) -> CondVarTimeoutResult { + CondVarTimeoutResult::Timeout + } +} +struct Mbox; +impl Mbox { + fn as_raw(&self) -> *mut kernel::bindings::apple_mbox { unsafe { mbox_pointer().cast() } } + fn set_recv_enabled(&self, enabled: bool) { assert!(enabled); event(3); } + fn hold_rx(&self) { event(5); } + fn release_rx(&self) { event(2); } +} +const PHASE_READY: u32 = 1; +struct SepData { + dev: kernel::device::Device, phase: AtomicU32, control: Lock, mbox: Lock>, + gate: Cell, sleep_rx_held: Flag, endpoint: Cell, control_wq: Wq, +} +impl SepData { + fn new() -> Self { + Self { dev: kernel::device::Device, phase: AtomicU32::new(PHASE_READY), + control: Lock(RefCell::new(Control { busy: false, wake_ok: true })), + mbox: Lock(RefCell::new(Some(Mbox))), gate: Cell::new(pm::SleepGate::Open), + sleep_rx_held: Flag(Cell::new(false)), endpoint: Cell::new(true), control_wq: Wq } + } + fn endpoint_present(&self, _: u32) -> bool { self.endpoint.get() } + fn sleep_gate(&self) -> pm::SleepGate { self.gate.get() } + fn set_sleep_gate(&self, g: pm::SleepGate) { self.gate.set(g); } + fn ring(_: &Mbox, msg: kernel::soc::apple::mailbox::Message) -> Result<()> { + assert_eq!(msg.msg0, proto::WAKE_KICK.msg0); event(4); RING.with(|v|v.get()) + } +} +''' + +RUST_TESTS = r''' +fn reset() { + unsafe { fixture() }; REAL.with(|v|v.set(false)); + EVENTS.with(|v|v.borrow_mut().clear()); RING.with(|v|v.set(Ok(()))); +} +fn param(n: i32, v: i32) { unsafe { parameter(n, v) } } +fn count(n: i32) -> i32 { unsafe { counter(n) } } +fn events() -> Vec { EVENTS.with(|v|v.borrow().clone()) } +fn clear_events() { EVENTS.with(|v|v.borrow_mut().clear()); } +fn real_wake() { REAL.with(|v|v.set(true)); param(12,1); } +#[test] +fn staging_errors_preserve_errno_and_cleanup() { + for (p,v,e,puts,unmaps) in [ + (4,0,ENOMEM,1,0),(3,0,ENODEV,0,0),(5,-2,ENOENT,0,0), + (9,1,EBUSY,0,0),(11,0,ENODEV,1,0),(16,-5,EIO,0,0), + (17,-22,EINVAL,0,0),(18,3,EINVAL,0,0),(19,1,EBUSY,1,1), + (7,0,EOPNOTSUPP,1,1), + ] { + reset(); param(p,v); let s=SepData::new(); + assert_eq!(s.sleep_enter(),Err(e)); assert_eq!(s.gate.get(),pm::SleepGate::Open); + assert!(!s.sleep_rx_held.0.get()); assert_eq!(events(),Vec::::new()); + assert_eq!(count(0),puts); assert_eq!(count(2),unmaps); assert_eq!(count(6),0); + } +} +#[test] +fn unsupported_is_refused_without_early_sleep() { + for p in [0,1,2,13,14,15] { + reset(); param(p,if p < 3 {0} else {1}); let s=SepData::new(); + assert_eq!(s.sleep_enter(),Err(EOPNOTSUPP)); + assert_eq!(s.gate.get(),pm::SleepGate::Open); assert!(!s.sleep_rx_held.0.get()); + assert_eq!(count(6),0); assert_eq!(count(7),0); assert_eq!(count(1),0); + assert_eq!(count(4),0); assert!(events().is_empty()); + } +} +#[test] +fn staging_holds_rx_and_unsent_unwind_only_releases() { + reset(); let s=SepData::new(); assert_eq!(s.sleep_enter(),Ok(())); + assert_eq!(s.gate.get(),pm::SleepGate::Draining); assert!(s.sleep_rx_held.0.get()); + assert_eq!(count(3),1); assert_eq!(count(6),0); assert_eq!(events(),vec![5]); + s.sleep_exit(); assert_eq!(s.gate.get(),pm::SleepGate::Open); + assert!(!s.sleep_rx_held.0.get()); assert_eq!(events(),vec![5,2]); + assert_eq!(count(0),1); assert_eq!(count(2),1); assert_eq!(count(3),0); + assert_eq!(pm::commit_finish(&s.dev),None); + s.sleep_exit(); assert_eq!(events(),vec![5,2]); +} +#[test] +fn admission_and_missing_mailbox_refuse_before_staging() { + for mode in 0..6 { + reset(); let s=SepData::new(); + match mode { + 0 => s.phase.store(0,Relaxed), 1 => s.endpoint.set(false), + 2 => s.control.lock().busy=true, 3 => *s.mbox.lock()=None, + 4 => s.gate.set(pm::SleepGate::Draining), _ => s.gate.set(pm::SleepGate::Told), + } + assert_eq!(s.sleep_enter(),Err(if mode==3 {ENODEV} else {EBUSY})); + assert_eq!(count(1),0); assert_eq!(count(6),0); assert!(!s.sleep_rx_held.0.get()); + assert!(events().is_empty()); + if mode < 4 { assert_eq!(s.gate.get(),pm::SleepGate::Open); } + } +} +#[test] +fn only_physical_wake_runs_the_actual_kick_in_arm_order() { + reset(); let s=SepData::new(); assert_eq!(s.sleep_enter(),Ok(())); + // Current-attempt sent guard is exercised only in C, with no Rust frames. + param(20,1); real_wake(); clear_events(); s.sleep_exit(); + assert_eq!(events(),vec![1,2,3,4]); assert_eq!(s.gate.get(),pm::SleepGate::Open); + assert!(!s.sleep_rx_held.0.get()); assert_eq!(count(3),0); + reset(); let s=SepData::new(); s.gate.set(pm::SleepGate::Told); + s.sleep_rx_held.0.set(true); s.sleep_exit(); + assert_eq!(s.gate.get(),pm::SleepGate::Lost); assert!(s.sleep_rx_held.0.get()); + assert!(events().is_empty()); assert_eq!(count(5),0); assert_eq!(count(0),0); +} +#[test] +fn failed_real_wake_stays_lost_and_refuses_new_firmware_sleep() { + for mode in 0..4 { + reset(); let s=SepData::new(); s.gate.set(pm::SleepGate::Told); + s.sleep_rx_held.0.set(true); real_wake(); + match mode { + 0 => param(6,-5), 1 => s.control.lock().wake_ok=false, + 2 => RING.with(|v|v.set(Err(EIO))), _ => *s.mbox.lock()=None, + } + s.sleep_exit(); assert_eq!(s.gate.get(),pm::SleepGate::Lost); + assert!(s.sleep_rx_held.0.get()); + if mode==0 { assert!(events().is_empty()); } // clear error precedes RX release + clear_events(); assert_eq!(s.sleep_enter(),Err(EIO)); assert!(events().is_empty()); + assert_eq!(count(6),0); + REAL.with(|v|v.set(false)); param(12,0); let reads=count(5); let refs=count(0); s.sleep_exit(); + assert_eq!(count(5),reads); assert_eq!(count(0),refs); + assert_eq!(s.gate.get(),pm::SleepGate::Lost); assert!(s.sleep_rx_held.0.get()); + assert!(events().is_empty()); + } + reset(); let s=SepData::new(); s.gate.set(pm::SleepGate::Lost); + s.sleep_rx_held.0.set(true); assert_eq!(s.sleep_enter(),Err(EIO)); + s.sleep_exit(); assert!(events().is_empty()); assert!(s.sleep_rx_held.0.get()); + assert_eq!(count(5),0); assert_eq!(count(0),0); +} +#[test] +fn lost_and_gate_permissions_are_preserved() { + for gate in [pm::SleepGate::Open,pm::SleepGate::Draining,pm::SleepGate::Told,pm::SleepGate::Lost] { + assert_eq!(pm::SleepGate::from_u32(gate as u32),gate); + assert_eq!(gate.may_request(),gate==pm::SleepGate::Open); + assert_eq!(gate.may_reply(),matches!(gate,pm::SleepGate::Open|pm::SleepGate::Draining)); + assert_eq!(gate.needs_wake(),matches!(gate,pm::SleepGate::Told|pm::SleepGate::Lost)); + } +} +''' + + +def sources(source, contracts, mutation): + """Assemble source only. Compilation/execution belongs exclusively to main.""" + pm = (source / 'drivers/soc/apple/pm.rs').read_text() + sep = (source / 'drivers/soc/apple/sep.rs').read_text() + shim = (source / 'drivers/soc/apple/pm_shim.c').read_text() + mailbox = (source / 'drivers/soc/apple/mailbox.c').read_text() + sleep = (source / 'drivers/soc/apple/sleep.c').read_text() + iopoll = (contracts / 'include/linux/iopoll.h').read_text() + names = { + 'shim': ['sep_pm_wait_gated', 'sep_pm_clear_dev_disable', 'sep_pm_dart_restore', + 'sep_pm_commit_rx', 'sep_pm_commit_take_rx', 'sep_pm_commit_stop', + 'sep_pm_commit_prepare', 'sep_pm_commit_finish', 'sep_pm_suspend_noirq'], + 'mailbox': ['apple_mbox_poll_one_supported', 'apple_mbox_poll_one'], + 'sleep': ['apple_sleep_commit_stops_coprocs', 'apple_sleep_enclave_register', + 'apple_sleep_enclave_unregister'], + 'pm': ['leave_for_commit', 'commit_finish', 'clear_dev_disable', 'dart_restore'], + 'sep': ['sleep_enter', 'sleep_leave_for_commit', 'sleep_hold_rx', 'sleep_release_rx', + 'sleep_exit', 'sleep_kick'], + } + texts = dict(shim=shim, mailbox=mailbox, sleep=sleep, pm=pm, sep=sep) + bodies = {n: function(texts[k], n) for k, group in names.items() for n in group} + if mutation: + name, before, after = MUTATIONS[mutation] + bodies[name] = replace_once(bodies[name], before, after) + macros = ['poll_timeout_us', 'read_poll_timeout', 'poll_timeout_us_atomic', + 'read_poll_timeout_atomic', 'readx_poll_timeout_atomic', 'readl_poll_timeout_atomic'] + c = C_PREFIX + '\n' + '\n'.join(macro(iopoll, n) for n in macros) + c += '\n' + '\n'.join(re.search(r'^#define ' + n + r'\s+.*$', shim, re.M).group() + for n in ['SEP_PM_POLL_US', 'SEP_PM_TIMEOUT_US']) + start = shim.index('static struct sep_pm_commit {') + c += '\n' + shim[start:shim.index('} sep_pm_commit;', start) + len('} sep_pm_commit;')] + # Use the actual provider helper, with only removal/capability dependency state + # injected by send for the post-send negative-poll scenarios. + c += '\n' + '\n'.join(bodies[n] for group in ['mailbox', 'sleep', 'shim'] for n in names[group]) + c += '\n' + C_CONTROLS + gate_start = pm.index('#[derive(Clone, Copy, Debug, PartialEq, Eq)]') + gate_end = pm.index('/// Stages `request`', gate_start) + rust = RUST_PREFIX + '\nmod pm {\nuse crate::*;\n' + rust += '''extern "C" { + fn sep_pm_commit_prepare(dev:*mut c_void, mbox:*mut c_void, a:u64, b:u32, c:u64, d:u64)->c_int; + fn sep_pm_commit_finish(dev:*mut c_void)->c_int; + fn sep_pm_clear_dev_disable(dev:*mut c_void)->c_int; + fn sep_pm_dart_restore(dev:*mut c_void); + } +''' + rust += pm[gate_start:gate_end] + rust += '\n'.join(bodies[n] for n in names['pm']) + '\n}\nimpl SepData {\n' + rust += '\n'.join(bodies[n] for n in names['sep']) + '\n}\n' + RUST_TESTS + return c, rust + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--source', type=Path, default=ROOT) + parser.add_argument('--contracts', type=Path) + parser.add_argument('--mutation', choices=MUTATIONS) + args = parser.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + args.out.mkdir(parents=True, exist_ok=True) + c, rust = sources(args.source, args.contracts or args.source, args.mutation) + cpath, rpath = args.out / 'control.c', args.out / 'control.rs' + obj, cbinary, binary = args.out / 'control.o', args.out / 'control-c', args.out / 'control' + cpath.write_text(c) + rpath.write_text(rust) + cc = ['cc', '-std=gnu11', '-Wall', '-Wextra', '-Wno-unused-parameter'] + # Finish all compilation before executing controls: setup failures cannot kill mutants. + subprocess.run(cc + ['-c', str(cpath), '-o', str(obj)], check=True) + subprocess.run(cc + ['-DCONTROL_MAIN', str(cpath), '-o', str(cbinary)], check=True) + subprocess.run(['rustc', '--edition=2021', '--test', str(rpath), '-C', + f'link-arg={obj.resolve()}', '-o', str(binary)], check=True) + for command in [[str(cbinary.resolve())], [str(binary.resolve()), '--test-threads=1']]: + result = subprocess.run(command, check=False) + if result.returncode: + # Keep the assertion output, without a Python traceback masking a real kill. + raise SystemExit(result.returncode) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 44bd56c0f8e25..87451bf113314 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -159,6 +159,9 @@ struct apple_pmp_report { /* Requests held until the PMP runs; T6030 does not defer them. */ u64 shadow, known; bool defer; + /* Firmware-driven sleep: the request word at its start, the DVFS words kept. */ + u64 sleep_request; + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; struct apple_pmp_ane ane; @@ -628,6 +631,11 @@ static void *devm_kzalloc(struct device *d, size_t n, int g) allocations[allocation_count++] = v; return v; } + +static void *devm_kcalloc(struct device *d, size_t n, size_t size, int g) +{ + return devm_kzalloc(d, n * size, g); +} #define kzalloc_obj(v) calloc(1, sizeof(v)) #define kfree free static char *kstrdup(const char *s, int g) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index 69f832bff9cb3..468edbf9446a1 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -126,6 +126,8 @@ static bool passive_present, passive_lockable, passive_data; static const char *machine; static const void *pmp_data; static struct device *pmp_device; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; diff --git a/tools/aurora-wake/bt-wake-cycle.sh b/tools/aurora-wake/bt-wake-cycle.sh index 3c10162256e97..2eb419b345823 100755 --- a/tools/aurora-wake/bt-wake-cycle.sh +++ b/tools/aurora-wake/bt-wake-cycle.sh @@ -17,9 +17,21 @@ set -u +# The PMU RTC, which can wake the system: rtc0 on J700, rtc1 on others. +pmu_rtc() { + local name + for name in /sys/class/rtc/rtc*/name; do + [[ $(cat "$name" 2> /dev/null) == apple-pmu-rtc* ]] || continue + name=${name%/name} + echo "${name##*/}" + return + done + echo rtc0 +} + mode= secs=120 -rtc=rtc1 +rtc=$(pmu_rtc) addr= out=/var/tmp/bt-wake/cycle-$(date +%Y%m%d-%H%M%S) bdf=0000:01:00.1 diff --git a/tools/aurora-wake/bt-wake-power.sh b/tools/aurora-wake/bt-wake-power.sh index af5d49dadb216..be6d1eb06c8f6 100755 --- a/tools/aurora-wake/bt-wake-power.sh +++ b/tools/aurora-wake/bt-wake-power.sh @@ -9,6 +9,17 @@ # - the battery energy used, when running on battery (energy_now). # Usage: bt-wake-power.sh [-s secs] [-n pairs] [-o outdir] set -u +# The PMU RTC, which can wake the system: rtc0 on J700, rtc1 on others. +pmu_rtc() { + local name + for name in /sys/class/rtc/rtc*/name; do + [[ $(cat "$name" 2> /dev/null) == apple-pmu-rtc* ]] || continue + name=${name%/name} + echo "${name##*/}" + return + done + echo rtc0 +} secs=300 pairs=2 out=/var/tmp/bt-wake/power-$(date +%Y%m%d-%H%M%S) @@ -19,6 +30,7 @@ done dev=/sys/bus/pci/devices/0000:01:00.1 bat=/sys/class/power_supply/macsmc-battery irq=$(awk -F: '/mt7932-bt/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) +rtc=$(pmu_rtc) mkdir -p "$out" orig=$(cat "$dev/power/wakeup") trap 'echo "$orig" > "$dev/power/wakeup"' EXIT @@ -42,7 +54,7 @@ for p in $(seq 1 "$pairs"); do e0=$(cat "$bat/energy_now") st=$(cat "$bat/status") t0=$(date +%s.%N) - rtcwake -d rtc1 -m freeze -s "$secs" > /dev/null 2>&1 + rtcwake -d "$rtc" -m freeze -s "$secs" > /dev/null 2>&1 t1=$(date +%s.%N) b1=$(irq_sum "$irq"); a1=$(irq_sum all); e1=$(cat "$bat/energy_now") dmesg > "$out/dmesg-$p-$mode.txt" diff --git a/tools/aurora-wake/s2idle-rtc-regression.sh b/tools/aurora-wake/s2idle-rtc-regression.sh index 89a131aca0b59..74697560040db 100755 --- a/tools/aurora-wake/s2idle-rtc-regression.sh +++ b/tools/aurora-wake/s2idle-rtc-regression.sh @@ -3,7 +3,7 @@ # # Unattended suspend-to-idle regression test for Apple silicon (J700 and others). # -# Suspends with an RTC alarm (rtc1, apple-pmu-rtc, by default) several times and +# Suspends with an RTC alarm (apple-pmu-rtc by default) several times and # checks after every resume that: # - the time actually spent suspended (CLOCK_BOOTTIME minus CLOCK_MONOTONIC) # lasted until the alarm: no spurious wakeup, e.g. from a running xHC, @@ -24,9 +24,21 @@ set -u +# The PMU RTC, which can wake the system: rtc0 on J700, rtc1 on others. +pmu_rtc() { + local name + for name in /sys/class/rtc/rtc*/name; do + [[ $(cat "$name" 2> /dev/null) == apple-pmu-rtc* ]] || continue + name=${name%/name} + echo "${name##*/}" + return + done + echo rtc0 +} + cycles=3 secs=20 -rtc=rtc1 +rtc=$(pmu_rtc) out= ignore= dry_run=0 diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py index 14c837ceeb5f3..95c02e87ddc02 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -19,7 +19,8 @@ def oslog(root): #define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctzll(mask)) #define APPLE_RTKIT_EP_OSLOG 8 struct buffer { unsigned sentinel; unsigned size; }; -struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack; }; +struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack, oslog_echo; u64 oslog_owed; }; +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) static unsigned warnings, notifications, requests, acks, errors; static u64 observed; static bool neo, m5; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 6c140662b6967..f7eafd88b68ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -146,6 +146,15 @@ def atc_restore_helpers(source): return atc_function(source, 'atcphy_setup_pipehandler') +def atc_common_state_helpers(source): + """What atcphy_restore_after_pd_off() calls to drop the common block's DisplayPort and + tunnel state after a power-down. Older sources do that inline: ''. + """ + if 'atcphy_forget_common_state(' not in atc_function(source, 'atcphy_restore_after_pd_off'): + return '' + return atc_function(source, 'atcphy_forget_common_state') + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 4c9f0ac4faad6..67d07ef2c2a63 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -86,12 +86,21 @@ struct dwc3_apple { void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending, shutting_down; + /* Stopped for a deep sleep, and the state to bring back. */ + bool sleep_stopped; + enum dwc3_apple_state sleep_state; + /* Votes sleep DDR while the host is up; the fixtures leave it off. */ + bool sleep_ddr; bool rpm_held, host_only; /* T8132 only: runtime PM before the first core probe; never set here. */ bool pre_core_rpm; bool dt_dis_u3_susphy; unsigned long usb2_power_off_pending, usb3_power_off_pending; }; +static void __attribute__((unused)) apple_sleep_usb_slp_ddr(bool on) +{ + (void)on; +} static unsigned int polls, fail_poll, reset_calls, reset_writes, core_writes; static unsigned int host_exits, clocks_disabled, callback_calls; static unsigned int gadget_exits, event_cleanups, legacy_suspends, @@ -329,6 +338,11 @@ static void dwc3_apple_setup_cio(struct dwc3_apple *a) (void)a; setup_cio++; } +/* The hibernation block's register setup; teardown order doesn't depend on it. */ +static void __attribute__((unused)) dwc3_apple_setup_hibernation(struct dwc3_apple *a) +{ + (void)a; +} static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, enum phy_mode *m) { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 5a49309b8f06e..a495eac79479b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -793,6 +793,8 @@ def poweroff_replug(root): { (void)d; (void)generation; return 0; } static void dcp_external_crtc_released(struct apple_dcp *d) { (void)d; } static bool dcp_uses_soft_dpms(struct apple_dcp *d) { return d->soft_dpms; } +/* The T8140 display power cycle has its own test on hardware; here it only has to be reached. */ +static void dcp_dpms_power_off(struct apple_dcp *d) { (void)d; } static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return !d->typec_output; } static void __attribute__((unused)) iomfb_blank_h17p(struct apple_dcp *d) { (void)d; assert(!"unexpected internal-panel blank in external-link test"); } @@ -1069,6 +1071,9 @@ def fn(n, name): for name in ('atcphy_mode_configuration', 'atcphy_hw', 'atcphy_dp_link_rate_configuration'): decls.append(block(files['atc'], files['atc'].index('struct ' + name + ' {')) + ';') decls.append(block(files['state'], files['state'].index('enum dwc3_apple_state {')) + ';') + role = 'static inline enum dwc3_apple_state dwc3_apple_state_for_role(' + if role in files['state']: + decls.append(block(files['state'], files['state'].index(role))) start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') end = files['atc'].index('\n};', start) + 3 decls.append(files['atc'][start:end]) @@ -1088,7 +1093,7 @@ def fn(n, name): 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', - 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_restore_after_pd_off', + 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_forget_common_state', 'atcphy_restore_after_pd_off', 'atcphy_usb2_init', 'atcphy_usb3_power_off', 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), @@ -1196,6 +1201,7 @@ def block(marker): int iomfb_method_profile,firmware_compat; const struct resource *firmware_clock; u32 firmware_display_windows,firmware_scratch,firmware_request; + bool s2r,s2r_ext; }; struct apple_dcp { struct device *dev; struct apple_dcp_hw_data hw; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index a56a3a3ccfbff..b2e1d718404da 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -4,7 +4,7 @@ from pathlib import Path import re from common import (block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE, - atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, + atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, atc_common_state_helpers, atc_cio_helpers) def parking(root): @@ -36,6 +36,8 @@ def function(source, name): /* T8132-only paths are covered by drivers/apple/atc-t8132-usb3.py. */ bool t8132_ss, t8132_common_owned, t8132_failed, cio_tunables; struct { void *usb3_dp; } tunables; + /* A sleep took the USB host's domains down; never set here. */ + bool sleep_dead; const struct atcphy_hw *hw; struct { unsigned char *core, *lpdptx, *pipehandler, *usb2phy; } regs; @@ -181,6 +183,7 @@ def function(source, name): parts.append(function(atc, name)) parts.append(dp) parts.append(atc_restore_helpers(atc)) + parts.append(atc_common_state_helpers(atc)) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_set_orientation', diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 92ccae9ed052f..64889dc06683c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -428,6 +428,8 @@ def ownership(root, usb_unreachable=False): helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n' helpers+=''.join(atc_function(s,name) for name in ('atcphy_park_retry_delay','atcphy_park_retry','atcphy_park_due','atcphy_mux_change')) + # The mux callback's body, split out so the wake can set the mode it had. + helpers+=''.join(atc_function(s,name) for name in ('atcphy_mux_done','atcphy_mux_update_to')) helpers+=ns['function'](s,'atcphy_mux_update')+'\n' # Lane fixtures provide the target mapping and locked body; use the real callback here. # The mux callback powers an idle-power-off PHY first; this PHY is not one. @@ -495,7 +497,7 @@ def function(text, name): mocks = r''' struct cd321x { struct tps6598x tps; - struct work update_work,resume_work; + struct work update_work,resume_work,rearm_work; int link_event,ready_event,restore_event; struct cd321x_pm_state pm; void *mux,*tbt_switch,*port_altmode_dp,*port_altmode_tbt; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index b293087d4917c..0ad3420e6fdb1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -95,7 +95,8 @@ def unit_for(source): struct dwc3 { bool sys_wakeup, dis_u3_susphy_quirk; struct platform_device *xhci; }; enum { DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_HOST, DWC3_APPLE_DEVICE }; /* pre_core_rpm is T8132 only: runtime PM before the first core probe; never set here. */ -struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy; }; +struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy, shutting_down, sleep_ddr; }; +static void __attribute__((unused)) apple_sleep_usb_slp_ddr(bool on) { (void)on; } static struct dwc3_apple apple_context; #define to_dwc3_apple(d) container_of(d,struct dwc3_apple,dwc) static bool inside_host_remove; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index e1d18f02a425d..7a2919f377b17 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -59,7 +59,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): actual=actual.replace('static struct apple_dcp_typec_port *sole_port(void) { return &port; }', 'static struct apple_dcp_typec_port *sole_port(void) { return &port; }\nstatic struct apple_dcp_typec_route * __attribute__((unused)) sole_route(void) { return &route; }',1) actual=actual.replace('bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;', 'struct typec_altmode *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;',1) - actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel;',1) + actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel,sleep_parked;',1) extra=r''' /* actual DCP leaves */ #define WRITE_ONCE(x,v) ((x)=(v)) diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c index 806148f64d1bf..8154098ee4157 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -19,6 +19,8 @@ struct device { void *of_node; }; static struct device owner; static struct device *pmp_device = &owner; static const void *pmp_data = &owner; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static bool t6030, requested; static int ready, check, power_result[2], report_result[3], powers, reports; static char calls[16];