From d9b438c4ca41fec09a61fc5c15ad3ccda3402ccd Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Tue, 10 Nov 2026 08:21:11 -0500 Subject: [PATCH 01/29] arm64: apple: support coordinated suspend-to-RAM on T8140 Coordinate device preparation, irreversible coprocessor stop and the EL2 reset-return mailbox. Recover the CPU, interrupt, memory, storage, display, radio, USB and audio state on a real firmware return. Keep pre-send SEP cancellation returnable and reject unsafe post-send unwind. Retain explicit s2idle and selected deep mode on readiness refusal. Preserve shared cold-probe and M5 integration behavior; unqualified T6050 GPU sleep restart remains disabled. Accept only SPMI parent dispatches whose original pending-bank samples are all empty, matching the native no-work contract. Preserve existing handling and diagnostics for every nonempty sampled set. This does not classify the historical IRQ64 events or claim hardware qualification. Signed-off-by: Diljit Singh --- .github/workflows/sep-source.yml | 41 + Documentation/PCI/controller/apple-t8140.rst | 13 + .../bindings/display/apple,dcp.yaml | 12 +- .../display/apple,t8140-display-power.yaml | 54 + .../devicetree/bindings/dma/apple,sio.yaml | 9 + .../devicetree/bindings/gpu/apple,agx.yaml | 7 + .../devicetree/bindings/iommu/apple,dart.yaml | 76 +- .../bindings/mailbox/apple,mailbox.yaml | 20 +- .../bindings/media/apple,t8140-isp.yaml | 107 +- .../devicetree/bindings/pci/apple,pcie.yaml | 50 +- .../devicetree/bindings/phy/apple,atcphy.yaml | 17 + .../bindings/power/apple,pmgr-pwrstate.yaml | 8 + .../soc/apple/apple,cluster-pwrgate.yaml | 47 + .../bindings/soc/apple/apple,t8140-sleep.yaml | 101 + .../bindings/sound/apple,t8140-aop-audio.yaml | 4 + .../devicetree/bindings/usb/apple,dwc3.yaml | 7 + .../bindings/usb/apple,sn201202x.yaml | 18 + MAINTAINERS | 7 + .../boot/dts/apple/t8140-j700-dcpext.dtsi | 8 +- arch/arm64/boot/dts/apple/t8140-j700.dts | 86 +- .../boot/dts/apple/t8140-peripherals.dtsi | 11 +- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 25 +- arch/arm64/boot/dts/apple/t8140.dtsi | 241 +- arch/arm64/configs/asahi.config | 3 +- arch/arm64/include/asm/cpu_ops.h | 53 + arch/arm64/include/asm/cpufeature.h | 1 + arch/arm64/kernel/cpu_ops.c | 19 + arch/arm64/kernel/cpufeature_impdef.c | 18 + arch/arm64/kernel/hibernate.c | 3 +- arch/arm64/kernel/machine_kexec.c | 3 +- arch/arm64/kernel/smp_spin_table.c | 285 ++ arch/arm64/kernel/suspend.c | 8 + drivers/accel/ane/ane_fw_cold.c | 96 +- drivers/bluetooth/mt7932_bt_pcie.c | 216 +- drivers/bluetooth/mt7932_bt_pcie_wake_test.c | 46 + drivers/cpufreq/apple-soc-cpufreq.c | 46 + drivers/dma/apple-admac.c | 70 +- drivers/dma/apple-sio-boot-test.c | 24 + drivers/dma/apple-sio-probe-test.c | 2 + drivers/dma/apple-sio.c | 106 +- drivers/edac/apple_soc_edac.c | 103 +- drivers/gpu/drm/apple/apple_drv.c | 105 +- drivers/gpu/drm/apple/audio.c | 19 +- drivers/gpu/drm/apple/dcp-fabric.c | 54 +- drivers/gpu/drm/apple/dcp-fabric.h | 4 + drivers/gpu/drm/apple/dcp-internal.h | 43 + drivers/gpu/drm/apple/dcp.c | 1474 ++++++- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 38 +- drivers/gpu/drm/apple/iomfb.h | 2 + drivers/gpu/drm/apple/iomfb_h17p.c | 18 +- drivers/gpu/drm/apple/iomfb_template.c | 116 + drivers/gpu/drm/apple/iomfb_template.h | 4 + drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/driver.rs | 68 + drivers/gpu/drm/asahi/g15/mod.rs | 4 + drivers/gpu/drm/asahi/g16/mod.rs | 4 + drivers/gpu/drm/asahi/g17/mod.rs | 90 +- drivers/gpu/drm/asahi/g17/port.rs | 5 + drivers/gpu/drm/asahi/g17/queue/render.rs | 4 + .../gpu/drm/asahi/g17/queue/render/parked.rs | 6 + drivers/gpu/drm/asahi/g17/reports.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 77 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 22 + drivers/gpu/drm/asahi/g17/sleep.c | 356 ++ drivers/gpu/drm/asahi/g17/sleep.h | 23 + drivers/gpu/drm/asahi/g17/sleep.rs | 820 ++++ drivers/gpu/drm/asahi/hw/t6040.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 1 + drivers/gpu/drm/asahi/hw/t6050.rs | 3 + drivers/gpu/drm/asahi/hw/t8140.rs | 1 + drivers/gpu/drm/asahi/regs.rs | 51 + drivers/hid/dockchannel-hid/dockchannel-hid.c | 42 +- drivers/input/misc/macsmc-input.c | 113 +- drivers/iommu/apple-dart.c | 726 +++- drivers/irqchip/irq-apple-aic.c | 183 + .../media/platform/apple/isp/isp-cam-test.c | 56 + drivers/media/platform/apple/isp/isp-cam.c | 11 +- drivers/media/platform/apple/isp/isp-cmd.c | 55 +- drivers/media/platform/apple/isp/isp-cmd.h | 18 + drivers/media/platform/apple/isp/isp-drv.c | 77 +- drivers/media/platform/apple/isp/isp-drv.h | 19 +- drivers/media/platform/apple/isp/isp-fw.c | 204 +- drivers/media/platform/apple/isp/isp-v4l2.c | 2 + drivers/mfd/macsmc-test.c | 177 + drivers/mfd/macsmc.c | 352 +- drivers/net/wireless/mediatek/mt7932/Makefile | 3 +- .../net/wireless/mediatek/mt7932/frontend.c | 4 +- .../net/wireless/mediatek/mt7932/lifecycle.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.c | 249 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 24 + drivers/net/wireless/mediatek/mt7932/power.c | 2 +- .../net/wireless/mediatek/mt7932/startup.c | 10 +- .../net/wireless/mediatek/mt7932/wow-test.c | 221 + drivers/net/wireless/mediatek/mt7932/wow.c | 199 + drivers/net/wireless/mediatek/mt7932/wow.h | 157 + drivers/nvme/host/apple.c | 311 +- .../pcie-apple-t8140-bootstrap-state.h | 10 +- .../pcie-apple-t8140-bootstrap-test.c | 20 +- .../controller/pcie-apple-t8140-bootstrap.c | 156 +- .../controller/pcie-apple-t8140-bootstrap.h | 24 + drivers/pci/controller/pcie-apple.c | 774 ++++ drivers/phy/apple/atc-test.c | 35 + drivers/phy/apple/atc.c | 835 +++- drivers/pinctrl/pinctrl-apple-gpio.c | 71 + drivers/pmdomain/apple/pmc.c | 36 + drivers/pmdomain/apple/pmgr-pwrstate.c | 125 + drivers/pmdomain/apple/pmp-report-test.c | 11 + .../pmdomain/apple/pmp-report-validation.h | 9 + drivers/pmdomain/apple/pmp-report.c | 248 +- drivers/power/supply/macsmc-power.c | 19 + drivers/rtc/Kconfig | 12 +- drivers/rtc/rtc-apple-pmu.c | 72 +- drivers/soc/apple/Kconfig | 49 +- drivers/soc/apple/Makefile | 5 + drivers/soc/apple/aop.rs | 257 +- drivers/soc/apple/apple-cluster-pwrgate.c | 50 + drivers/soc/apple/control.rs | 92 +- drivers/soc/apple/mailbox.c | 225 +- drivers/soc/apple/mcc.c | 42 + drivers/soc/apple/pm.rs | 180 +- drivers/soc/apple/pm_shim.c | 449 ++- drivers/soc/apple/pmp-export.c | 155 + drivers/soc/apple/pmp.rs | 195 +- drivers/soc/apple/pmp_bootargs.rs | 14 + drivers/soc/apple/proto.rs | 68 + drivers/soc/apple/rtkit-helper.c | 129 +- drivers/soc/apple/rtkit-internal.h | 13 + drivers/soc/apple/rtkit.c | 602 ++- drivers/soc/apple/sart.c | 103 + drivers/soc/apple/sensor_shim.c | 152 +- drivers/soc/apple/sensor_shim_test.c | 169 + drivers/soc/apple/sep.rs | 228 +- drivers/soc/apple/shim.h | 10 +- drivers/soc/apple/sleep-test.c | 156 + drivers/soc/apple/sleep.c | 3563 +++++++++++++++++ drivers/spmi/spmi-apple-controller.c | 110 +- drivers/tty/serial/apple_dockchannel.h | 13 +- .../tty/serial/apple_dockchannel_early_uart.c | 4 +- drivers/tty/serial/apple_dockchannel_uart.c | 5 +- drivers/usb/core/hcd.c | 25 +- drivers/usb/dwc3/core.h | 3 + drivers/usb/dwc3/dwc3-apple-state-test.c | 9 + drivers/usb/dwc3/dwc3-apple-state.h | 19 + drivers/usb/dwc3/dwc3-apple.c | 529 ++- drivers/usb/dwc3/host.c | 12 + drivers/usb/host/xhci-plat.c | 12 +- drivers/usb/typec/tipd/core.c | 287 +- drivers/usb/typec/tipd/spmi.c | 7 +- drivers/usb/typec/tipd/tps6598x.h | 11 + include/linux/irqchip/irq-apple-aic.h | 17 + include/linux/mfd/macsmc.h | 11 + include/linux/phy/apple-atc.h | 79 + include/linux/soc/apple/dart.h | 68 + include/linux/soc/apple/mailbox.h | 15 + include/linux/soc/apple/pci-apple-piodma.h | 14 + include/linux/soc/apple/pmp-report.h | 30 + include/linux/soc/apple/pmp.h | 20 + include/linux/soc/apple/rtkit.h | 104 + include/linux/soc/apple/sart.h | 6 + include/linux/soc/apple/sleep.h | 269 ++ include/linux/suspend.h | 29 + include/linux/usb/hcd.h | 1 + kernel/power/suspend.c | 12 + rust/helpers/helpers.c | 1 + rust/helpers/suspend.c | 13 + rust/kernel/platform.rs | 6 + rust/kernel/soc/apple/mailbox.rs | 23 + rust/kernel/soc/apple/rtkit.rs | 26 + scripts/ci/apple-dwc3-atc-compile.sh | 128 + sound/soc/apple/aop_audio.rs | 34 + sound/soc/apple/aop_t8140.rs | 73 +- sound/soc/apple/mca.c | 32 + tools/asahi/aop-bootargs-tests.py | 22 +- tools/asahi/apple-dart-dapf-tests.py | 148 + tools/asahi/apple-dart-fw/fixture.c | 2 +- tools/asahi/apple-dart-range-tests.py | 4 +- tools/asahi/display26-tests.py | 41 +- .../asahi/mt7932-association-events-tests.py | 5 + tools/asahi/mtp-pm-tests.py | 8 +- tools/asahi/neo-stability-tests.py | 3 +- tools/asahi/pmu-rtc-tests.py | 66 +- tools/asahi/rtkit-oslog-tests.py | 5 + tools/asahi/rtkit-retained-tests.py | 25 +- tools/asahi/run-host-tests.py | 13 +- tools/asahi/sep-suspend-errors-tests.py | 733 ++++ tools/asahi/t6030-pmp-graph-tests.c | 8 + tools/asahi/t6030-pmp-supplier-tests.c | 2 + tools/aurora-wake/bt-wake-cycle.sh | 14 +- tools/aurora-wake/bt-wake-power.sh | 14 +- tools/aurora-wake/s2idle-rtc-regression.sh | 16 +- .../selftests/drivers/gpu/apple-dcp/boot.py | 3 +- .../selftests/drivers/gpu/apple-dcp/common.py | 9 + .../drivers/gpu/apple-dcp/dwc3/mocks.c | 14 + .../drivers/gpu/apple-dcp/lifecycle.py | 8 +- .../selftests/drivers/gpu/apple-dcp/phy.py | 5 +- .../selftests/drivers/gpu/apple-dcp/pm.py | 4 +- .../drivers/gpu/apple-dcp/teardown_pm.py | 3 +- .../drivers/gpu/apple-dcp/transaction.py | 2 +- .../drivers/gpu/asahi-g15/pmp-vote.c | 2 + 200 files changed, 20735 insertions(+), 569 deletions(-) create mode 100644 Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml create mode 100644 drivers/gpu/drm/asahi/g17/sleep.c create mode 100644 drivers/gpu/drm/asahi/g17/sleep.h create mode 100644 drivers/gpu/drm/asahi/g17/sleep.rs create mode 100644 drivers/net/wireless/mediatek/mt7932/wow-test.c create mode 100644 drivers/net/wireless/mediatek/mt7932/wow.c create mode 100644 drivers/net/wireless/mediatek/mt7932/wow.h create mode 100644 drivers/soc/apple/apple-cluster-pwrgate.c create mode 100644 drivers/soc/apple/sensor_shim_test.c create mode 100644 drivers/soc/apple/sleep-test.c create mode 100644 drivers/soc/apple/sleep.c create mode 100644 include/linux/irqchip/irq-apple-aic.h create mode 100644 include/linux/phy/apple-atc.h create mode 100644 include/linux/soc/apple/sleep.h create mode 100644 rust/helpers/suspend.c create mode 100755 scripts/ci/apple-dwc3-atc-compile.sh create mode 100644 tools/asahi/apple-dart-dapf-tests.py create mode 100644 tools/asahi/sep-suspend-errors-tests.py diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index 27cffa4ae1db5..9ca6d6a843f32 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -9,6 +9,12 @@ on: - 'drivers/soc/apple/**' - 'include/uapi/linux/apple-sep.h' - 'tools/aurora-sep/**' + - 'Kconfig' + - '**/Kconfig*' + - 'arch/arm64/configs/defconfig' + - 'drivers/usb/dwc3/**' + - 'include/linux/phy/apple-atc.h' + - 'scripts/ci/apple-dwc3-atc-compile.sh' - '.github/workflows/sep-source.yml' jobs: @@ -114,3 +120,38 @@ jobs: --make_options LLVM=1 \ --kunitconfig=drivers/soc/apple/sep.kunitconfig \ --jobs="$(nproc)" --timeout=900 'apple_sep*' + + dwc3-atc-compile: + name: DWC3/ATC ${{ matrix.profile }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + max-parallel: 2 + matrix: + profile: [builtin-no-atc, modular-no-atc, builtin-modular-atc] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Install arm64 C build dependencies + run: | + sudo apt-get update -q + sudo apt-get install -y -q clang-18 lld-18 llvm-18 libelf-dev libssl-dev \ + flex bison bc pkg-config make perl + echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" + - name: Compile the Apple DWC3 object with the resolved PHY profile + env: + PROFILE: ${{ matrix.profile }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: scripts/ci/apple-dwc3-atc-compile.sh "$PROFILE" + - name: Keep configuration and compiler receipts + if: always() + uses: actions/upload-artifact@v4 + with: + name: dwc3-atc-${{ matrix.profile }} + path: ${{ runner.temp }}/dwc3-atc-${{ matrix.profile }}/artifacts/ + retention-days: 14 diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 53948d60375dc..4908e46f58712 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -64,6 +64,19 @@ recovery remain refused. Supporting these operations requires a proven engine teardown/reset and endpoint recovery contract first. Renaming the provider does not qualify those operations. +Deep sleep keeps both functions bound and the radio powered, with its firmware +running. Their drivers stop for it, the link goes through L2 and PERST#, and the SoC +powers the engine off. On the way back the engine's whole register file must read as it +did before the first command, the state the bootstrap checks for; if anything differs, +the radios stay down. Only then does the retained state reopen for one more validated +read, on the same memory, which the retrained link needs before native ECAM. The +functions' configuration is then restored and their drivers resume, with no firmware +load. If a driver finds that its side didn't survive, the host removes both functions, +which resets the chip, and scans again, so that they probe and load their firmware as at +boot. If the link didn't come back, the radio's rail is cycled first. An abandoned sleep +never ran that power-off, so it replays nothing. This needs tunables +from a loader that publishes them, and is refused otherwise. + Suspend-to-idle is allowed only after radio admission, with no IRQ fault, idle command/FIFO checks and intact arena contents. Resume restores the configuration banks after genpd power restoration without submitting another diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 6a494d59a7f89..f6fc69113420b 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -78,11 +78,14 @@ properties: description: Optional host display-engine interrupt sources. mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 mbox-names: + minItems: 1 items: - const: mbox + - const: secure iommus: maxItems: 1 @@ -142,6 +145,13 @@ properties: items: pattern: '^(dp-phy|typec[0-9]+)$' + apple,display-power: + $ref: /schemas/types.yaml#/definitions/phandle + description: + A node whose ps, pwrgate and pmc-dcp registers hold the display power + words this coprocessor's driver writes around a firmware restart, such + as an apple,t8140-display-power node. + apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array items: diff --git a/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml new file mode 100644 index 0000000000000..41bd5586820f2 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,t8140-display-power.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 display power words + +maintainers: + - Diljit Singh + +description: + The words a display coprocessor's driver writes to power its domains off + and on around a restart of its firmware, as macOS does. A display + coprocessor names this node in its apple,display-power property. The words + belong to the power manager and the display block, so no driver binds it. + +properties: + compatible: + const: apple,t8140-display-power + + reg: + items: + - description: the display PMC word that counts the powered coprocessors + - description: the power manager's power state words + - description: the power manager's power gate words + + reg-names: + items: + - const: pmc-dcp + - const: ps + - const: pwrgate + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + }; diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index ad512ac8491df..b40cca489279f 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -82,6 +82,11 @@ allOf: reg: minItems: 3 maxItems: 3 + reg-names: + items: + - const: asc + - const: auxiliary + - const: asc-tail else: if: properties: @@ -137,6 +142,10 @@ properties: status: true + reg-names: + minItems: 1 + maxItems: 3 + '#dma-cells': const: 1 description: diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 992783b2dc68f..2d7b803e77963 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -98,6 +98,13 @@ properties: GFX supplier power cannot represent GPU work on firmware that keeps its coprocessor domain powered while the cores are idle. + apple,pmp-report-entry: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + PMP report entry that tells the PMP the GPU is in use. The driver withdraws + this request before it lowers the GPU's power state for a system sleep, and + makes it again after raising it. + apple,idle-off-delay-ms: minimum: 2 maximum: 1000 diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 0bee5d822fb21..3f0a7adf403d8 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -79,13 +79,15 @@ properties: - description: DART registers - description: Address filter (DAPF) registers, which restrict the physical - addresses the DART lets its masters reach + addresses the DART lets its masters reach, or the slice of the + fabric filter (GAPF) that protects the power state of the + coprocessor CPU behind this DART, cleared to power that CPU down reg-names: minItems: 1 items: - const: dart - - enum: [dapf, auxiliary] + - enum: [dapf, gapf, auxiliary] interrupts: maxItems: 1 @@ -117,6 +119,64 @@ properties: may select a firmware-visible alias of the translation aperture. A consumer may further specify its own aperture in its binding. + apple,dapf-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-matrix + items: + items: + - description: First address of a range + - description: Last address of a range + description: + The address filter the loader programs into the dapf registers, one + entry per range. A sleep that reaches the hardware clears it, and the + DART writes it back before its consumer's firmware starts again. + + apple,dapf-r20: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The last word of each filter entry, one per range in apple,dapf-ranges. + 1 where absent. + + apple,restore-on-power-up: + type: boolean + description: + Reset and restore the DART, and write its address filter back, every + time it powers up, even where its registers still read as they were + left. For a DART that keeps its values through a sleep in which the + SoC's sleep engine runs its consumer, but whose address filter only + takes effect once written again. + + apple,sleep-prepare: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes before a consumer that gates itself is told about a + firmware-driven system suspend, after the DART's state is saved. Streams + they stop are started again at the restore. Needs apple,sleep-restore. + + apple,sleep-restore: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after such a suspend, before the translations are put + back. With this, the consumer saves and restores the DART around the + suspend, as the order relative to its firmware's start matters. + + apple,sleep-restore-late: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Value + description: + Register writes after the translations are back and before the streams + are enabled. + apple,dma-offset: $ref: /schemas/types.yaml#/definitions/uint64 description: > @@ -168,6 +228,18 @@ properties: Tree by the bootloader and required to initialize dynamically powered DART instances. + apple,resume-tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + The same tuples for a DART whose bootloader has already applied them, + written back where a power-off lost them, before its consumer's + firmware starts again. + apple,bypass-sids: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index add726470307d..cb70ea20b8872 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -76,6 +76,14 @@ properties: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox + - description: + A coprocessor's secure RTBuddy port, the ADT's + iop,secure-rtbuddy-proxy. It is polled, with no interrupts described. + items: + - enum: + - apple,t8140-secure-rtbuddy-proxy + - const: apple,secure-rtbuddy-proxy + - description: An older ASC mailbox interface found on T2 and A11 that is also used for the NVMe coprocessor and the system management @@ -134,6 +142,16 @@ properties: maxItems: 2 allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,secure-rtbuddy-proxy + then: + required: + - interrupts + - interrupt-names - if: properties: compatible: @@ -212,8 +230,6 @@ allOf: required: - compatible - reg - - interrupts - - interrupt-names - "#mbox-cells" additionalProperties: false diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index 1671fa01bec96..9a2f39007602a 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -4,13 +4,14 @@ $id: http://devicetree.org/schemas/media/apple,t8140-isp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 Image Signal Processor +title: Apple T8140 and T8142 Image Signal Processors maintainers: - Ryan Murray description: - The T8140 ISP controls the attached camera sensor and processes its frames + The T8140 and T8142 ISPs control the attached camera sensor and process + its frames through bootloader-loaded firmware. The host exchanges commands and image buffers through three DART instances. A separate watchdog register window must be serviced while capturing to receive sensor pixels. @@ -57,21 +58,8 @@ properties: - description: Third ISP DART stream power-domains: - items: - - description: ISP system domain - - description: ISP CPU domain - - description: ISP CPU core 0 domain - - description: ISP CPU core 1 domain - - description: ISP FE domain - - description: ISP DPRX domain - - description: ISP secure domain - - description: ISP STS0 domain - - description: ISP STS1 domain - - description: ISP Pearl domain - - description: ISP BE domain - - description: ISP CLR domain - - description: ISP CVD domain - - description: ISP CVM domain + minItems: 8 + maxItems: 16 memory-region: maxItems: 1 @@ -135,6 +123,50 @@ properties: additionalProperties: false additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-isp + then: + properties: + power-domains: + items: + - description: ISP system domain + - description: ISP CPU domain + - description: ISP CPU core 0 domain + - description: ISP CPU core 1 domain + - description: ISP FE domain + - description: ISP sensor 0 domain + - description: ISP sensor 1 domain + - description: ISP DPRX domain + - description: ISP secure domain + - description: ISP STS0 domain + - description: ISP STS1 domain + - description: ISP Pearl domain + - description: ISP BE domain + - description: ISP CLR domain + - description: ISP CVD domain + - description: ISP CVM domain + - if: + properties: + compatible: + contains: + const: apple,t8142-isp + then: + properties: + power-domains: + items: + - description: ISP system domain + - description: ISP CPU domain + - description: ISP CPU core 0 domain + - description: ISP FE domain + - description: ISP DPRX domain + - description: ISP STS0 domain + - description: ISP BE domain + - description: ISP CLR domain + required: - compatible - reg @@ -171,7 +203,8 @@ examples: iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, - <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_fe>, <&ps_ispsens0>, <&ps_ispsens1>, + <&ps_isp_dprx>, <&ps_isp_secure>, <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, <&ps_isp_cvm>; @@ -191,3 +224,41 @@ examples: }; }; }; + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@4a0000000 { + compatible = "apple,t8142-isp", "apple,isp"; + reg = <0x4 0xa0000000 0x0 0x1801000>, + <0x4 0xa2920000 0x0 0x1000>, + <0x4 0xa2921000 0x0 0x100>, + <0x4 0xe3b10000 0x0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_fe>, + <&ps_isp_dprx>, <&ps_isp_sts0>, + <&ps_isp_be>, <&ps_isp_clr>; + memory-region = <&isp_heap>; + memory-region-names = "heap"; + apple,dart-vm-size = <0x0 0xe0000000>; + apple,platform-id = <0x71>; + apple,temporal-filter = <1>; + + sensor-presets { + preset0 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1920 1080>; + apple,crop = <0 584 3040 1710>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 78f154ce2edcd..084d75bdac4fc 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -196,8 +196,7 @@ properties: description: Initialization tuples for the oe-fabric register region. apple,tunable-axi2af: - $ref: '#/$defs/t6050-tunable' - description: T6050 tuples for the axi2af window, applied before a train. + description: Register tuples for the controller's AXI bridge. apple,tunable-cio3pllcore: $ref: '#/$defs/t6050-tunable' @@ -207,17 +206,22 @@ properties: $ref: '#/$defs/t6050-tunable' description: T6050 tuples for the pcieclkgen window. + apple,tunable-cio3pll: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's PLL. + + apple,tunable-clkgen: + $ref: '#/$defs/tunable' + description: Initialization tuples for the shared PHY block's clock generator. + apple,tunable-phy: - $ref: '#/$defs/t6050-tunable' - description: T6050 tuples for the phy-window window. + description: Register tuples for the controller's PHY window. apple,tunable-phy-ip-pll: - $ref: '#/$defs/t6050-tunable' - description: T6050 PHY PLL tuples for the phy-ip window. + description: PLL register tuples relative to the controller's phy-ip window. apple,tunable-phy-ip-auspma: - $ref: '#/$defs/t6050-tunable' - description: T6050 PHY lane tuples for the phy-ip window. + description: Lane register tuples relative to the controller's phy-ip window. apple,piodma: $ref: /schemas/types.yaml#/definitions/phandle @@ -315,6 +319,16 @@ allOf: apple,msi-vector-base: const: 64 power-domains: false + apple,tunable-axi2af: + $ref: '#/$defs/t6050-tunable' + apple,tunable-phy: + $ref: '#/$defs/t6050-tunable' + apple,tunable-phy-ip-pll: + $ref: '#/$defs/t6050-tunable' + apple,tunable-phy-ip-auspma: + $ref: '#/$defs/t6050-tunable' + apple,tunable-cio3pll: false + apple,tunable-clkgen: false required: - interrupt-names - apple,msi-vector-base @@ -338,12 +352,16 @@ allOf: - apple,link-trained-by-bootloader else: properties: - apple,tunable-axi2af: false + apple,tunable-axi2af: + $ref: '#/$defs/tunable' apple,tunable-cio3pllcore: false apple,tunable-pcieclkgen: false - apple,tunable-phy: false - apple,tunable-phy-ip-pll: false - apple,tunable-phy-ip-auspma: false + apple,tunable-phy: + $ref: '#/$defs/tunable' + apple,tunable-phy-ip-pll: + $ref: '#/$defs/tunable' + apple,tunable-phy-ip-auspma: + $ref: '#/$defs/tunable' patternProperties: "^pci@": properties: @@ -482,8 +500,9 @@ allOf: properties: reg: minItems: 8 - maxItems: 8 + maxItems: 13 reg-names: + minItems: 8 items: - const: config - const: rc @@ -493,6 +512,11 @@ allOf: - const: phy0 - const: phy1 - const: phy2 + - const: axi2af + - const: cio3pll + - const: clkgen + - const: phy-block + - const: phy-ip interrupts: maxItems: 3 else: diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 5364f6dd8871b..d8273b43145c7 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -125,6 +125,10 @@ allOf: - description: USB2 PHY specific controls - description: USB3 PIPE interface controls - description: Secondary eUSB2 PHY register bank (T8140) + - description: + Power and hibernation control of the USB controller the PHY + serves (T8140). The PHY drives that controller's reset and + takes it through hibernation for a system sleep. reg-names: minItems: 5 items: @@ -134,6 +138,7 @@ allOf: - const: usb2phy - const: pipehandler - const: usb2phy-reg + - const: pwrctl required: - power-domains - if: @@ -443,6 +448,18 @@ properties: T8140, copied from Apple's Device Tree by the bootloader. Unlike the other tunables these are applied to the usb2phy-reg region. + apple,axi2af-gate: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset in the axi2af region + - description: Bits + description: > + The fabric bridge bits macOS clears, in this order, before the PHY's + common power domain goes off for a system sleep and sets again once it + is back. With pwrctl, the controller then keeps its power through the + sleep. + patternProperties: "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": $ref: /schemas/types.yaml#/definitions/uint32-array diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 8281b883bd2ac..420f47e4522bc 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -23,6 +23,7 @@ allOf: properties: apple,firmware-owned: false apple,sleep-retained: false + apple,sleep-driver-owned: false apple,always-on: false apple,min-state: false apple,force-disable: false @@ -135,6 +136,13 @@ properties: system sleep. Runtime power management is unaffected. type: boolean + apple,sleep-driver-owned: + description: + The driver behind this domain lowers it before a system sleep and brings + it back afterwards, in an order its hardware needs. The power controller + leaves it alone after a wake. + type: boolean + apple,firmware-owned: description: The boot loader hands over active hardware whose power domain remains diff --git a/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml new file mode 100644 index 0000000000000..8d4e6ab0029db --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,cluster-pwrgate.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC CPU cluster power-gate enables + +maintainers: + - Diljit Singh + +description: | + One PMGR word per CPU cluster, whose top bit lets the cluster's cores power + down in WFI. A core that PMGR powers on comes up allowed to, but the boot CPU + was started by the boot ROM and only follows the word, which the boot loader + may leave clear. The words are set early in boot, in the order listed. + +properties: + compatible: + items: + - enum: + - apple,t8140-cluster-pwrgate + - const: apple,cluster-pwrgate + + reg: + minItems: 1 + maxItems: 8 + description: One word per cluster. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml new file mode 100644 index 0000000000000..3498a9e913cdc --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-sleep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC system sleep controls + +maintainers: + - Diljit Singh + +description: | + The registers a kernel running directly at EL2 on an Apple SoC writes to enter + and return from suspend-to-RAM and to power its secondary CPUs off. They belong + to several hardware blocks, some of which other drivers also use, so this node + overlaps those drivers' register ranges and the driver does not claim them. + Each SoC has its own compatible, as its power domains differ. + +properties: + compatible: + enum: + - apple,t8140-sleep + + reg: + maxItems: 28 + + reg-names: + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: amcc2 + - const: amcc3 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: mcc2 + - const: ptd + - const: clk-perf + - const: cluster-dcs + + apple,wake-enable: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The wake source IDs left enabled across suspend-to-RAM. Every other + source is masked. + + apple,apcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PCIe host whose shared registers and tunables the storage controller's + link depends on. They are needed even with that host disabled. + + apple,pinctrl: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The pin controller whose pins a deep sleep resets. Its pins are saved + before a sleep and put back after it, before the storage controller + starts. + + apple,pinctrl-skip-pins: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Pins of that controller to leave alone across a sleep, neither saved nor + put back. + + apple,debug-port: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The USB-C port controller that carries the debug link, stopped for a sleep. + + apple,display-coprocessors: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The DCP and the external DCP, which must be off before a commit, in that + order. + +required: + - compatible + - reg + - reg-names + - apple,wake-enable + - apple,display-coprocessors + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index b023080e48c7f..7329ac57a538f 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -82,6 +82,10 @@ properties: Chassis identifier used to name the sound card, matching the apple,aop-audio convention. + mic-supply: + description: + The microphone array's power rail, switched on around each capture. + apple,machine-kind: $ref: /schemas/types.yaml#/definitions/string description: diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 7745fef5b686f..6612a5370ee36 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -150,6 +150,13 @@ properties: $ref: "#/$defs/apple,tunable" description: Tunables applied to the drd-ausbc region. + apple,sleep-ddr: + type: boolean + description: + While the controller runs as a host, a system sleep keeps DRAM for its + always-on side, as macOS's USB_SLP_DDR vote does from the host's start + to its stop. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index 282d26ae0c6d2..9540e20101706 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -41,6 +41,24 @@ properties: wakeup-source: true + apple,sleep-ssps: + type: boolean + description: + Tell the controller the system's power state (SSPS) around a + firmware-driven suspend, S3 before it and S0 after. + + apple,debug-lock-key: + $ref: /schemas/types.yaml#/definitions/string + description: + The four-character key that unlocks the controller's debug mode, for + rearming the debug link after a sleep ended it. + + apple,usb-remote-wake: + type: boolean + description: + This controller serves the USB controller's root port, and watches its + USB2 lines for a remote wakeup during a firmware-driven system sleep. + connector: $ref: /schemas/connector/usb-connector.yaml# diff --git a/MAINTAINERS b/MAINTAINERS index 18d06932e001b..5fceded2c2c1d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,6 +2093,7 @@ F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml @@ -2109,6 +2110,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml F: Documentation/devicetree/bindings/sound/apple,t6041-aop-audio.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -2130,8 +2132,10 @@ F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c +F: drivers/soc/apple/sleep*.c F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h +F: include/linux/soc/apple/sleep.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/aop_t8140.rs @@ -2748,6 +2752,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,cluster-pwrgate.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -2813,7 +2818,9 @@ F: drivers/video/backlight/apple_dwi_bl.c F: drivers/watchdog/apple_wdt.c F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h +F: include/linux/irqchip/irq-apple-aic.h F: include/linux/mfd/macsmc.h +F: include/linux/phy/apple-atc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 69d6eff9aa756..dd7580f5f13df 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -23,6 +23,8 @@ /* First mapper in the measured ADT iommu-parent list. */ iommus = <&dcpext0_dart 23>; power-domains = <&ps_dispext0_cpu>; + /* DISPEXT0's words and pmc-dcp, for the restart after a sleep */ + apple,display-power = <&dcp_power>; apple,connector-type = "USB-C"; status = "disabled"; @@ -65,7 +67,11 @@ }; }; -/* Retain the inherited firmware supply once the loader publishes it. */ +/* + * Retain the inherited firmware supply once the loader publishes it. After a sleep the DCP + * driver raises it, the way macOS does, not the PMGR resync. + */ &ps_dispext0_cpu { apple,always-on; + apple,sleep-driver-owned; }; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index ce94f916ff754..fe079fa9967d6 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -27,13 +27,16 @@ sep = &sep; serial0 = &serial0; pmp = &pmp; - rtc0 = &smc_rtc; - rtc1 = &pmu_rtc; + /* First the RTC that can wake the system */ + rtc0 = &pmu_rtc; + rtc1 = &smc_rtc; wifi0 = &wifi0; /* used by the loader to hand over the display */ dcp = &dcp; disp0 = &display; disp0-piodma = &disp0_piodma; + /* and the SIO, with its firmware */ + sio = &sio; }; chosen { @@ -411,20 +414,53 @@ regulator-name = "jack-codec"; regulator-always-on; }; + + /* + * The mic array's rail: SMC pcIO output 25, the ADT's + * function-input_power_enable. macOS switches it around each capture. + */ + mic_array_supply: regulator-mic-array { + compatible = "regulator-fixed"; + regulator-name = "mic-array"; + gpio = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; }; #include "isp-imx558.dtsi" -&ave_dart0 { - status = "okay"; +/* + * The J700 has one video encoder, the ADT's ave1 at 0x289100000 under + * VENC1_SYS. Encoder 0, which t8140.dtsi describes, isn't there: the ADT has + * no node for it and VENC0_SYS refuses its target. Its DARTs and domains + * would wait forever on a domain that never rises. + */ +&pmp_report_ave0 { + status = "disabled"; }; -&ave_dart1 { - status = "okay"; +&ps_venc0_sys { + status = "disabled"; }; -&ave { - status = "okay"; +&ps_venc0_dma { + status = "disabled"; +}; + +&ps_venc0_pipe4 { + status = "disabled"; +}; + +&ps_venc0_pipe5 { + status = "disabled"; +}; + +&ps_venc0_me0 { + status = "disabled"; +}; + +&ps_venc0_me1 { + status = "disabled"; }; &serial0 { @@ -558,6 +594,19 @@ status = "okay"; }; +&dcp_secure_mbox { + status = "okay"; +}; + +/* The SIO itself is enabled by the loader that hands it over. */ +&sio_dart { + status = "okay"; +}; + +&sio_mbox { + status = "okay"; +}; + &dcp { /* 2408x1506 at 219 ppi; 525 nits peak from the backlight node */ panel: panel { @@ -652,6 +701,7 @@ &aop_audio { apple,chassis-name = "J700"; apple,machine-kind = "MacBook Neo"; + mic-supply = <&mic_array_supply>; }; &admac_leap_ns { @@ -744,7 +794,8 @@ }; &smc_gpio { - apple,pcio-templates = <10 0x800000 0xa0000>; + apple,pcio-templates = <10 0x800000 0xa0000>, + <25 0x0 0x190000>; }; &aop_spmi0 { @@ -1850,13 +1901,22 @@ }; }; -/* USB PD controllers, on the nub SPMI A0 bus */ +&sleep { + apple,debug-port = <&hpm0>; +}; + +/* + * USB PD controllers, on the nub SPMI A0 bus. macOS sends them SSPS 3 and 0 around a sleep + * and nothing else: no USBw, as the J700 ADT has no dock-remote-wake. A USB key press + * wakes the J700 through the host's own wake source. + */ &nub_spmi_a0 { status = "okay"; hpm1: usb-pd@a { compatible = "apple,sn201202x"; reg = <0xa SPMI_USID>; + apple,sleep-ssps; interrupts = <37 IRQ_TYPE_EDGE_RISING>, <39 IRQ_TYPE_EDGE_RISING>, <43 IRQ_TYPE_EDGE_RISING>, @@ -1881,6 +1941,8 @@ hpm0: usb-pd@c { compatible = "apple,sn201202x"; reg = <0xc SPMI_USID>; + apple,debug-lock-key = "007J"; + apple,sleep-ssps; interrupts = <11 IRQ_TYPE_EDGE_RISING>, <13 IRQ_TYPE_EDGE_RISING>, <17 IRQ_TYPE_EDGE_RISING>, @@ -2117,6 +2179,10 @@ /* The loader publishes external resources after validating their firmware. */ #include "t8140-j700-dcpext.dtsi" +&sleep { + apple,display-coprocessors = <&dcp>, <&dcpext>; +}; + &display { iommus = <&disp0_dart 0>, <&dispext0_dart 0>; }; diff --git a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi index 43a7d80025bbd..e2fefb5ec5da4 100644 --- a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi @@ -227,12 +227,11 @@ /* ADT /arm-io/dcp-exclave-mailbox. */ dcp_secure_mbox: mailbox@452e7c000 { - compatible = "apple,t8140-secure-mailbox", "apple,secure-mailbox"; + compatible = "apple,t8140-secure-rtbuddy-proxy", + "apple,secure-rtbuddy-proxy"; reg = <0x4 0x52e7c000 0x0 0x4000>; - interrupt-parent = <&aic>; - interrupts = , - ; - iommus = <&dcp_dart 24>; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; status = "disabled"; }; @@ -474,12 +473,14 @@ status = "disabled"; }; + /* Enabled by the boot loader with its firmware, through its alias. */ sio: dma-controller@385e00000 { compatible = "apple,t8140-sio", "apple,sio"; /* Split the ASC bank around the separately owned mailbox window. */ reg = <0x3 0x85e00000 0x0 0x8000>, <0x3 0x85850000 0x0 0x4000>, <0x3 0x85e0c000 0x0 0x7c000>; + reg-names = "asc", "auxiliary", "asc-tail"; #dma-cells = <1>; dma-channels = <128>; mboxes = <&sio_mbox>; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 7123e79b3498a..cac4c68fd3d9f 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -61,6 +61,7 @@ * does not survive this state going off for long. */ apple,always-on; + apple,sleep-driver-owned; }; ps_venc0_sys: power-controller@240 { @@ -206,16 +207,18 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_sys_st"; - power-domains = <&ps_ans>, <&ps_apcie_st>; + power-domains = <&ps_ans>, <&ps_apcie_st>, <&ps_apcie_phy_sw>; }; + /* SYS_ST's parent, kept on: only a real sleep lowers it, through the sleep controller. */ ps_apcie_phy_sw: power-controller@3c0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = "apcie_phy_sw"; - power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + power-domains = <&ps_apcie_sys_gp>; + apple,always-on; }; ps_pmp: power-controller@3d0 { @@ -271,7 +274,8 @@ * enables between ISP_FE and ISP_SECURE. Plain power * gating only: a local domain parked with DEV_DISABLE (the t8112 * apple,force-disable pattern) never reached ACTIVE again on J700 - * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). + * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). No + * automatic power management either: macOS writes them exactly 0xf. */ ps_sep: power-controller@c00 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; @@ -287,6 +291,7 @@ reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cpu"; power-domains = <&ps_isp_sys>; }; @@ -296,6 +301,7 @@ reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cpu_core0"; power-domains = <&ps_isp_cpu>; }; @@ -305,6 +311,7 @@ reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cpu_core1"; power-domains = <&ps_isp_cpu>; }; @@ -314,6 +321,7 @@ reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_fe"; power-domains = <&ps_isp_sys>; }; @@ -323,6 +331,7 @@ reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_dprx"; power-domains = <&ps_isp_fe>; }; @@ -332,6 +341,7 @@ reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_secure"; power-domains = <&ps_isp_fe>; }; @@ -341,6 +351,7 @@ reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_sts0"; power-domains = <&ps_isp_fe>; }; @@ -350,6 +361,7 @@ reg = <0x4038 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_sts1"; power-domains = <&ps_isp_fe>; }; @@ -359,6 +371,7 @@ reg = <0x4040 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_pearl"; power-domains = <&ps_isp_fe>; }; @@ -368,6 +381,7 @@ reg = <0x4048 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_be"; power-domains = <&ps_isp_fe>; }; @@ -377,6 +391,7 @@ reg = <0x4050 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_clr"; power-domains = <&ps_isp_be>; }; @@ -386,6 +401,7 @@ reg = <0x4058 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cvd"; power-domains = <&ps_isp_be>; }; @@ -395,6 +411,7 @@ reg = <0x4060 4>; #power-domain-cells = <0>; #reset-cells = <0>; + apple,no-auto-pm; label = "isp_cvm"; power-domains = <&ps_isp_cvd>; }; @@ -866,6 +883,8 @@ #reset-cells = <0>; label = "sio_cpu"; status = "okay"; + /* The sleep reads the SIO's run bit at the commit, so it stays on until then. */ + apple,always-on; }; ps_fpwm0: power-controller@2b0 { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 4242e655a3af7..75dbb34c522e3 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -551,7 +551,42 @@ pmp_dart: iommu@300300000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x00300000 0x0 0xc000>; + reg = <0x3 0x00300000 0x0 0xc000>, + <0x3 0x00310000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* + * The loader's address filter, from the ADT's dapf-instance-0, + * written back after a sleep. + */ + apple,dapf-ranges = /bits/ 64 <0x220020000 0x220030000 + 0x220060000 0x220070000 + 0x2200a0000 0x2200b0000 + 0x2200e0000 0x2200f0000 + 0x220404000 0x220404070 + 0x220604000 0x220604070 + 0x220804000 0x220804070 + 0x30819d000 0x30819d00c + 0x210e40000 0x210e7ffff + 0x211e40000 0x211e7ffff + 0x40196c000 0x40197bfff + 0x480d14000 0x480e23fff + 0x210e78000 0x210ef0000 + 0x211e78000 0x211ef0000 + 0x401868000 0x40186df08 + 0x24f004000 0x24f004084 + 0x24f008000 0x24f00c140 + 0x250004000 0x250004084 + 0x250008000 0x25000c140 + 0x251004000 0x251004084 + 0x251008000 0x25100c140 + 0x22001c000 0x22001e000 + 0x22005c000 0x22005e000 + 0x22009c000 0x22009e000 + 0x2200dc000 0x2200de000 + 0x3082c8000 0x3082c8050 + 0x220104000 0x22011c000>; + apple,dapf-r20 = <0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x4 0x1 0x1 0x1 0x1 0x1 0x1 + 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1 0x1>; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -672,7 +707,9 @@ dcpext0_dart: iommu@416340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16340000 0x0 0xc000>; + reg = <0x4 0x16340000 0x0 0xc000>, + <0x3 0x0079c700 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -770,6 +807,13 @@ */ apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; apple,dma-offset = /bits/ 64 <0x0>; + /* What macOS writes around a sleep, as offset and value. */ + apple,sleep-prepare = <0x80 0xa102>, <0x704 0xffffffff>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x318 0x1>; + apple,sleep-restore-late = <0x80 0x8000>, <0x228 0x3f3f3f3f>, + <0x22c 0x1010101>; }; sep: sep@282600000 { @@ -825,6 +869,73 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + /* + * The CPU clusters' power-gate enables, in /arm-io/pmgr reg[22]: + * cluster-id 0 (E) at +0x0, 1 (P) at +0x40. iBoot sets E's only. + */ + cluster_pwrgate: power-gate@300724000 { + compatible = "apple,t8140-cluster-pwrgate", "apple,cluster-pwrgate"; + reg = <0x3 0x00724000 0x0 0x4>, + <0x3 0x00724040 0x0 0x4>; + }; + + /* + * Addresses from the ADT: /arm-io/pmgr reg[21], reg[22], + * reg[24] + 0x4000, reg[27], reg[28], reg[31] + 0x4000, /arm-io/wdt + * reg[0], /arm-io/pmgr reg[29] and reg[4]. The AMCC words were + * measured under aura. Sizes cover only the words used: others in + * these blocks are locked by the fabric. + */ + sleep: sleep-controller@300700000 { + compatible = "apple,t8140-sleep"; + reg = <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>, + <0x3 0x00734000 0x0 0x10>, + <0x3 0x00768000 0x0 0x40>, + <0x3 0x00780000 0x0 0xc>, + <0x3 0x082d4000 0x0 0x4>, + <0x3 0x082b0000 0x0 0x30>, + <0x3 0x082b8000 0x0 0x44>, + <0x3 0x08004000 0x0 0x48>, + <0x2 0x20025000 0x0 0x10>, + <0x2 0x20065000 0x0 0x10>, + <0x2 0x200a5000 0x0 0x10>, + <0x2 0x200e5000 0x0 0x10>, + <0x3 0x028a9c14 0x0 0x4>, + <0x2 0x10e70000 0x0 0x1000>, + <0x2 0x11e70000 0x0 0x1000>, + <0x3 0x08714000 0x0 0x84>, + <0x3 0x02800000 0x0 0x10000>, + <0x3 0x0005c000 0x0 0x4>, + <0x3 0x08280000 0x0 0x10000>, + <0x3 0x08908040 0x0 0x4>, + <0x3 0x00040034 0x0 0xc>, + <0x2 0x20400580 0x0 0x10>, + <0x2 0x20600580 0x0 0x10>, + <0x2 0x20800580 0x0 0x10>, + <0x3 0x003d0bc8 0x0 0x10>, + <0x3 0x00058000 0x0 0x10004>, + <0x3 0x00780010 0x0 0x4>; + reg-names = "ps", "pwrgate", "cpu-start", "cllt", "cluster", + "commit", "wdt", "aon-misc", "wake", + "amcc0", "amcc1", "amcc2", "amcc3", "pmc-dcp", + "dvfm-e", "dvfm-p", "spmi", "pmc-voters", "dcs", + "ps-aon", "spmi-a0", "clk-mux", "mcc0", "mcc1", "mcc2", + "ptd", "clk-perf", "cluster-dcs"; + apple,apcie = <&pcie0>; + /* All but the pins macOS leaves alone after a wake */ + apple,pinctrl = <&pinctrl_ap>; + apple,pinctrl-skip-pins = <49 50 51 134>; + /* + * NUB.SPMI0.SW3: the PMU's RTC alarm. ATC0_PMGR_USB_WAKEUP: the + * USB host's, which macOS enables at boot from the ADT and never + * masks. SMC.CPUWakeupAP and SMC.OutboxNotEmpty: the power button, + * the lid, AC, Wi-Fi and Bluetooth. MTP.DOCK.CHANNELS.AP0.IRQ and + * MTP.OUTBOX.N.EMPTY_IRQ: the internal keyboard and trackpad. + */ + apple,wake-enable = <131 70 145 165 80 99>; + }; + wdt: watchdog@3082b0000 { compatible = "apple,t8140-wdt", "apple,t8103-wdt"; reg = <0x3 0x082b0000 0x0 0x4000>; @@ -1190,9 +1301,15 @@ <0x3 0x92028000 0x0 0x8000>, <0x3 0x97020000 0x0 0x4000>, <0x3 0x97024000 0x0 0x4000>, - <0x3 0x97028000 0x0 0x4000>; + <0x3 0x97028000 0x0 0x4000>, + <0x3 0x96000000 0x0 0x1000000>, + <0x3 0x95046200 0x0 0x4000>, + <0x3 0x95044000 0x0 0x4000>, + <0x3 0x97000000 0x0 0x40000>, + <0x3 0x97040000 0x0 0x28000>; reg-names = "config", "rc", "port0", "port1", "port2", - "phy0", "phy1", "phy2"; + "phy0", "phy1", "phy2", + "axi2af", "cio3pll", "clkgen", "phy-block", "phy-ip"; interrupt-parent = <&aic>; interrupts = , , @@ -1239,8 +1356,9 @@ dwc3_0: usb@40a280000 { compatible = "apple,t8140-dwc3", "apple,t8103-dwc3"; reg = <0x4 0x0a280000 0x0 0xcd00>, - <0x4 0x0a28cd00 0x0 0x3200>; - reg-names = "dwc3-core", "dwc3-apple"; + <0x4 0x0a28cd00 0x0 0x3200>, + <0x4 0x0a200000 0x0 0x4000>; + reg-names = "dwc3-core", "dwc3-apple", "drd-cfg"; interrupt-parent = <&aic>; interrupts = ; dr_mode = "otg"; @@ -1251,6 +1369,8 @@ resets = <&atcphy0>; phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; phy-names = "usb2-phy", "usb3-phy"; + /* usb-drd's USB_SLP_DDR-V clock gate in the ADT */ + apple,sleep-ddr; status = "disabled"; }; @@ -1262,6 +1382,22 @@ power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + /* + * macOS's writes around a sleep that keeps the host: exceptions + * and block errors off, streams stopped, a flush; at the wake, its + * config words before the translations, then the rest. 0x104 is + * Linux's own unmasked error interrupt. + */ + apple,sleep-prepare = <0x4060 0xffffffff>, <0x4000 0x0>, + <0x704 0xffffffff>, <0xc20 0xffff>, + <0x80 0xa000>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x308 0x10000000>, + <0x310 0x3fffffff>, <0x318 0x1>, + <0x320 0xff00000>, <0x328 0xff017ff>; + apple,sleep-restore-late = <0x228 0x3f3f3f3f>, <0x22c 0x1010101>, + <0x104 0x0>, <0x4060 0x0>; status = "disabled"; }; @@ -1273,6 +1409,17 @@ power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + /* As dwc3_0_dart_0's. */ + apple,sleep-prepare = <0x4060 0xffffffff>, <0x4000 0x0>, + <0x704 0xffffffff>, <0xc20 0xffff>, + <0x80 0xa000>; + apple,sleep-restore = <0x210 0x0>, <0x20c 0x800005f5>, + <0x220 0xf0f0f>, <0x224 0x80808>, + <0x300 0x1>, <0x308 0x10000000>, + <0x310 0x3fffffff>, <0x318 0x1>, + <0x320 0xff00000>, <0x328 0xff017ff>; + apple,sleep-restore-late = <0x228 0x3f3f3f3f>, <0x22c 0x1010101>, + <0x104 0x0>, <0x4060 0x0>; status = "disabled"; }; @@ -1283,9 +1430,12 @@ <0x4 0x08000000 0x0 0x8000>, <0x4 0x0aa90000 0x0 0x4000>, <0x4 0x0aa84000 0x0 0x4000>, - <0x4 0x0a300000 0x0 0x4000>; + <0x4 0x0a300000 0x0 0x4000>, + <0x4 0x0aa80000 0x0 0x4000>; reg-names = "core", "lpdptx", "axi2af", "usb2phy", - "pipehandler", "usb2phy-reg"; + "pipehandler", "usb2phy-reg", "pwrctl"; + /* Bit 0 of both, around every ATC0_COMMON change on macOS. */ + apple,axi2af-gate = <0x104 0x1>, <0x108 0x1>; #phy-cells = <1>; #reset-cells = <0>; @@ -1308,6 +1458,7 @@ memory-region-names = "ttbs", "pagetables", "handoff"; operating-points-v2 = <&gpu_opp>; apple,clpc = <&clpc>; + apple,pmp-report-entry = <&pmp_report_gfx>; status = "disabled"; }; @@ -1363,7 +1514,9 @@ dcp_dart: iommu@412340000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x12340000 0x0 0xc000>; + reg = <0x4 0x12340000 0x0 0xc000>, + <0x3 0x0079c880 0x0 0x4>; + reg-names = "dart", "gapf"; interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; @@ -1444,8 +1597,22 @@ #apple,bw-doorbell-cells = <2>; }; + /* + * Display power words the DCP driver writes around a warm restart. + * /arm-io/disp0 reg[5] + 0xa9c14, and /arm-io/pmgr reg[21] and reg[22]. + * The firmware owns DISP_CPU only while it runs. No driver binds this. + */ + dcp_power: display-power@3028a9c14 { + compatible = "apple,t8140-display-power"; + reg = <0x3 0x028a9c14 0x0 0x4>, + <0x3 0x00700000 0x0 0x14000>, + <0x3 0x00724000 0x0 0x1000>; + reg-names = "pmc-dcp", "ps", "pwrgate"; + }; + dcp: dcp@412e00000 { compatible = "apple,t8140-dcp", "apple,dcp"; + apple,display-power = <&dcp_power>; reg = <0x4 0x12e00000 0x0 0x4000>, <0x4 0x10000000 0x0 0xafc000>, <0x4 0x12240000 0x0 0x10000>, @@ -1455,8 +1622,8 @@ <0x4 0x12850000 0x0 0x4000>; reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3", "disp-4", "iop-vbar"; - mboxes = <&dcp_mbox>; - mbox-names = "mbox"; + mboxes = <&dcp_mbox>, <&dcp_secure_mbox>; + mbox-names = "mbox", "secure"; iommus = <&dcp_dart 23>; power-domains = <&ps_disp_cpu>; clocks = <&clk_disp0_154>, <&clk_disp0_194>; @@ -1840,6 +2007,17 @@ status = "disabled"; }; +#define T8140_ISP_DART_TUNABLES \ + apple,resume-tunable = <0x20c 0x80000521 0x80000521>, \ + <0x220 0x000f0f0f 0x000f0f0f>, \ + <0x224 0x00ffffff 0x00080808>, \ + <0x300 0x00001f31 0x00000001>, \ + <0x308 0x3ffffffc 0x10000000>, \ + <0x310 0x3ffffffc 0x3ffffffc>, \ + <0x318 0x00001f31 0x00000001>, \ + <0x320 0x3ffffffc 0x0ff00000>, \ + <0x328 0x3ffffffc 0x0ff017fc> + /* * ISP (live J700 ADT /arm-io/isp, sensor-type 0x70, H17 "ISP17a" * firmware, iBoot-loaded). "coproc" is the ISP block, the IRQ @@ -1851,11 +2029,38 @@ */ isp_dart0: iommu@3228c0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x3 0x228c0000 0x0 0xc000>; + reg = <0x3 0x228c0000 0x0 0xc000>, + <0x3 0x228d0000 0x0 0x4000>; + reg-names = "dart", "dapf"; + /* The ADT's dart-tunables-instance-0 */ + T8140_ISP_DART_TUNABLES; + /* + * The loader's address filter, from the ADT's dart-isp + * dapf-instance-0, written back after a sleep: the + * firmware's PMGR, PMP and DSID windows. + */ + apple,dapf-ranges = /bits/ 64 <0x300704000 0x300704063 + 0x300730000 0x300733fff + 0x300e3c000 0x300e3c003 + 0x31062c000 0x310648003 + 0x401660000 0x401660003 + 0x220004000 0x2200162fc + 0x220044000 0x2200562fc + 0x220084000 0x2200962fc + 0x2200c4000 0x2200d62fc + 0x220104000 0x2201162fc + 0x302824000 0x302824004 + 0x3003c0000 0x3003e4000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_isp_sys>; + /* + * It keeps Linux's values through a wake that kept DRAM up, + * where the sleep engine runs the ISP through it, but its + * address filter works again only once written. + */ + apple,restore-on-power-up; /* * 3-level tables (ias <= 36) as iBoot and the native host * ran them: the ISP fetches at vm-base 0x10000000000 and the @@ -1872,6 +2077,8 @@ isp_dart1: iommu@3228e0000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x228e0000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-1 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1892,6 +2099,8 @@ isp_dart2: iommu@322900000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x3 0x22900000 0x0 0xc000>; + /* The ADT's dart-tunables-instance-2 */ + T8140_ISP_DART_TUNABLES; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; @@ -1928,15 +2137,21 @@ interrupt-parent = <&aic>; /* ADT [323 324 325 326 336]; 323 carries the IPC mailbox */ interrupts = ; + /* + * In macOS's order: the camera sensor domains come up + * after ISP_FE and before the rest of the pipeline. + */ power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, - <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_fe>, <&ps_ispsens0>, <&ps_ispsens1>, + <&ps_isp_dprx>, <&ps_isp_secure>, <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, <&ps_isp_cvm>; apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; }; +#undef T8140_ISP_DART_TUNABLES ane_dart0: iommu@401800000 { compatible = "apple,t8140-ane-dart", "apple,t8140-dart", diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 5581318240726..2aaa70c5008ec 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -98,7 +98,7 @@ CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_MACSMC=y -CONFIG_RTC_DRV_APPLE_PMU=m +CONFIG_RTC_DRV_APPLE_PMU=y CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_MFD_MACSMC=y @@ -161,3 +161,4 @@ CONFIG_EDAC_APPLE_SOC=m # unless the bootloader supplies a window (/chosen/aurora,ctxr-exec-window) or # the ctxr_exec_window= command-line option is given. CONFIG_ARM64_CTXR_EXECMEM=y +CONFIG_APPLE_SOC_SLEEP=y diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index a444c8915e886..4ed50e0dc0b5c 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -7,6 +7,7 @@ #include #include +#include /** * struct cpu_operations - Callback operations for hotplugging CPUs. @@ -46,6 +47,58 @@ struct cpu_operations { #endif }; +/** + * struct spin_table_power_ops - Platform control of parking spin-table CPUs + * (smp_spin_table.c). + * + * @cpu_stop: Tell the hardware the calling CPU is stopping. The CPU keeps + * running, and stays coherent, until @cpu_off. + * @cpu_off: Power off the calling CPU, which must not return. + * @cpu_on: Start a CPU that @cpu_off powered off, at its reset vector. + * + * Each takes the CPU's MPIDR hardware ID. + */ +struct spin_table_power_ops { + void (*cpu_stop)(u64 mpidr); + void (*cpu_off)(u64 mpidr); + int (*cpu_on)(u64 mpidr); +}; + +#ifdef CONFIG_HOTPLUG_CPU +int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +int spin_table_hold_parked(void); +void spin_table_unhold_parked(void); +int spin_table_stop_parked(void); +void spin_table_release_stopped(void); +void spin_table_stop_finished(void); +bool spin_table_cpus_stuck(void); +#else +static inline int spin_table_hold_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_unhold_parked(void) {} + +static inline int spin_table_stop_parked(void) +{ + return -EOPNOTSUPP; +} + +static inline void spin_table_release_stopped(void) {} +static inline void spin_table_stop_finished(void) {} + +static inline int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return -EOPNOTSUPP; +} + +static inline bool spin_table_cpus_stuck(void) +{ + return false; +} +#endif + int __init init_cpu_ops(int cpu); extern const struct cpu_operations *get_cpu_ops(int cpu); diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 0a421a88b3012..65982bd36a11f 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -1089,6 +1089,7 @@ static inline bool cpu_has_lpa2(void) } void __init init_cpucap_indirect_list_impdef(void); +void arm64_impdef_cpu_reset_restore(void); void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index e133011f64b5b..7e88e4b6ca501 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -15,6 +15,9 @@ #include extern const struct cpu_operations smp_spin_table_ops; +#ifdef CONFIG_HOTPLUG_CPU +extern const struct cpu_operations smp_rvbar_spin_table_ops; +#endif #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL extern const struct cpu_operations acpi_parking_protocol_ops; #endif @@ -24,6 +27,9 @@ static const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; static const struct cpu_operations *const dt_supported_cpu_ops[] __initconst = { &smp_spin_table_ops, +#ifdef CONFIG_HOTPLUG_CPU + &smp_rvbar_spin_table_ops, +#endif &cpu_psci_ops, NULL, }; @@ -109,6 +115,19 @@ int __init init_cpu_ops(int cpu) return -EOPNOTSUPP; } +#ifdef CONFIG_HOTPLUG_CPU + /* + * A boot object that sends a reset CPU back through its release address + * says so with /chosen/aurora,s2r-mailbox. Its spin-table CPUs can then + * leave the kernel, through the parking spin table. + */ + if (cpu_ops[cpu] == &smp_spin_table_ops && + of_property_present(of_chosen, "aurora,s2r-mailbox")) { + cpu_ops[cpu] = &smp_rvbar_spin_table_ops; + pr_info_once("spin-table CPUs can park: the boot object brings them back\n"); + } +#endif + return 0; } diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c index 4078f95146c32..f08e210bcb813 100644 --- a/arch/arm64/kernel/cpufeature_impdef.c +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -363,6 +364,23 @@ static const struct arm64_cpu_capabilities arm64_impdef_features[] = { {}, }; +/* + * Restore what these features keep per CPU, on a CPU that was reset without + * going through CPU hotplug: its enable callbacks do not run again, and a + * context switch only writes ACTLR when the thread's value changes. + */ +void arm64_impdef_cpu_reset_restore(void) +{ +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + write_sysreg(current->thread.actlr, actlr_el1); +#endif +#ifdef CONFIG_ARM64_APPLE_AMX + if (cpus_have_cap(ARM64_HAS_AMX_APPLE)) + cpu_disable_el0_amx(NULL); +#endif +} + void __init init_cpucap_indirect_list_impdef(void) { init_cpucap_indirect_list_from_array(arm64_impdef_features); diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 424291c547f02..9398c8d0ecb03 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -336,7 +337,7 @@ int swsusp_arch_suspend(void) unsigned long flags; struct sleep_stack_data state; - if (cpus_are_stuck_in_kernel()) { + if (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck()) { pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); return -EBUSY; } diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index b8e6b6c8aa653..1bbc27e9be509 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -68,7 +68,8 @@ int machine_kexec_prepare(struct kimage *kimage) return -EOPNOTSUPP; } - if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { + if (kimage->type != KEXEC_TYPE_CRASH && + (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck())) { pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); return -EBUSY; } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 49029eace3ad9..2d7660bd22f80 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include extern void secondary_holding_pen(void); @@ -125,3 +127,286 @@ const struct cpu_operations smp_spin_table_ops = { .cpu_prepare = smp_spin_table_cpu_prepare, .cpu_boot = smp_spin_table_cpu_boot, }; + +/* + * A spin table whose CPUs can leave the kernel. + * + * Plain spin-table has no way out of the kernel, so disabling the non-boot + * CPUs for system sleep fails with -EOPNOTSUPP. Some boot monitors leave a + * CPU's reset vector locked to their own image and, after a reset, send the + * CPU back through the same release address. Such a monitor publishes + * /chosen/aurora,s2r-mailbox, and its "spin-table" CPUs then get these ops + * (cpu_ops.c), so the device tree keeps the method every kernel knows. One can + * also name "aurora,rvbar-spin-table" as a CPU's method. + * + * A CPU that really powers off can stop the rest of the system on such a chip, + * so a CPU leaving the kernel parks here instead: still coherent, interrupts + * masked. From there it goes back through secondary_holding_pen, or, when the + * platform suspends the system, it stops and takes its last WFI after the commit. + * It is then started again at its reset vector. Stopping and starting a CPU is + * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug + * fails cleanly, as on plain spin-table. + * + * A CPU left awake at the commit can hang the machine, and the commit can't be + * undone. So each parked CPU first answers a hold, while the suspend can still + * refuse, and only a held CPU stops. + */ +#ifdef CONFIG_HOTPLUG_CPU +static const struct spin_table_power_ops *spin_table_power __read_mostly; +static struct cpumask spin_table_cpus_parked; +static struct cpumask spin_table_cpus_held; +static struct cpumask spin_table_cpus_unpark; +static struct cpumask spin_table_cpus_off; +static bool spin_table_hold_requested; +static bool spin_table_stop_requested; +static bool spin_table_stop_released; + +#define SPIN_TABLE_WAIT_US 100000 + +int spin_table_set_power_ops(const struct spin_table_power_ops *ops) +{ + return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; +} + +/* A refused suspend: the held CPUs go back to the park. */ +void spin_table_unhold_parked(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + dsb(ish); + sev(); +} + +/* + * Every CPU not online is held or off. One that is neither, parked or not (a start that + * failed, a CPU never started), may be awake anywhere. + */ +static bool spin_table_all_held(bool report) +{ + unsigned int cpu; + bool all = true; + + for_each_present_cpu(cpu) { + if (cpu_online(cpu) || cpumask_test_cpu(cpu, &spin_table_cpus_held) || + cpumask_test_cpu(cpu, &spin_table_cpus_off)) + continue; + if (!report) + return false; + pr_err("CPU%u did not answer the hold for a suspend\n", cpu); + all = false; + } + + return all; +} + +/* + * Before a suspend's last refusal, interrupts off, other CPUs offline: every parked CPU + * answers and waits, still parked, for spin_table_stop_parked() or + * spin_table_unhold_parked(). Nothing is written to the hardware. Returns -ETIMEDOUT, + * with the hold undone, if any CPU not online is neither held nor off. + */ +int spin_table_hold_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_hold_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (spin_table_all_held(false)) + return 0; + udelay(1); + } + + if (spin_table_all_held(true)) + return 0; + spin_table_unhold_parked(); + + return -ETIMEDOUT; +} + +/* + * At the commit, after spin_table_hold_parked(): the held CPUs stop and wait for + * spin_table_release_stopped(). Returns -ETIMEDOUT if any didn't stop. The request + * stays up until the wake, so a late one still stops. + */ +int spin_table_stop_parked(void) +{ + int us; + + WRITE_ONCE(spin_table_stop_requested, true); + dsb(ish); + sev(); + + for (us = 0; us < SPIN_TABLE_WAIT_US; us++) { + if (cpumask_empty(&spin_table_cpus_parked)) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +/* After the commit: the stopped CPUs take their last WFI. */ +void spin_table_release_stopped(void) +{ + WRITE_ONCE(spin_table_stop_released, true); + dsb(ish); + sev(); +} + +/* + * Back from a suspend, before starting any CPU. Every CPU lost its power with the + * SoC, so one that never saw the stop is started at its reset vector too. + */ +void spin_table_stop_finished(void) +{ + WRITE_ONCE(spin_table_hold_requested, false); + WRITE_ONCE(spin_table_stop_requested, false); + WRITE_ONCE(spin_table_stop_released, false); + cpumask_or(&spin_table_cpus_off, &spin_table_cpus_off, &spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_parked); + cpumask_clear(&spin_table_cpus_held); + cpumask_clear(&spin_table_cpus_unpark); +} + +static int smp_rvbar_spin_table_cpu_boot(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + int ret; + + /* The CPU may run straight into the pen: release it before it starts. */ + write_pen_release(mpidr); + + if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { + ret = spin_table_power->cpu_on(mpidr); + if (ret) { + cpumask_set_cpu(cpu, &spin_table_cpus_off); + return ret; + } + } else if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) { + cpumask_set_cpu(cpu, &spin_table_cpus_unpark); + } + + dsb(ish); + sev(); + + return 0; +} + +static bool smp_rvbar_spin_table_cpu_can_disable(unsigned int cpu) +{ + return cpu != 0; +} + +static int smp_rvbar_spin_table_cpu_disable(unsigned int cpu) +{ + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + return -EOPNOTSUPP; + + return READ_ONCE(spin_table_power) ? 0 : -EOPNOTSUPP; +} + +#ifdef CONFIG_KEXEC_CORE +static void smp_rvbar_spin_table_restart(void) +{ + typeof(cpu_soft_restart) *restart; + + cpu_install_idmap(); + restart = (void *)__pa_symbol(cpu_soft_restart); + restart(0, __pa_symbol(secondary_holding_pen), 0, 0, 0); +} +#else +/* Never reached: cpu_die() doesn't leave its loop without KEXEC_CORE. */ +static void smp_rvbar_spin_table_restart(void) +{ +} +#endif + +/* Held until the commit's stop, or until a refusal sends it back to the park. */ +static void smp_rvbar_spin_table_hold(unsigned int cpu, u64 mpidr) +{ + cpumask_set_cpu(cpu, &spin_table_cpus_held); + + while (READ_ONCE(spin_table_hold_requested)) { + if (READ_ONCE(spin_table_stop_requested)) { + spin_table_power->cpu_stop(mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_off); + cpumask_clear_cpu(cpu, &spin_table_cpus_held); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + while (!READ_ONCE(spin_table_stop_released)) + wfe(); + spin_table_power->cpu_off(mpidr); + } + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_held); +} + +static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) +{ + u64 mpidr = cpu_logical_map(cpu); + + /* The early-boot and crash paths get here without hotplug support. */ + if (!READ_ONCE(spin_table_power) || !IS_ENABLED(CONFIG_KEXEC_CORE)) { + while (1) + wfe(); + } + + cpumask_set_cpu(cpu, &spin_table_cpus_parked); + + while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { + if (READ_ONCE(spin_table_hold_requested)) + smp_rvbar_spin_table_hold(cpu, mpidr); + wfe(); + } + + cpumask_clear_cpu(cpu, &spin_table_cpus_unpark); + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + + smp_rvbar_spin_table_restart(); +} + +static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) +{ + int us; + + if (!READ_ONCE(spin_table_power)) + return 0; + + for (us = 0; us < SPIN_TABLE_WAIT_US; us += 10) { + if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) + return 0; + udelay(10); + } + + return -ETIMEDOUT; +} + +const struct cpu_operations smp_rvbar_spin_table_ops = { + .name = "aurora,rvbar-spin-table", + .cpu_init = smp_spin_table_cpu_init, + .cpu_prepare = smp_spin_table_cpu_prepare, + .cpu_boot = smp_rvbar_spin_table_cpu_boot, + .cpu_can_disable = smp_rvbar_spin_table_cpu_can_disable, + .cpu_disable = smp_rvbar_spin_table_cpu_disable, + .cpu_die = smp_rvbar_spin_table_cpu_die, + .cpu_kill = smp_rvbar_spin_table_cpu_kill, +}; + +/* + * An offlined parking CPU waits in this kernel's text and stack, which a kexec or a + * hibernation restore overwrites: it is stuck for those, power ops or not. + */ +bool spin_table_cpus_stuck(void) +{ + unsigned int cpu; + + for_each_possible_cpu(cpu) + if (get_cpu_ops(cpu) == &smp_rvbar_spin_table_ops) + return true; + return false; +} +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index c41724a40b756..2a69c79b4ed91 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,13 @@ void notrace __cpu_suspend_exit(void) /* Restore additional feature-specific configuration */ ptrauth_suspend_exit(); + + /* + * A firmware-driven system suspend really reset this CPU. Nothing else on + * this path restores implementation-defined state. + */ + if (pm_resume_via_firmware()) + arm64_impdef_cpu_reset_restore(); } /* diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 5ea7e8fc865b8..a9224c63326a1 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -218,8 +218,8 @@ static void ane_fw_snapshot_pmgr(struct ane_fw_backend *b) s->sys = ane_rd32(b, b->pmgr + 0x290); s->cpu = ane_rd32(b, b->pmgr + 0xc000); s->subblock = ane_rd32(b, b->pmgr + 0xc008); - dev_info(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", - s->sys, s->cpu, s->subblock); + dev_dbg(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", + s->sys, s->cpu, s->subblock); } /* Step 2: PWGATE, with ANE_SYS on and before the child power step. */ @@ -229,7 +229,7 @@ static void ane_fw_snapshot_gate(struct ane_fw_backend *b) s->gate_request = ane_rd32(b, b->gate + 0xc); s->gate_ack = ane_rd32(b, b->gate + 0x30); - dev_info(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); + dev_dbg(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); } /* Step 3: the engine and its DARTs, after the child power step. */ @@ -247,8 +247,8 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->pending = ane_rd32(b, b->engine + b->soc->irq_status); for (i = 0; i < 8; i++) s->scratch[i] = ane_rd32(b, b->engine + b->soc->scratch + i * 4); - dev_info(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", - s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); + dev_dbg(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", + s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); for (i = 0; i < ANE_FW_DARTS; i++) { s->dart[i].version = ane_rd32(b, b->dart[i] + 8); s->dart[i].protect = ane_rd32(b, b->dart[i] + 0x200); @@ -257,20 +257,20 @@ static void ane_fw_snapshot_engine(struct ane_fw_backend *b) s->dart[i].error = ane_rd32(b, b->dart[i] + 0x100); for (j = 0; j < 16; j++) s->dart[i].enabled[j] = ane_rd32(b, b->dart[i] + 0xc00 + 4 * j); - dev_info(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", - i, s->dart[i].version, s->dart[i].protect, - s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); + dev_dbg(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", + i, s->dart[i].version, s->dart[i].protect, + s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); for (j = 0; j < 16; j++) if (s->dart[i].enabled[j]) - dev_info(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, - s->dart[i].enabled[j]); + dev_dbg(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, + s->dart[i].enabled[j]); } for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { void __iomem *r = b->engine + 0x1810000 + i * 0x40; - dev_info(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", - i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), - ane_rd32(b, r + 0x20)); + dev_dbg(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", + i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), + ane_rd32(b, r + 0x20)); } } @@ -514,7 +514,6 @@ static int ane_fw_rvbar_ready(struct ane_fw_backend *b); /* Reverse of the proven power-up: sub-block, PWGATE, CPU, then ANE_SYS. */ static int ane_fw_power_down(struct ane_fw_backend *b) { - u32 sys; int ret; ane_wr32(b, ane_rd32(b, b->pmgr + 0xc008) & ~0xf, b->pmgr + 0xc008); @@ -532,20 +531,14 @@ static int ane_fw_power_down(struct ane_fw_backend *b) /* No engine or DART access from here until ANE_SYS is back on. */ b->powered = false; b->power_ref = false; - pm_runtime_put_sync(b->power); - /* - * PMGR reads are safe in any power state. ANE_SYS's actual state - * (bits 7:4) reads 0 once genpd has really turned the domain off. - */ - sys = ane_rd32(b, b->pmgr + 0x290); /* - * Here only requested: while other devices of the domain (the ANE - * DARTs) are active, genpd turns it off in the noirq phase of the - * system suspend; the genpd notifier records when it really does. + * This only drops our reference. The ANE DARTs share the domain and stay + * active, so genpd turns it off in the noirq phase of the system suspend. + * ane_fw_genpd_event() records that, and the resume reports a domain that + * stayed on. */ - dev_info(&b->pdev->dev, - "H17 ANE: powered down for system sleep; ANE_SYS requested off, %s at the put (%#x)\n", - sys & 0xf0 ? "still on" : "off", sys); + pm_runtime_put_sync(b->power); + dev_info(&b->pdev->dev, "H17 ANE: powered down for system sleep\n"); return 0; } @@ -679,10 +672,10 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) * although genpd really turned ANE_SYS off says whether they live in * an always-on block. */ - dev_info(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", - ane_fw_cold_engine_valid(&b->cold) ? "at its reset values" : - "with registers kept", b->sys_offs ? "really went off" : "stayed on", - b->cold.status, ane_rd32(b, b->engine + 0x1810000)); + dev_dbg(dev, "H17 ANE: cold cycle finds the engine %s after ANE_SYS %s (status %#x, DAPF0 %#x)\n", + ane_fw_cold_engine_valid(&b->cold) ? "at its reset values" : + "with registers kept", b->sys_offs ? "really went off" : "stayed on", + b->cold.status, ane_rd32(b, b->engine + 0x1810000)); if (!ane_fw_cold_engine_restartable(&b->cold)) return -EPERM; ret = ane_fw_rvbar_ready(b); @@ -719,8 +712,8 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) return ret; b->transport.allow_free = true; b->resume_ready_ns = ktime_get_ns(); - dev_info(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", - div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); + dev_dbg(dev, "H17 ANE: booted again from the preload after system sleep in %llu ms\n", + div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC)); /* Every open program back behind its handle, as the application left it. */ ane_h16_session_restart(&b->session); ret = ane_h16_session_replay(&b->session, &b->resume_replayed, &b->resume_first_ns); @@ -854,10 +847,15 @@ static void ane_fw_sleep_resume(struct ane_fw_backend *b) } ane_h16_session_sleep(&b->session, false); /* genpd's record of the domain across this sleep (boot time, so it counts s2idle). */ - dev_info(&b->pdev->dev, - "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", - b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), - b->sys_off_pmgr, b->sys_on_pmgr); + if (!b->sys_offs || (b->sys_off_pmgr & 0xf0)) + dev_warn(&b->pdev->dev, + "H17 ANE: ANE_SYS stayed on through the sleep: genpd off %u, on %u; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, b->sys_off_pmgr, b->sys_on_pmgr); + else + dev_dbg(&b->pdev->dev, + "H17 ANE: ANE_SYS while asleep: genpd off %u, on %u, off for %llu ms; PMGR %#x at off, %#x at on\n", + b->sys_offs, b->sys_ons, div_u64(b->sys_off_ns, NSEC_PER_MSEC), + b->sys_off_pmgr, b->sys_on_pmgr); /* One line per wake; times from PM_POST_SUSPEND, which follows the thaw. */ ready = b->resume_ready_ns ? div_u64(b->resume_ready_ns - start, NSEC_PER_MSEC) : -1; first = b->resume_first_ns ? div_u64(b->resume_first_ns - start, NSEC_PER_MSEC) : -1; @@ -991,7 +989,7 @@ static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface * ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); if (!ret) { b->next_dva += size; - dev_info(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); } return ret; } @@ -1061,8 +1059,8 @@ static void ane_fw_cold_shared_note(void *ctx, const char *event, u64 dva, u64 s { struct ane_fw_backend *b = ctx; - dev_info(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", - event, tag, size, dva); + dev_dbg(&b->pdev->dev, "SHAREDMALLOC %s tag=%#x bytes=%llu dva=%#llx\n", + event, tag, size, dva); } static const struct ane_fw_transport_ops ane_fw_ops = { @@ -1157,7 +1155,7 @@ static int ane_fw_power_up(struct ane_fw_backend *b, bool raw_boot, bool cold_en if ((ane_rd64(b, b->engine + 0x18a4010) & 3) != 3 || ane_rd64(b, b->engine + 0x18a4018)) return -EIO; - dev_info(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); + dev_dbg(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); return 0; } @@ -1367,7 +1365,7 @@ static int ane_fw_boot(struct ane_fw_backend *b) ret = ane_fw_scratch_wait(b, 7, 0x08042006); if (ret) return ret; - dev_info(&b->pdev->dev, "ANE boot: READY\n"); + dev_dbg(&b->pdev->dev, "ANE boot: READY\n"); if (ane_rd32(b, scratch) != 8 || ane_rd32(b, scratch + 4) != 0xc440 || ane_rd32(b, scratch + 12) != 0xf00000 || ane_rd32(b, scratch + 16)) return -EPROTO; @@ -1401,14 +1399,14 @@ static int ane_fw_boot(struct ane_fw_backend *b) ane_wr32(b, 0x08042006, scratch + 3 * 4); ret = ane_fw_scratch_wait(b, 3, 0); if (!ret) - dev_info(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); + dev_dbg(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); /* The firmware's table order, which the macOS channel selectors index. */ for (i = 0; !ret && i < ANE_FW_CHANNELS; i++) { const struct ane_fw_channel *c = &b->transport.channels[i]; - dev_info(&b->pdev->dev, - "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", - ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); + dev_dbg(&b->pdev->dev, + "ANE channel %s: table position %u, type %u, doorbell bit %u, %u slots at IPC +%#llx\n", + ane_fw_channel_name(i), c->index, c->type, c->bit, c->slots, c->offset); } return ret; } @@ -1470,12 +1468,12 @@ static int ane_fw_startup(struct ane_fw_backend *b) return -EPROTO; if (commands[i].id == 3 && ane_fw_config_status(command, 16)) return -EPROTO; - dev_info(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); + dev_dbg(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); if (commands[i].id == 3) - dev_info(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", - get_unaligned_le32(command + 8)); + dev_dbg(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", + get_unaligned_le32(command + 8)); if (commands[i].id == 0x22) - dev_info(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); + dev_dbg(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); /* macOS order: START, then SET_SNE_PMU_BASE2, then CONFIG_GET. */ if (commands[i].id == 0 && b->soc->sleep_warm) { ret = ane_fw_pmu_base(b); diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 72910ef976f8e..d60458d69c590 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -65,6 +65,8 @@ /* Vendor command telling the controller that the host goes to sleep. */ #define BT7932_OP_HOST_SLEEP 0xfc6f #define BT7932_HOST_SLEEP_PLEN 7 +/* How long a wake reason may take to arrive once the transport is back. */ +#define BT7932_WAKE_REASON_MS 200 #define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" #define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" @@ -153,6 +155,8 @@ struct bt7932 { bool tx_blocked; /* nothing is sent until resume */ bool irq_held; /* IRQ off until the PCI state is restored */ bool pci_saved; /* saved PCI state not yet restored */ + bool wake_quiesced; /* in host sleep with HCI set up to wake, deep sleep */ + bool wake_expected; /* the next wake report is what woke the host */ enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; }; @@ -300,6 +304,70 @@ static bool bt7932_wake_report(struct bt7932 *bt, const u8 *data, size_t len) } } +/* + * Names a wake report (see bt7932_wake_event()) for the log, and points @addr + * at the remote address it carries, or NULL. + */ +static const char *bt7932_wake_name(const u8 *data, size_t len, const u8 **addr) +{ + /* Where the address starts: after status and handle, or LE report fields. */ + size_t offset = 0; + const char *name = "event"; + + switch (data[0]) { + case HCI_EV_CONN_REQUEST: + name = "connection request"; + offset = HCI_EVENT_HDR_SIZE; + break; + case HCI_EV_CONN_COMPLETE: + name = "connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_SYNC_CONN_COMPLETE: + name = "synchronous connection complete"; + offset = HCI_EVENT_HDR_SIZE + 3; + break; + case HCI_EV_HARDWARE_ERROR: + name = "hardware error"; + break; + case HCI_EV_LE_META: + if (len < HCI_EVENT_HDR_SIZE + 1) + break; + /* Subevent, report count, event type (two bytes if extended), address type. */ + switch (data[HCI_EVENT_HDR_SIZE]) { + case HCI_EV_LE_ADVERTISING_REPORT: + name = "LE advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_DIRECT_ADV_REPORT: + name = "LE directed advertising report"; + offset = HCI_EVENT_HDR_SIZE + 4; + break; + case HCI_EV_LE_EXT_ADV_REPORT: + name = "LE extended advertising report"; + offset = HCI_EVENT_HDR_SIZE + 5; + break; + } + break; + } + *addr = offset && len >= offset + sizeof(bdaddr_t) ? data + offset : NULL; + return name; +} + +/* Caller owns lock and expects the wake reason. */ +static void bt7932_log_wake(struct bt7932 *bt, const u8 *data, size_t len) +{ + const u8 *addr; + const char *name = bt7932_wake_name(data, len, &addr); + + if (addr) + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s from %pMR\n", name, addr); + else + dev_info(&bt->pdev->dev, "woken by Bluetooth: %s\n", name); + bt->wake_expected = false; + wake_up_all(&bt->wake_wait); +} + static int bt7932_validate_ptx(const u8 *data, size_t size) { static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; @@ -520,11 +588,13 @@ static int bt7932_sleep_locked(struct bt7932 *bt) } if (value > 1) return -EPROTO; - /* A previous host-sleep notification must not cancel control 3. */ - if (bt->pm_state == BT7932_QUIESCING) + /* + * A previous host-sleep notification must not cancel control 3. Once + * quiesced, only the host ends the host sleep, so leave the rest to + * bt7932_restore_transport(), as macOS ignores it then too. + */ + if (bt->pm_state == BT7932_QUIESCING || bt->pm_state == BT7932_QUIESCED) return 0; - if (bt->pm_state == BT7932_QUIESCED) - return -EPROTO; if (bt->pm_state == BT7932_RESTORING) { if (value) { if (!bt->waking) { @@ -725,8 +795,11 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) * remember that it arrived. */ if (geo->packet == HCI_EVENT_PKT && bt->wake_tracking && - bt7932_wake_report(bt, descriptor + 16, length)) + bt7932_wake_report(bt, descriptor + 16, length)) { + if (bt->wake_expected) + bt7932_log_wake(bt, descriptor + 16, length); pm_wakeup_dev_event(&bt->pdev->dev, 0, true); + } hci_recv_frame(bt->hdev, skb); } slot = (slot + 1) & BT7932_MASK; @@ -743,9 +816,9 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) return 0; } -static irqreturn_t bt7932_irq_thread(int irq, void *data) +/* Takes what the firmware published: sleep notifications, completions, packets. */ +static void bt7932_service(struct bt7932 *bt) { - struct bt7932 *bt = data; u16 producer; int channel, ret; @@ -785,6 +858,11 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) queue_work(bt->tx_wq, &bt->tx_work); out: mutex_unlock(&bt->lock); +} + +static irqreturn_t bt7932_irq_thread(int irq, void *data) +{ + bt7932_service(data); return IRQ_HANDLED; } @@ -1125,7 +1203,7 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) * Follows the wakeup setting of the PCI function, see bt7932_probe(). * Only while it is on does hci_suspend_sync() leave page scan and LE passive * scan set up for the devices flagged for remote wakeup, and only then does - * bt7932_suspend() keep the transport running with its interrupt armed. + * bt7932_suspend() leave the controller able to wake the system. */ static bool bt7932_wakeup(struct hci_dev *hdev) { @@ -1487,29 +1565,33 @@ static bool bt7932_hci_pm(struct bt7932 *bt) } /* - * Suspend-to-idle can keep the transport running so that the controller can - * wake the system: with wakeup enabled on the PCI function, the controller - * up and HCI suspended by this driver. + * The controller can wake the system with wakeup enabled on the PCI function, the controller + * up and HCI suspended by this driver. In suspend-to-idle the transport stays up and the wake + * is its MSI. A deep sleep takes the link down: the controller goes into host sleep, which is + * the quiesce, and wakes the system through the SMC, as macOS does. */ static bool bt7932_may_wake(struct bt7932 *bt) { - return device_may_wakeup(&bt->pdev->dev) && pm_suspend_no_platform() && - bt7932_hci_pm(bt) && test_bit(HCI_UP, &bt->hdev->flags); + return device_may_wakeup(&bt->pdev->dev) && bt7932_hci_pm(bt) && + test_bit(HCI_UP, &bt->hdev->flags); } static void bt7932_track_wake(struct bt7932 *bt, bool on) { mutex_lock(&bt->lock); bt->wake_tracking = on; + if (!on) + bt->wake_expected = false; mutex_unlock(&bt->lock); } /* - * Before system sleep, the host tells the controller that it is going to - * sleep with a vendor command whose parameters are all ones; nothing is sent - * when the host is back. It is answered by a vendor event, if at all, so - * place it on the command ring directly rather than through the HCI command - * flow, and wait until the controller has taken it. + * For suspend-to-idle with the transport kept running: a vendor command whose + * parameters are all ones. macOS uses the same bytes, but only as a request to + * its own transport, which turns them into the host sleep of a deep sleep (see + * bt7932_suspend()) and never sends them on. It is answered by a vendor event, + * if at all, so place it on the command ring directly rather than through the + * HCI command flow, and wait until the controller has taken it. * * Returns -EAGAIN if nothing was published (the controller could not be * reached without a fault, or the ring was full), -EBUSY if the command was @@ -1583,6 +1665,7 @@ static int bt7932_arm_wakeup(struct bt7932 *bt) ret = bt->fault ? -EIO : 0; if (!ret) { bt->wake_armed = true; + bt->wake_expected = true; /* A failed wake handshake would fault the transport. */ bt->tx_blocked = true; } @@ -1607,7 +1690,6 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) mutex_lock(&bt->lock); bt->wake_armed = false; bt->tx_blocked = false; - bt->wake_tracking = false; bt->pci_saved = false; mutex_unlock(&bt->lock); wake_up_all(&bt->wake_wait); @@ -1617,6 +1699,28 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) } } +/* + * After a sleep set up for Bluetooth wake, the controller delivers what woke + * the host once the transport runs again. Take it while HCI is still + * suspended, so that the core records the wake reason too, and log it. + * Ends wake tracking. + */ +static void bt7932_take_wake_reason(struct bt7932 *bt) +{ + /* The interrupt thread leaves the rings alone until the restore ends. */ + bt7932_service(bt); + wait_event_timeout(bt->wake_wait, + !READ_ONCE(bt->wake_expected) || READ_ONCE(bt->fault), + msecs_to_jiffies(BT7932_WAKE_REASON_MS)); + mutex_lock(&bt->lock); + if (bt->wake_expected && !bt->fault) + dev_dbg(&bt->pdev->dev, "Bluetooth did not wake the host\n"); + mutex_unlock(&bt->lock); + bt7932_track_wake(bt, false); + /* The core takes a wake reason only until hci_resume_dev(). */ + flush_work(&bt->hdev->rx_work); +} + /* * With wakeup enabled, the transport stays up through suspend-to-idle * instead of being quiesced: @@ -1637,6 +1741,12 @@ static void bt7932_disarm_wakeup(struct bt7932 *bt) * The PCIe link and host bridge stay up in suspend-to-idle anyway, as the * Wi-Fi function on the same link is never stopped, and ASPM stays off on * this link as the transport was admitted. + * + * A deep sleep with wakeup enabled quiesces the transport after HCI has set + * up its wake scans. The quiesce (sleep control 3, notification 3) is the + * host sleep that macOS enters at S2R. The controller keeps scanning and + * wakes the system through the SMC. bt7932_resume() ends the host sleep + * with control 0, as macOS does, and then takes the wake reason. */ static int bt7932_suspend(struct device *dev) { @@ -1644,8 +1754,9 @@ static int bt7932_suspend(struct device *dev) bool wake; int ret; + /* A transport that's down has nothing to stop, and it can't cost a deep sleep. */ if (READ_ONCE(bt->fault)) - return -EIO; + return pm_suspend_no_platform() ? -EIO : 0; if (!bt->registered) return 0; wake = bt7932_may_wake(bt); @@ -1656,16 +1767,19 @@ static int bt7932_suspend(struct device *dev) ret = hci_suspend_dev(bt->hdev); if (ret) { bt7932_track_wake(bt, false); - return ret; + if (pm_suspend_no_platform()) + return ret; + dev_warn(dev, "HCI did not suspend: %d\n", ret); } } - if (wake) { + if (wake && pm_suspend_no_platform()) { ret = bt7932_arm_wakeup(bt); if (!ret) { - dev_dbg(dev, "system sleep with wakeup armed\n"); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } bt7932_track_wake(bt, false); + wake = false; if (ret == -EBUSY) { /* * The host sleep command may still take effect, so the @@ -1676,8 +1790,7 @@ static int bt7932_suspend(struct device *dev) hci_resume_dev(bt->hdev); bt7932_reset(bt->hdev); return ret; - } - if (ret != -EAGAIN) { + } else if (ret != -EAGAIN) { dev_err(dev, "transport failed while arming wakeup\n"); hci_resume_dev(bt->hdev); return ret; @@ -1686,9 +1799,28 @@ static int bt7932_suspend(struct device *dev) ret = bt7932_quiesce(bt); if (!ret) { dev_dbg(dev, "system sleep quiesce acknowledged\n"); + if (!wake) + return 0; + /* hci_suspend_dev() may have stopped short of the wake setup. */ + if (!bt->hdev->suspended || + bt->hdev->suspend_state != BT_SUSPEND_CONFIGURE_WAKE) { + bt7932_track_wake(bt, false); + dev_warn(dev, "no Bluetooth wake for this sleep\n"); + return 0; + } + mutex_lock(&bt->lock); + bt->wake_quiesced = true; + bt->wake_expected = true; + mutex_unlock(&bt->lock); + dev_dbg(dev, "sleep with Bluetooth wake armed\n"); return 0; } + if (wake) + bt7932_track_wake(bt, false); dev_warn(dev, "system sleep quiesce failed: %d\n", ret); + /* The wake finds out what's left. A deep sleep takes the power regardless. */ + if (!pm_suspend_no_platform()) + return 0; /* Undo our request before the PM core rolls back the rest of the bus. */ if (!READ_ONCE(bt->fault)) bt7932_restore_transport(bt); @@ -1732,23 +1864,43 @@ static int bt7932_resume_noirq(struct device *dev) static int bt7932_resume(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + bool wake_quiesced = bt->wake_quiesced; int ret; + bt->wake_quiesced = false; + /* A deep sleep retrains the link. With the chip gone, nothing here may touch it. */ + if (!pci_device_is_present(bt->pdev)) { + bt7932_track_wake(bt, false); + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "radio gone after the sleep"); + mutex_unlock(&bt->lock); + apple_piodma_radio_recover(); + return -ENODEV; + } + if (bt->wake_armed) { bt7932_disarm_wakeup(bt); + bt7932_take_wake_reason(bt); if (READ_ONCE(bt->fault)) return -EIO; return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } - if (READ_ONCE(bt->fault)) + if (READ_ONCE(bt->fault)) { + bt7932_track_wake(bt, false); return -EIO; + } if (!bt->registered) return 0; if (READ_ONCE(bt->pm_state) != BT7932_RUNNING) { ret = bt7932_restore_transport(bt); - if (ret) + if (ret) { + bt7932_track_wake(bt, false); + apple_piodma_radio_recover(); return ret; + } } + if (wake_quiesced) + bt7932_take_wake_reason(bt); dev_dbg(dev, "transport restored after system sleep\n"); return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } @@ -1778,9 +1930,15 @@ static void bt7932_remove(struct pci_dev *pdev) u16 command = U16_MAX; bool quiesced, drained; - /* Closing HCI may send Reset: keep the transport and IRQ alive for it. */ - if (bt->registered) + /* + * The HCI core stops its command and event work before it closes a device it + * unregisters, so a Reset on that close goes unsent and waits out its 2 s timeout. + * The quiesce below stops the firmware. Close still runs with the transport alive. + */ + if (bt->registered) { + hci_clear_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_unregister_dev(bt->hdev); + } quiesced = bt->ipc_ready && !READ_ONCE(bt->fault) && !bt7932_quiesce(bt); mutex_lock(&bt->lock); WRITE_ONCE(bt->stopping, true); diff --git a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c index 61e93ce1f31db..8a57f9f30ca1a 100644 --- a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c +++ b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c @@ -65,11 +65,57 @@ static void bt7932_wake_event_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, bt7932_wake_event(error, sizeof(error)), BT7932_WAKE_ERROR); } +/* The wake log names the event and the remote address it carries. */ +static void bt7932_wake_name_bredr_test(struct kunit *test) +{ + static const u8 request[] = { HCI_EV_CONN_REQUEST, 10, + 1, 2, 3, 4, 5, 6, 0x40, 0x25, 0x00, 0x01 }; + static const u8 complete[] = { HCI_EV_CONN_COMPLETE, 11, 0x00, 0x01, 0x00, + 1, 2, 3, 4, 5, 6, 0x01, 0x00 }; + static const u8 sync_short[] = { HCI_EV_SYNC_CONN_COMPLETE, 1, 0x00 }; + static const u8 error[] = { HCI_EV_HARDWARE_ERROR, 1, 0x00 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(request, sizeof(request), &addr), + "connection request"); + KUNIT_EXPECT_PTR_EQ(test, addr, &request[2]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(complete, sizeof(complete), &addr), + "connection complete"); + KUNIT_EXPECT_PTR_EQ(test, addr, &complete[5]); + bt7932_wake_name(sync_short, sizeof(sync_short), &addr); + KUNIT_EXPECT_NULL(test, addr); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(error, sizeof(error), &addr), "hardware error"); + KUNIT_EXPECT_NULL(test, addr); +} + +static void bt7932_wake_name_le_test(struct kunit *test) +{ + /* Subevent, one report, event type, address type, address, data length, RSSI. */ + static const u8 adv[] = { HCI_EV_LE_META, 12, HCI_EV_LE_ADVERTISING_REPORT, 1, + 0x00, 0x01, 1, 2, 3, 4, 5, 6, 0x00, 0xc0 }; + /* The extended report's event type takes two bytes. */ + static const u8 ext[] = { HCI_EV_LE_META, 11, HCI_EV_LE_EXT_ADV_REPORT, 1, + 0x13, 0x00, 0x01, 1, 2, 3, 4, 5, 6 }; + static const u8 adv_short[] = { HCI_EV_LE_META, 2, HCI_EV_LE_ADVERTISING_REPORT, 1 }; + const u8 *addr; + + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(adv, sizeof(adv), &addr), + "LE advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &adv[6]); + KUNIT_EXPECT_STREQ(test, bt7932_wake_name(ext, sizeof(ext), &addr), + "LE extended advertising report"); + KUNIT_EXPECT_PTR_EQ(test, addr, &ext[7]); + bt7932_wake_name(adv_short, sizeof(adv_short), &addr); + KUNIT_EXPECT_NULL(test, addr); +} + static struct kunit_case bt7932_wake_test_cases[] = { KUNIT_CASE(bt7932_wake_event_bredr_test), KUNIT_CASE(bt7932_wake_event_le_test), KUNIT_CASE(bt7932_wake_event_replies_test), KUNIT_CASE(bt7932_wake_event_error_test), + KUNIT_CASE(bt7932_wake_name_bredr_test), + KUNIT_CASE(bt7932_wake_name_le_test), {} }; diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index fb87ae6ae137a..379fa031ce599 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -837,6 +838,19 @@ static unsigned int apple_soc_cpufreq_pstate_freq(struct cpufreq_policy *policy, static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; + struct apple_fast_die *fd = priv->fast_die; + + /* + * A firmware-driven sleep stops the PMP, and its driver starts it again only after the + * CPUs are back: the first samples after the wake find it down. Give it the same time to + * come back as at boot. Jiffies don't count the sleep. + */ + if (fd && fd->report && pm_suspend_via_firmware()) { + spin_lock_bh(&fd->lock); + fd->start = jiffies; + fd->warned = false; + spin_unlock_bh(&fd->lock); + } if (priv->info->lazy_verify) { int pstate = apple_soc_cpufreq_drain(priv); @@ -856,6 +870,37 @@ static int apple_soc_cpufreq_suspend(struct cpufreq_policy *policy) return cpufreq_generic_suspend(policy); } +/* + * A firmware-driven sleep powers the cluster down, and it comes back in a performance state + * of the hardware's choosing, not the one asked for at suspend. Take what it reads as the + * expected state again, before the governor starts, rather than stopping every request as if + * someone else owned it. + */ +static int apple_soc_cpufreq_resume(struct cpufreq_policy *policy) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned long flags; + u64 reg; + + if (!priv->info->verify_transition || !pm_resume_via_firmware()) + return 0; + + raw_spin_lock_irqsave(&priv->lock, flags); + reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + if (!(reg & APPLE_DVFS_CMD_BUSY)) { + unsigned int now = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); + + if (now != priv->expected_pstate) + dev_info(priv->cpu_dev, "DVFS state %u after the sleep, %u before\n", + now, priv->expected_pstate); + priv->expected_pstate = now; + priv->transition_failed = false; + } + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq) { @@ -1571,6 +1616,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, .suspend = apple_soc_cpufreq_suspend, + .resume = apple_soc_cpufreq_resume, }; static int __init apple_soc_cpufreq_module_init(void) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index b7336f97ee9fc..34599811779fb 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dmaengine.h" @@ -139,6 +140,12 @@ struct admac_chan { struct list_head issued; struct list_head to_free; + + /* The client's last configuration, set up again after a system sleep. */ + struct dma_slave_config cfg; + bool cfg_valid; + /* A system sleep took the channel's registers; set it up before a transfer. */ + bool lost; }; struct admac_sram { @@ -629,11 +636,30 @@ static void admac_reset_rings(struct admac_chan *adchan) writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, adchan->no)); } +static int admac_device_config(struct dma_chan *chan, struct dma_slave_config *config); + +/* Registers can't be reached before a client powers the fabric, so this waits for it. */ +static void admac_restore_chan(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + + if (!adchan->lost) + return; + adchan->lost = false; + + if (ad->split_channels) + writel_relaxed(0x18080001, ad->base + REG_CTL); + writel_relaxed(adchan->carveout, ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); + if (adchan->cfg_valid) + admac_device_config(&adchan->chan, &adchan->cfg); +} + static void admac_start_current_tx(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; int ch = adchan->no; + admac_restore_chan(adchan); admac_reset_rings(adchan); writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, ch)); if (ad->split_channels) { @@ -686,10 +712,19 @@ static int admac_resume(struct dma_chan *chan) int ret = 0; spin_lock_irqsave(&adchan->lock, flags); - if (adchan->current_tx && adchan->current_tx->error) + if (adchan->current_tx && adchan->current_tx->error) { ret = -EIO; - else + } else if (adchan->lost && adchan->current_tx) { + struct admac_tx *tx = adchan->current_tx; + + /* The rings went with the registers: start again from the last period reclaimed. */ + tx->submitted_pos = tx->reclaimed_pos; + tx->nperiods_outstanding = 0; + adchan->nperiod_acks = 0; + admac_start_current_tx(adchan); + } else { admac_start_chan(adchan); + } spin_unlock_irqrestore(&adchan->lock, flags); return ret; @@ -1052,6 +1087,9 @@ static int admac_device_config(struct dma_chan *chan, | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), ad->base + REG_CHAN_FIFOCTL(ad, adchan->no)); + adchan->cfg = *config; + adchan->cfg_valid = true; + return 0; } @@ -1081,8 +1119,36 @@ static int admac_runtime_resume(struct device *dev) return 0; } +/* + * A firmware-driven sleep, refused or not, takes the registers of a channel a + * client still holds (genpd gates the domain at noirq), and its client won't + * configure it again: only a new hw_params does that. + */ +static int __maybe_unused admac_resume_early(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + unsigned long flags; + int i; + + if (!pm_suspend_via_firmware()) + return 0; + + mutex_lock(&ad->cache_alloc_lock); + ad->ctl_programmed = false; + mutex_unlock(&ad->cache_alloc_lock); + + for (i = 0; i < ad->nchannels; i++) { + spin_lock_irqsave(&ad->channels[i].lock, flags); + ad->channels[i].lost = true; + spin_unlock_irqrestore(&ad->channels[i].lock, flags); + } + + return 0; +} + /* Allocated channels must stay powered, including paused cyclic transfers. */ static const struct dev_pm_ops admac_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, admac_resume_early) RUNTIME_PM_OPS(admac_runtime_suspend, admac_runtime_resume, NULL) }; diff --git a/drivers/dma/apple-sio-boot-test.c b/drivers/dma/apple-sio-boot-test.c index 3d8f93d11d8bb..5994518f619b0 100644 --- a/drivers/dma/apple-sio-boot-test.c +++ b/drivers/dma/apple-sio-boot-test.c @@ -187,12 +187,36 @@ static void sio_boot_dma_arrays_test(struct kunit *test) } } +/* Only a channel with nothing on it may pass through a firmware restart. */ +static void sio_sleep_busy_test(struct kunit *test) +{ + struct sio_chan *chan = kunit_kzalloc(test, sizeof(*chan), GFP_KERNEL); + struct sio_tx *tx = kunit_kzalloc(test, sizeof(*tx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, chan); + KUNIT_ASSERT_NOT_NULL(test, tx); + spin_lock_init(&chan->vc.lock); + + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); + chan->current_tx = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->current_tx = NULL; + chan->idle = tx; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->idle = NULL; + chan->ring = true; + KUNIT_EXPECT_TRUE(test, sio_chan_busy(chan)); + chan->ring = false; + KUNIT_EXPECT_FALSE(test, sio_chan_busy(chan)); +} + static struct kunit_case sio_boot_test_cases[] = { KUNIT_CASE(sio_boot_order_test), KUNIT_CASE(sio_boot_optional_tables_test), KUNIT_CASE(sio_boot_bad_params_test), KUNIT_CASE(sio_boot_control_ack_failure_test), KUNIT_CASE(sio_boot_dma_arrays_test), + KUNIT_CASE(sio_sleep_busy_test), {} }; diff --git a/drivers/dma/apple-sio-probe-test.c b/drivers/dma/apple-sio-probe-test.c index c486c5072f07c..9b9c5a0774320 100644 --- a/drivers/dma/apple-sio-probe-test.c +++ b/drivers/dma/apple-sio-probe-test.c @@ -72,6 +72,8 @@ static void sio_match_data_test(struct kunit *test) KUNIT_EXPECT_EQ(test, hw->fifo_threshold, 0x800U); KUNIT_EXPECT_FALSE(test, hw->clear_period_desc); KUNIT_EXPECT_FALSE(test, hw->check_boot_tables); + KUNIT_EXPECT_FALSE(test, hw->firmware_only); + KUNIT_EXPECT_FALSE(test, sio_hw_t8140.firmware_only); } static struct kunit_case sio_probe_test_cases[] = { diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index e1a8ec545b8c6..3b9d804f206d1 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include "dmaengine.h" @@ -160,6 +162,11 @@ struct sio_hw_data { u32 cfg_word4; /* configuration word 4 */ bool cfg_six_words; /* the configuration ends in a zero sixth word */ bool clear_period_desc; + /* + * Only run the firmware, so that it can be stopped for a system sleep. Nothing + * uses its DMA here, and its service parameters haven't been traced. + */ + bool firmware_only; /* DMA arrays owned by the driver, if not protocol 9's */ const struct sio_dma_arrays *arrays; int (*check_boot_tables)(struct device *dev); @@ -173,6 +180,8 @@ struct sio_data { struct dma_device dma; struct device *dev; struct apple_rtkit *rtk; + /* The firmware is stopped for a system sleep. */ + bool asleep; u32 protocol; void *shmem; dma_addr_t shmem_iova; @@ -2014,6 +2023,11 @@ static int sio_probe(struct platform_device *pdev) if (err) return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + if (hw->firmware_only) { + dev_info(&pdev->dev, "firmware running, no DMA service\n"); + goto rpm_put; + } + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); if (err) return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); @@ -2057,6 +2071,9 @@ static void sio_remove(struct platform_device *pdev) { struct sio_data *sio = platform_get_drvdata(pdev); + if (sio->hw->firmware_only) + return; + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&sio->dma); } @@ -2074,7 +2091,94 @@ static __maybe_unused int sio_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); +/* A channel with a transfer can't be carried through a firmware restart. */ +static bool sio_chan_busy(struct sio_chan *siochan) +{ + unsigned long flags; + bool busy; + + spin_lock_irqsave(&siochan->vc.lock, flags); + busy = siochan->current_tx || siochan->idle || siochan->ring; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return busy; +} + +static __maybe_unused int sio_suspend_late(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int i, ret; + + if (!pm_suspend_via_firmware() || sio->asleep) + return 0; + + for (i = 0; i < sio->nchannels; i++) { + if (sio_chan_busy(&sio->channels[i])) { + dev_err(dev, "channel %d is busy, refusing the sleep\n", i); + return -EBUSY; + } + } + + ret = apple_rtkit_sleep_stop(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) { + dev_err(dev, "firmware did not stop for sleep: %d\n", ret); + if (apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) + dev_err(dev, "firmware did not come back\n"); + return ret; + } + sio->asleep = true; + + return 0; +} + +static __maybe_unused int sio_resume_early(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int ret; + + if (!sio->asleep) + return 0; + /* + * A refused sleep: an idled firmware doesn't wake with its power kept, and SIO_CPU shares + * SIO's switch, so it can't be power-cycled alone. The sleep engine's own PMGR reset of it + * restarts it (S); if that fails it stays stopped until the next real sleep. + */ + if (!pm_resume_via_firmware()) { + if (!apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL)) { + sio->asleep = false; + return 0; + } + ret = apple_sleep_engine_reset(dev); + if (!ret) + ret = apple_rtkit_power_cycled_return(sio->rtk, + sio->base + REG_CPU_CONTROL); + if (ret) { + dev_warn(dev, "sleep refused: firmware not back after a reset (%d), stopped until a real sleep\n", + ret); + return 0; + } + sio->asleep = false; + dev_info(dev, "sleep refused: firmware back after a reset\n"); + return 0; + } + /* Endpoints and parameters stay with the firmware: nothing is sent again. */ + ret = apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL); + if (ret) { + /* Still down: the next sleep restarts it, or resets it if refused. */ + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + return 0; + } + sio->asleep = false; + dev_dbg(dev, "firmware back from sleep\n"); + + return 0; +} + +static const struct dev_pm_ops sio_pm_ops = { + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + LATE_SYSTEM_SLEEP_PM_OPS(sio_suspend_late, sio_resume_early) + RUNTIME_PM_OPS(sio_suspend, sio_resume, NULL) +}; static struct platform_driver apple_sio_driver = { .driver = { diff --git a/drivers/edac/apple_soc_edac.c b/drivers/edac/apple_soc_edac.c index a72e7f4dc351a..a4f717029cc5c 100644 --- a/drivers/edac/apple_soc_edac.c +++ b/drivers/edac/apple_soc_edac.c @@ -5,6 +5,7 @@ * Copyright The Aurora Silicon Contributors */ +#include #include #include #include @@ -36,9 +37,23 @@ struct apple_soc_edac_range { const char *name; }; +/* + * A fabric NI whose error status sits in its aperture: a block of status words, + * one per error type, whose set bits name the port. They're acknowledged by + * writing the set bits back. + */ +struct apple_soc_edac_ni { + u8 index; /* interrupt index */ + u8 reg; /* reg index of the NI's aperture */ + u16 status; /* offset of the first status word */ + u32 words; /* bit n: a status word at status + 4 * n */ +}; + struct apple_soc_edac_hw { const struct apple_soc_edac_range *ranges; size_t nranges; + const struct apple_soc_edac_ni *ni; + size_t nni; unsigned int num_irqs; unsigned int num_status_regs; bool event_log; @@ -51,6 +66,8 @@ struct apple_soc_edac_irq { const char *source; unsigned int index; int irq; + const struct apple_soc_edac_ni *ni; + void __iomem *ni_base; unsigned int retries; unsigned int backoff_ms; unsigned long armed_jiffies; @@ -126,6 +143,16 @@ static const struct apple_soc_edac_range t8140_error_ranges[] = { { 25, 1, "pio" }, }; +/* From the ADT's niN-metadata and macOS's NI decoder tables */ +static const struct apple_soc_edac_ni t8140_ni[] = { + { 18, 9, 0x1900, 0x1ffd7ff }, /* SoC NI 0 */ + { 19, 10, 0x1b00, 0x1ffd7ff }, /* SoC NI 1 */ + { 20, 11, 0x1500, 0x1ffd7ff }, /* SoC NI 2 */ + { 21, 12, 0x2500, 0x1ffd7ff }, /* MCC NI */ + { 22, 13, 0x2100, 0x7fefff }, /* AFC MCC NI */ + { 23, 14, 0x1d00, 0x3ff7ff }, /* GFX NI */ +}; + static const struct apple_soc_edac_hw t6000_error_hw = { .ranges = t6000_error_ranges, .nranges = ARRAY_SIZE(t6000_error_ranges), @@ -148,6 +175,8 @@ static const struct apple_soc_edac_hw t8112_error_hw = { static const struct apple_soc_edac_hw t8140_error_hw = { .ranges = t8140_error_ranges, .nranges = ARRAY_SIZE(t8140_error_ranges), + .ni = t8140_ni, + .nni = ARRAY_SIZE(t8140_ni), .num_irqs = 26, }; @@ -214,6 +243,38 @@ static bool apple_soc_edac_t6041_status(struct apple_soc_edac_irq *irq_data, return false; } +/* + * Log and acknowledge an NI's set status words. Returns whether any was set; + * @cleared says whether all of them read back clear. + */ +static bool apple_soc_edac_ni_status(struct apple_soc_edac_irq *irq_data, + bool *cleared) +{ + const struct apple_soc_edac_ni *ni = irq_data->ni; + unsigned long words = ni->words; + unsigned int bit, offset; + bool found = false; + u32 status, left; + + *cleared = true; + for_each_set_bit(bit, &words, 32) { + offset = ni->status + 4 * bit; + status = readl(irq_data->ni_base + offset); + if (!status) + continue; + writel(status, irq_data->ni_base + offset); + left = readl(irq_data->ni_base + offset); + dev_err_ratelimited(irq_data->edac->dev, + "%s: status %#x at %#x, %#x after the acknowledge\n", + irq_data->source, status, offset, left); + found = true; + if (left) + *cleared = false; + } + + return found; +} + static void apple_soc_edac_rearm(struct work_struct *work) { struct apple_soc_edac_irq *irq_data = @@ -244,12 +305,27 @@ static irqreturn_t apple_soc_edac_irq_thread(int irq, void *data) return IRQ_HANDLED; } + snprintf(msg, sizeof(msg), "%s source %u", irq_data->source, + irq_data->index); + if (irq_data->ni) { + bool cleared; + + if (apple_soc_edac_ni_status(irq_data, &cleared)) { + edac_device_handle_ue(edac->dci, 0, 0, msg); + dev_err_ratelimited(edac->dev, "hardware error: %s (IRQ %d)\n", + msg, irq); + if (cleared) + return IRQ_HANDLED; + } else { + dev_err_ratelimited(edac->dev, "%s asserted with no status set\n", + msg); + } + } + /* No status was pending, or this source has no clear contract. */ disable_irq_nosync(irq); - snprintf(msg, sizeof(msg), "%s source %u", irq_data->source, - irq_data->index); - if (!edac->hw->num_status_regs || irq_data->index < 45) { + if (!irq_data->ni && (!edac->hw->num_status_regs || irq_data->index < 45)) { edac_device_handle_ue(edac->dci, 0, 0, msg); dev_err_ratelimited(edac->dev, "hardware error: %s (IRQ %d)\n", msg, irq); @@ -280,6 +356,24 @@ static irqreturn_t apple_soc_edac_irq_thread(int irq, void *data) return IRQ_HANDLED; } +static int apple_soc_edac_map_ni(struct platform_device *pdev, + struct apple_soc_edac_irq *irq_data) +{ + const struct apple_soc_edac_hw *hw = irq_data->edac->hw; + size_t i; + + for (i = 0; i < hw->nni; i++) { + if (hw->ni[i].index != irq_data->index) + continue; + irq_data->ni_base = devm_platform_ioremap_resource(pdev, hw->ni[i].reg); + if (IS_ERR(irq_data->ni_base)) + return PTR_ERR(irq_data->ni_base); + irq_data->ni = &hw->ni[i]; + } + + return 0; +} + static int apple_soc_edac_probe(struct platform_device *pdev) { const struct apple_soc_edac_hw *hw; @@ -356,6 +450,9 @@ static int apple_soc_edac_probe(struct platform_device *pdev) irq_data->irq = irq; irq_data->backoff_ms = APPLE_SOC_EDAC_BACKOFF_MS; irq_data->armed_jiffies = jiffies; + ret = apple_soc_edac_map_ni(pdev, irq_data); + if (ret) + goto err_del_device; INIT_DELAYED_WORK(&irq_data->rearm, apple_soc_edac_rearm); ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 38d31dd0e110b..5e5fe26b65502 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -47,6 +47,7 @@ #include #include #include +#include #include "apple_boot_display.h" #include "apple_resume.h" @@ -67,8 +68,15 @@ struct apple_drm_private { * the screen, the DRM device stays unregistered and its DCPs running. */ bool aborted; + /* This suspend found every firmware already stopped. */ + bool s2r_skipped; + /* A dark wake left the outputs as the suspend did, for the next resume. */ + bool dark_held; + struct notifier_block pm_nb; }; +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data); + /* Component readiness is later than platform probe/bind. */ static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; @@ -1766,6 +1774,9 @@ static int apple_drm_init(struct device *dev) goto err_unbind; } + apple->pm_nb.notifier_call = apple_drm_pm_notify; + register_pm_notifier(&apple->pm_nb); + return 0; err_unlock: @@ -1789,6 +1800,8 @@ static void apple_drm_uninit(struct device *dev) if (apple->aborted) return; + unregister_pm_notifier(&apple->pm_nb); + /* Before anything of the DRM device is torn down. */ t8132_ext_drm_fini(&apple->drm); apple_drm_quiesce_connectors(&apple->drm); @@ -1907,12 +1920,38 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); #ifdef CONFIG_PM_SLEEP +/* + * Left stopped by a suspend that never reached the commit. Nothing can commit to + * them, and a refused sleep has to be able to try again. + */ +static bool apple_drm_firmware_stopped(struct apple_drm_private *apple) +{ + struct drm_crtc *crtc; + bool stopped = false; + + drm_for_each_crtc(crtc, &apple->drm) { + if (t8132_ext_crtc(crtc) || + !dcp_s2r_firmware_stopped(to_apple_crtc(crtc)->dcp)) + return false; + stopped = true; + } + + return stopped; +} + static int apple_platform_suspend(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple && !apple->aborted) - return drm_mode_config_helper_suspend(&apple->drm); + /* Still suspended from before a dark wake. */ + if (apple && apple->dark_held) + return 0; + + if (apple && !apple->aborted) { + apple->s2r_skipped = apple_drm_firmware_stopped(apple); + if (!apple->s2r_skipped) + return drm_mode_config_helper_suspend(&apple->drm); + } return 0; } @@ -1934,29 +1973,71 @@ static bool apple_drm_hold_resume_events(struct apple_drm_private *apple, u32 cr return held; } +static void apple_drm_resume_outputs(struct apple_drm_private *apple) +{ + u32 dropped = 0; + + apple->dark_held = false; + if (apple->drm.mode_config.suspend_state) + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped && apple_drm_hold_resume_events(apple, dropped)) + dropped = 0; + drm_mode_config_helper_resume(&apple->drm); + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - u32 dropped = 0; - if (apple && !apple->aborted) { - if (apple->drm.mode_config.suspend_state) - dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); - /* Have userspace set the mode of a CRTC resumed disabled again. */ - if (dropped && apple_drm_hold_resume_events(apple, dropped)) - dropped = 0; - drm_mode_config_helper_resume(&apple->drm); - if (dropped) - drm_kms_helper_hotplug_event(&apple->drm); + if (!apple || apple->aborted || apple->s2r_skipped) + return 0; + + /* A dark wake goes back to sleep with user space frozen: nothing is lit for it. */ + if (pm_dark_wake()) { + apple->dark_held = true; + dev_info(dev, "dark wake: outputs stay off\n"); + return 0; } + apple_drm_resume_outputs(apple); return 0; } +/* The suspend after a dark wake failed before it suspended this device. */ +static void apple_platform_complete(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple && apple->dark_held && !pm_dark_wake()) + apple_drm_resume_outputs(apple); +} + +/* It failed before ->prepare() reached this device, so ->complete() didn't run either. */ +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_drm_private *apple = container_of(nb, struct apple_drm_private, pm_nb); + + if (action == PM_POST_SUSPEND && apple->dark_held) { + dev_warn(apple->drm.dev, "dark wake over without a resume: outputs back now\n"); + apple_drm_resume_outputs(apple); + } + + return NOTIFY_DONE; +} + static const struct dev_pm_ops apple_platform_pm_ops = { .suspend = apple_platform_suspend, .resume = apple_platform_resume, + .complete = apple_platform_complete, }; +#else +static int apple_drm_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + return NOTIFY_DONE; +} #endif static struct platform_driver apple_platform_driver = { diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index de8fc565a11dd..92ff5ce615c85 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1692,7 +1693,23 @@ static __maybe_unused int dcpaud_resume(struct device *dev) return 0; } -static DEFINE_RUNTIME_DEV_PM_OPS(dcpaud_pm_ops, dcpaud_suspend, dcpaud_resume, NULL); +/* The SIO channel and the sink's link both end in a sleep that reaches the hardware. */ +static __maybe_unused int dcpaud_system_suspend(struct device *dev) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + + if (pm_suspend_via_firmware() && dcpaud->substream && dcpaud->substream->ref_count) { + dev_err(dev, "a stream is open, refusing the sleep\n"); + return -EBUSY; + } + + return pm_runtime_force_suspend(dev); +} + +static const struct dev_pm_ops dcpaud_pm_ops = { + SYSTEM_SLEEP_PM_OPS(dcpaud_system_suspend, pm_runtime_force_resume) + RUNTIME_PM_OPS(dcpaud_suspend, dcpaud_resume, NULL) +}; static const struct of_device_id dcpaud_of_match[] = { { .compatible = "apple,dpaudio" }, diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index eb11794f62cf3..6add812e7d2d3 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -592,6 +592,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, scoped_guard(mutex, &dcp->tb_lock) { route->deactivate_xbar_done = false; route->deactivate_error = 0; + route->sleep_parked = false; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -639,7 +640,10 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return route->deactivate_error; if (!route->deactivate_xbar_done) { dcp_typec_route_phy_off(route); - if (!route->tunnel || route->xbar_up) + /* Parked for a sleep, its output is already down and not held. */ + if (route->sleep_parked) + ret = 0; + else if (!route->tunnel || route->xbar_up) ret = mux_control_deselect(route->active_xbar ?: route->xbar); else ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); @@ -653,6 +657,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) ret = 0; /* Deselect always unlocks: never consume that selection twice. */ route->deactivate_xbar_done = true; + route->sleep_parked = false; route->xbar_up = false; if (ret) { /* Idle is unproven; retain the owner and the real error. */ @@ -4328,6 +4333,53 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) return dcp_fabric_run_probe(!!routes, dcp_register_typec_routes_present, &ctx); } +/* + * A sleep that reaches its commit powers the crossbar off. Take the output of a direct + * route down while it still has power, as a released display's is, and select it again + * after the wake. The route stays, and with it the display on the port. + */ +void dcp_typec_route_park(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route || route->tunnel || !route->selected) + return; + guard(mutex)(&dcp->tb_lock); + if (route->deactivate_xbar_done || route->sleep_parked) + return; + /* Deselect always unlocks. */ + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + route->sleep_parked = true; + dev_dbg(dcp->dev, "display route parked for the sleep: crossbar output down: %d\n", ret); +} + +void dcp_typec_route_unpark(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + route = dcp->active_typec_route; + if (!route) + return; + guard(mutex)(&dcp->tb_lock); + if (!route->sleep_parked) + return; + route->sleep_parked = false; + ret = mux_control_try_select(route->active_xbar ?: route->xbar, route->mux_index); + if (ret) { + /* Nothing is held, so the route's release has nothing to give back. */ + route->deactivate_xbar_done = true; + dev_err(dcp->dev, "display route not back after the sleep: crossbar output: %d\n", + ret); + return; + } + dev_dbg(dcp->dev, "display route back after the sleep: crossbar output up\n"); +} + /* * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY * (see dcp_typec_route_deactivate()). Point it at the HDMI output again diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 2db549172d1d5..c06939487174c 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -51,6 +51,8 @@ struct apple_dcp_typec_route { /* Lower mux release is consumed even when idle programming fails. */ bool deactivate_xbar_done; int deactivate_error; + /* Its crossbar output is down for a sleep, and not held. */ + bool sleep_parked; }; int dcp_typec_phy_on(struct apple_dcp *dcp); @@ -111,6 +113,8 @@ bool dcp_is_t8132_ext(const struct apple_dcp *dcp); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); +void dcp_typec_route_park(struct apple_dcp *dcp); +void dcp_typec_route_unpark(struct apple_dcp *dcp); void dcp_typec_routes_disable(struct apple_dcp *dcp); void dcp_typec_routes_disable_dev(struct device *dev); void dcp_typec_retrain_work(struct work_struct *work); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9573412f94b58..fef631d5970e1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -264,6 +265,13 @@ struct apple_dcp_hw_data { * (desktop 26.x), see dcp_sink_metadata(). */ bool sink_metadata_edid; + /* + * With H17P firmware: a firmware-driven system sleep stops the internal processor's + * firmware and powers it down, as DPMS off does, then restarts it warm, see + * dcp_s2r_suspend(). @s2r_ext: the same for an external one, see dcp_s2r_ext_suspend(). + */ + bool s2r; + bool s2r_ext; }; enum dcp_test_point { @@ -368,6 +376,8 @@ const struct drm_edid *dcp_panel_edid_snapshot(struct apple_dcp *dcp, unsigned int count, u64 *generation); bool dcp_panel_metadata_current(struct apple_dcp *dcp, u64 generation); +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on); + struct apple_dcp { /* Generation-specific wire/transaction state, sharing the common KMS owner. */ struct iomfb_m4 *v26_6; @@ -665,6 +675,38 @@ struct apple_dcp { /* Messages from the coprocessor are held in the mailbox for sleep. */ bool rx_held; + /* A firmware-driven suspend stops the firmware, see dcp_s2r_suspend(). */ + bool s2r_active; + bool s2r_stopped; + /* DPMS off stopped the firmware and lowered the domains; DPMS on brings them back. */ + bool dpms_powered_off; + /* Refused its idle with the display off: the firmware still runs, the display is dark. */ + bool s2r_refused_off; + /* The firmware's secure mailbox, where it has one. */ + struct apple_rtkit *secure_rtk; + bool secure_answered; + bool secure_polling; + /* Polled every jiffy until then, see dcp_secure_poll(). */ + unsigned long secure_fast_until; + int secure_map_ret; + s64 secure_map_us; + struct delayed_work secure_poll_wq; + /* Stopped and not restarted, which outlives a suspend that gets refused. */ + bool s2r_fw_down; + /* DCPEXT kept down through a dark wake, see dcp_s2r_dark_finish(). */ + bool s2r_dark_held; + struct notifier_block s2r_pm_nb; + /* This driver raised the display domains, and nothing has lowered them since. */ + bool s2r_domains_up; + /* Counted in pmc-dcp, the number of powered DCPs */ + bool pmc_up; + bool s2r_power_on; + bool s2r_power_oob; + u8 s2r_step; + struct completion s2r_dart_done; + struct device *s2r_disp_dart; + struct device *s2r_dcp_dart; + /* eDP display without DP-HDMI conversion */ bool main_display; @@ -977,6 +1019,7 @@ int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, const void *data, size_t size, void *cookie); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); +void iomfb_reset_transport(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout); void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8dd13cb9c6c33..42a5d410cd515 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -19,13 +19,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -149,6 +152,135 @@ static void dcp_release_context(void *data) kfree(dcp); } +#define DCP_SECURE_EP 0x30 +#define DCP_SECURE_TIMEOUT_MS 200 + +/* + * The firmware's secure mailbox. No other software answers it under Linux, so the + * driver does, as macOS's secure side does: 3 is answered with 2, and 4 needs nothing. + * What these messages mean isn't known. + */ +static void dcp_secure_recv(void *cookie, u8 ep, u64 msg) +{ + struct apple_dcp *dcp = cookie; + int ret; + + dev_dbg(dcp->dev, "secure port: ep %#x msg %#llx\n", ep, msg); + if (ep != DCP_SECURE_EP || (msg >> 60) != 3) + return; + + ret = apple_rtkit_send_message(dcp->secure_rtk, ep, 2ULL << 60, NULL, false); + if (ret) + dev_warn(dcp->dev, "secure port: no reply sent: %d\n", ret); +} + +static const struct apple_rtkit_ops dcp_secure_rtkit_ops = { + .recv_message = dcp_secure_recv, +}; + +/* + * The port's interrupts aren't in the DT. Its handshakes, idle and restart poll it themselves; + * this answers the rest. Every jiffy for a second after its start or a message, as more tend to + * follow, otherwise every 100 ms: woken every jiffy, an idle CPU never reaches deep idle. + */ +#define DCP_SECURE_POLL_FAST_MS 1000 +#define DCP_SECURE_POLL_SLOW_MS 100 + +static void dcp_secure_poll(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + secure_poll_wq); + bool fast = time_before(jiffies, dcp->secure_fast_until); + + if (!READ_ONCE(dcp->secure_polling)) + return; + + if (apple_rtkit_poll(dcp->secure_rtk) > 0) { + if (!fast) + dev_info_once(dcp->dev, "secure port: a message while idle\n"); + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + fast = true; + } + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, + fast ? 1 : msecs_to_jiffies(DCP_SECURE_POLL_SLOW_MS)); +} + +static void dcp_secure_poll_start(struct apple_dcp *dcp) +{ + dcp->secure_fast_until = jiffies + msecs_to_jiffies(DCP_SECURE_POLL_FAST_MS); + WRITE_ONCE(dcp->secure_polling, true); + queue_delayed_work(system_unbound_wq, &dcp->secure_poll_wq, 0); +} + +static void dcp_secure_poll_stop(struct apple_dcp *dcp) +{ + WRITE_ONCE(dcp->secure_polling, false); + cancel_delayed_work_sync(&dcp->secure_poll_wq); +} + +/* After the firmware's HELLO on its main mailbox. */ +static void dcp_secure_port_start(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->secure_rtk) + return; + + ret = apple_rtkit_port_handshake(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) { + dev_warn(dcp->dev, "secure port handshake not finished: %d\n", ret); + return; + } + dcp->secure_answered = true; + dcp_secure_poll_start(dcp); + dev_info(dcp->dev, "secure port up\n"); +} + +/* Between the main mailbox's AP idle and its IOP idle. */ +static void dcp_secure_port_idle(void *data) +{ + struct apple_dcp *dcp = data; + int ret; + + if (!dcp->secure_rtk || !dcp->secure_answered) + return; + + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_idle(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP idle not acknowledged: %d\n", ret); +} + +static void dcp_release_secure_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + dcp_secure_poll_stop(dcp); + apple_rtkit_free(dcp->secure_rtk); + dcp->secure_rtk = NULL; +} + +/* Nothing is read from the port until its handshake is wanted. */ +static int dcp_secure_port_init(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + + if (of_property_match_string(dev->of_node, "mbox-names", "secure") < 0) + return 0; + + dcp->secure_rtk = apple_rtkit_init(dev, dcp, "secure", 0, &dcp_secure_rtkit_ops); + if (IS_ERR(dcp->secure_rtk)) { + int ret = PTR_ERR(dcp->secure_rtk); + + dcp->secure_rtk = NULL; + return dev_err_probe(dev, ret, "Failed to initialize the secure port\n"); + } + + INIT_DELAYED_WORK(&dcp->secure_poll_wq, dcp_secure_poll); + + return devm_add_action_or_reset(dev, dcp_release_secure_rtkit, dcp); +} + static void dcp_release_rtkit(void *data) { struct apple_dcp *dcp = data; @@ -1610,11 +1742,16 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) return iomfb_v14_7_atomic_check(dcp, crtc, commit); - if (dcp->crashed) - return -EINVAL; - crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + /* + * A crashed DCP can't be lit, but a commit that leaves its CRTC off has to pass: + * the system suspend's disable-all includes every CRTC, and a stopped external DCP + * would refuse every sleep after the first. + */ + if (dcp->crashed) + return crtc_state && crtc_state->enable ? -EINVAL : 0; + /* External H17P does not program adaptive sync. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && @@ -3902,6 +4039,7 @@ int dcp_start(struct platform_device *pdev) /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ init_completion(&dcp->start_done); + init_completion(&dcp->s2r_dart_done); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) return iomfb_v26_6_start(dcp); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) @@ -4214,6 +4352,9 @@ static bool dcp_h17g_panel(struct apple_dcp *dcp) dcp_has_panel(dcp); } +static void dcp_dpms_power_off(struct apple_dcp *dcp); +static void dcp_dpms_power_on(struct apple_dcp *dcp); + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { /* The 14.7 panel has its own known-level blank and restore hooks. */ @@ -4283,6 +4424,7 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { + dcp_dpms_power_on(dcp); if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { @@ -4435,11 +4577,15 @@ void dcp_poweroff(struct platform_device *pdev) cancel_delayed_work(&dcp->swap_watchdog_wq); - /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + /* + * Internal H17P DPMS is a brightness present, never a pipe stop. With hw.s2r the + * display then powers off as well, after the backlight. + */ if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + dcp_dpms_power_off(dcp); return; } @@ -5615,6 +5761,12 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; + /* After a warm restart its log counters need echoing, or it refuses its next idle. */ + if (dcp->hw.s2r || dcp->hw.s2r_ext) + apple_rtkit_set_oslog_echo(dcp->rtk, true); + ret = dcp_secure_port_init(dcp); if (ret) goto err_piodma; @@ -5633,6 +5785,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } + dcp_secure_port_start(dcp); enable_work(&dcp->dimensions_wq); schedule_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); @@ -6012,6 +6165,8 @@ static void dcp_t8132_ext_routes(struct apple_dcp *dcp) dcp->nr_typec_routes); } +static int dcp_s2r_pm_register(struct apple_dcp *dcp); + static int dcp_platform_probe_pipe(struct platform_device *pdev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); @@ -6191,6 +6346,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) } platform_set_drvdata(pdev, dcp); + ret = dcp_s2r_pm_register(dcp); + if (ret) + return ret; if (iomfb_is_m4(dcp)) { dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; @@ -6811,10 +6969,13 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) */ static void dcp_hold_for_sleep(struct apple_dcp *dcp) { - /* A failed component bind leaves no coprocessor session to hold. */ + /* + * A failed component bind leaves no coprocessor session to hold. Firmware that DPMS + * stopped sends nothing, and its presentation stays paused until DPMS restarts it. + */ if (IS_ERR_OR_NULL(dcp->rtk) || !dcp_uses_soft_dpms(dcp) || !dcp_presents_nits(dcp) || - READ_ONCE(dcp->crashed)) + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->s2r_fw_down)) return; iomfb_pause_presentation(dcp, true); @@ -6914,10 +7075,13 @@ static int dcp_platform_prepare(struct device *dev) return 0; } +static void dcp_s2r_dark_finish(struct apple_dcp *dcp); + static void dcp_platform_complete(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_s2r_dark_finish(dcp); if (!dcp->external) return; mutex_lock(&dcp->hpd_mutex); @@ -6925,105 +7089,1247 @@ static void dcp_platform_complete(struct device *dev) mutex_unlock(&dcp->hpd_mutex); } -static int dcp_platform_suspend(struct device *dev) +/* + * Suspend-to-RAM with hw.s2r: the sleep engine waits for the DCP's domain to be off, so + * the firmware has to stop first. Measured from macOS on T8140, in order: + * + * 1. A414(0), A478 x2, A476(1), A478, A412, A473(0). + * 2. D572(0) from the firmware: the display DART's streams go off. + * 3. A500, unnamed, then D107s and D575(0) from the firmware. + * 4. RTKit idle, then the ASC's run bit off. + * 5. The DCP DART's streams off. That's the coprocessor's own shutdown, apart + * from step 2. The sleep driver then lowers the domains. + * + * macOS also restarts the firmware 893 ms before the commit, calls A478, A473(0), + * A115, A500, and stops it again. Why isn't known, and that's not done here. + * + * The firmware takes IOP idle only with its secure mailbox answered, see + * dcp_s2r_answer_secure_port(). + */ +#define DCP_S2R_DISP_STREAMS (BIT(0) | BIT(3) | BIT(4) | BIT(16)) +#define DCP_S2R_DCP_STREAMS (BIT(23) | BIT(24)) +#define DCP_S2R_D572_TIMEOUT_MS 1000 /* macOS: about 170 ms after A473(0) */ +#define DCP_PS_MIN GENMASK(19, 16) + +static struct device *dcp_s2r_find_dart(struct device_node *np) { - struct apple_dcp *dcp = dev_get_drvdata(dev); + struct device_node *dart; + struct platform_device *pdev; - /* Its power sequences may crash the DCP and blank the panel. */ - if (dcp_takeover_was_aborted(dcp)) - return -EBUSY; + dart = of_parse_phandle(np, "iommus", 0); + if (!dart) + return NULL; + pdev = of_find_device_by_node(dart); + of_node_put(dart); - /* - * T8132: setPowerState powers the display down; the coprocessor keeps - * running and its power domains stay on, as below, unless the internal - * display is blanked with appledrm.power_gate_ms set: then the - * coprocessor and its domains are powered down too (iomfb_m4_suspend()). - */ - if (iomfb_is_m4(dcp)) { - int ret = iomfb_m4_suspend(dcp); + return pdev ? &pdev->dev : NULL; +} - if (!ret) - device_set_wakeup_path(dev); - return ret; +static void dcp_s2r_put_darts(struct apple_dcp *dcp) +{ + if (dcp->s2r_dcp_dart) + put_device(dcp->s2r_dcp_dart); + if (dcp->s2r_disp_dart) + put_device(dcp->s2r_disp_dart); + dcp->s2r_dcp_dart = NULL; + dcp->s2r_disp_dart = NULL; +} + +static void dcp_s2r_find_darts(struct apple_dcp *dcp) +{ + struct device_node *piodma; + + /* A suspend that failed after an earlier lookup still holds those. */ + dcp_s2r_put_darts(dcp); + dcp->s2r_dcp_dart = dcp_s2r_find_dart(dcp->dev->of_node); + piodma = of_get_child_by_name(dcp->dev->of_node, "piodma"); + if (piodma) { + dcp->s2r_disp_dart = dcp_s2r_find_dart(piodma); + of_node_put(piodma); } - if (READ_ONCE(dcp->ext_dead)) - return 0; - iomfb_v14_7_backlight_suspend(dcp); +} - /* Serialize PM against explicit bring-up, including queued startup work. - * Before a startup attempt the device remains suspendable. After any - * attempt we cannot prove DMA/firmware quiescence, even on failure. - */ - if (dcp->external) { - const char *why; +void dcp_s2r_dart_power(struct apple_dcp *dcp, u32 on) +{ + if (!READ_ONCE(dcp->s2r_active)) + return; - mutex_lock(&dcp->hpd_mutex); - why = dcp_external_sleep_vetoed(dcp); - if (why) { - mutex_unlock(&dcp->hpd_mutex); - if (dcp->external_native) - dev_warn(dev, "suspend refused: %s\n", why); - else - dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); - return -EBUSY; - } - WRITE_ONCE(dcp->external_suspended, true); - mutex_unlock(&dcp->hpd_mutex); + if (dcp->s2r_disp_dart) + apple_dart_display_streams(dcp->s2r_disp_dart, DCP_S2R_DISP_STREAMS, on, false); + complete(&dcp->s2r_dart_done); +} + +static bool dcp_s2r_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.s2r; +} + +static void dcp_s2r_cpu_stop(struct apple_dcp *dcp) +{ + /* macOS writes it twice. */ + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel(0, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); +} + +/* Up from bind, unless its handshake failed then. A bound firmware needs it to take IOP idle. */ +static void dcp_s2r_answer_secure_port(struct apple_dcp *dcp) +{ + if (!dcp->secure_answered) + dcp_secure_port_start(dcp); +} + +/* After a refused idle: AP on to both mailboxes, as the GPU and the AOP send it. */ +static void dcp_s2r_idle_undo(struct apple_dcp *dcp) +{ + int ret; + + ret = apple_rtkit_ap_power(dcp->rtk, true); + if (ret) + dev_warn(dcp->dev, "AP on after the refused idle: %d\n", ret); + if (!dcp->secure_rtk || !dcp->secure_answered || READ_ONCE(dcp->secure_polling)) + return; + ret = apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port AP on after the refused idle: %d\n", ret); + dcp_secure_poll_start(dcp); +} + +/* + * macOS's order up to RTKit idle. On success the firmware is idle and its run bit + * is still set. + */ +static int dcp_s2r_quiesce(struct apple_dcp *dcp, u32 *status) +{ + ktime_t start = ktime_get(); + /* Off since a refused idle: no second set_power_state(0), whose D572 may never come. */ + bool display_off = dcp->s2r_refused_off; + bool idle_sent = false; + s64 power_ms; + bool idle; + int ret; + + reinit_completion(&dcp->s2r_dart_done); + dcp_s2r_find_darts(dcp); + WRITE_ONCE(dcp->s2r_active, true); + + if (display_off) { + dev_info(dcp->dev, "display still off from a refused idle\n"); + goto display_is_off; } - if (dcp_is_external_h17p(dcp)) { - int ret; + /* In band, 3 of 3 on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, false, true, false); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(0): %d\n", ret); + goto err; + } + power_ms = ktime_ms_delta(ktime_get(), start); - dcp_disable_typec_work(dcp, false); - ret = dcp_external_disconnect(dcp, false); - if (ret) { - dcp_enable_typec_work(dcp); - return ret; - } - device_set_wakeup_path(dev); - return 0; + /* Without D572 the display isn't known to be off, and nothing below can undo this. */ + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) { + dev_err(dcp->dev, "no D572 %lld ms after set_power_state(0)\n", + ktime_ms_delta(ktime_get(), start)); + ret = -ETIMEDOUT; + goto err; } + dev_dbg(dcp->dev, "display off: set_power_state(0) %lld ms, D572 %lld ms\n", power_ms, + ktime_ms_delta(ktime_get(), start)); + display_off = true; - dcp_drain_for_sleep(dcp); - /* - * The Type-C route reports cable removal through - * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays - * connected, and resume powers the CRTC back up through dcp_poweron() - * as after DPMS off. - */ - dcp_disable_typec_work(dcp, false); - cancel_delayed_work_sync(&dcp->swap_watchdog_wq); - if (dcp->external_native) - disable_work_sync(&dcp->external_audio_work); +display_is_off: + /* D572 itself is still being answered, and a busy channel can't be sent on. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before A500\n"); - if (dcp->avep) - av_service_disconnect(dcp); + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + goto err; + } + dev_dbg(dcp->dev, "A500 replied after %lld ms\n", ktime_ms_delta(ktime_get(), start)); - if (dcp->hdmi_hpd_irq) { - disable_irq(dcp->hdmi_hpd_irq); - if (!dcp->active_typec_route) { - disconnected_hpd_event(dcp->connector); - dcp_dptx_disconnect(dcp, 0); - } + /* The firmware's callbacks after A500 have to be answered first. */ + usleep_range(5000, 6000); + if (read_poll_timeout(iomfb_channels_idle, idle, idle, 1000, 100000, false, dcp)) + dev_warn(dcp->dev, "firmware calls still in flight before idle\n"); + + idle_sent = true; + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); + if (ret) { + /* Answer what it still has queued, its log counters among them, and ask again. */ + dev_warn(dcp->dev, "firmware refused idle after %lld ms: %d, control %#x status %#x: retrying once\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + apple_rtkit_poll(dcp->rtk); + msleep(30); + ret = apple_rtkit_idle_with(dcp->rtk, dcp_secure_port_idle, dcp); } + if (ret) { + dev_err(dcp->dev, "firmware did not go idle after %lld ms: %d, control %#x status %#x\n", + ktime_ms_delta(ktime_get(), start), ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + goto err; + } + + /* Both set once the firmware's last message has been read; macOS waits for them. */ + if (readl_poll_timeout(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS, *status, + (*status & (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) == + (APPLE_DCP_COPROC_CPU_STATUS_IDLE | + APPLE_DCP_COPROC_CPU_STATUS_STOPPED), 100, 5000)) + dev_warn(dcp->dev, "idle status not set: %#x\n", *status); + dev_dbg(dcp->dev, "firmware idle after %lld ms, status %#x\n", + ktime_ms_delta(ktime_get(), start), *status); + + WRITE_ONCE(dcp->s2r_active, false); + dcp->s2r_refused_off = false; + + return 0; + +err: + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); /* - * Set the device as a wakeup device, which forces its power - * domains to stay on. We need this as we do not support full - * shutdown properly yet. + * Refused with the display off: leave the firmware running, never stop it under its + * own feet. DPMS on lights the display again on the running firmware. */ - device_set_wakeup_path(dev); - /* Last: the steps above still talk to the coprocessor. */ - dcp_hold_for_sleep(dcp); + if (display_off) { + if (idle_sent) + dcp_s2r_idle_undo(dcp); + dcp->s2r_refused_off = true; + dev_warn(dcp->dev, "display left off with the firmware running\n"); + return ret; + } + /* The display may be off and the firmware half idle. Nothing here brings it back. */ + WRITE_ONCE(dcp->crashed, true); - return 0; + return ret; } -static int dcp_platform_resume(struct device *dev) +/* Its power state can only be lowered once this has finished. */ +static int dcp_s2r_stop_streams(struct apple_dcp *dcp) { - struct apple_dcp *dcp = dev_get_drvdata(dev); + if (!dcp->s2r_dcp_dart) + return -ENODEV; - if (dcp_takeover_was_aborted(dcp)) - return 0; + return apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, false, true); +} + +static int dcp_s2r_stop_firmware(struct apple_dcp *dcp) +{ + u32 status; + int ret; + + ret = dcp_s2r_quiesce(dcp, &status); + if (ret) + return ret; + + apple_rtkit_mailbox_off(dcp->rtk); + dcp_s2r_cpu_stop(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + return dcp_s2r_stop_streams(dcp); +} + +/* + * macOS stops this firmware with one call the size of A500, then about 4 ms later AP + * then IOP idle and the run bit off twice. Nothing has shown it needing a secure + * port. + * + * An attached display goes first, as macOS's DP port releases its display at system + * sleep: the pipe off, HPD low and the display released, as DPMS off does, then the + * crossbar output. The route and the connector stay, and after the wake the link + * connects again, as macOS replays the port's HPD. + */ +static bool dcp_s2r_ext_capable(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.s2r_ext; +} + +/* + * Taken as stopped, but a start since raised its domains and failed after the run bit, + * as one a signal ended. Left running, it refuses every sleep; stopped, it comes back + * from the sleep's power cycle like any other. + */ +static void dcp_s2r_stop_failed_start(struct apple_dcp *dcp) +{ + if (!dcp->s2r_domains_up || + !(readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL) & + APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return; + dev_warn(dcp->dev, "a failed start left the firmware running: stopping it for the sleep\n"); + dcp_s2r_cpu_stop(dcp); +} + +/* + * pmc-dcp counts the powered DCPs, as macOS keeps it: 0, 1, 2, 1, 0 across a wake. A DCP + * counts from when this driver powers it up until it powers it off or a sleep stops it. + */ +static DEFINE_MUTEX(dcp_pmc_lock); +static unsigned int dcp_pmc_count; + +static void dcp_pmc_set(struct apple_dcp *dcp, void __iomem *pmc, bool up) +{ + guard(mutex)(&dcp_pmc_lock); + if (up != dcp->pmc_up) { + dcp->pmc_up = up; + if (up) + dcp_pmc_count++; + else if (dcp_pmc_count) + dcp_pmc_count--; + } + writel(dcp_pmc_count, pmc); +} + +/* The sleep controller lowers the domains and pmc-dcp itself. */ +static void dcp_pmc_forget(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_pmc_lock); + if (dcp->pmc_up && dcp_pmc_count) + dcp_pmc_count--; + dcp->pmc_up = false; +} + +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp); + +/* + * Crashed firmware takes no stop sequence, and refusing would hold every later sleep off until + * reboot. Its CPU stops cold, as for a failed start, and the wake starts it again, as macOS + * recovers a crashed DCP. A system sleep only: DPMS off still refuses it. + */ +static bool dcp_s2r_stops_crashed(struct apple_dcp *dcp) +{ + return READ_ONCE(dcp->crashed) && !IS_ERR_OR_NULL(dcp->rtk) && + pm_suspend_target_state != PM_SUSPEND_ON; +} + +static int dcp_s2r_stop_crashed(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + dev_warn(dcp->dev, "firmware crashed: stopped cold for the sleep, started again after it\n"); + apple_rtkit_mailbox_off(dcp->rtk); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static int dcp_s2r_ext_suspend(struct apple_dcp *dcp) +{ + int ret; + + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + dcp_typec_route_park(dcp); + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + return 0; + } + + /* No A500 and no disconnect: the link and its crossbar output just go down. */ + if (dcp_s2r_stops_crashed(dcp)) { + dcp_disable_typec_work(dcp, false); + dcp_typec_route_park(dcp); + return dcp_s2r_stop_crashed(dcp); + } + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) + dev_dbg(dcp->dev, "a display is attached: released for the sleep, connected again after it\n"); + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dcp_enable_typec_work(dcp); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* The display's link went above: its crossbar output follows, as on macOS. */ + dcp_typec_route_park(dcp); + + ret = iomfb_s2r_500_h17p(dcp); + if (ret) { + dev_err(dcp->dev, "no reply to A500: %d\n", ret); + WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + usleep_range(4000, 5000); + + ret = apple_rtkit_sleep_stop(dcp->rtk, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (ret) { + dev_err(dcp->dev, "firmware did not go idle: %d, control %#x status %#x\n", ret, + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + WRITE_ONCE(dcp->crashed, true); + dcp_typec_route_unpark(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + WRITE_ONCE(dcp->s2r_fw_down, true); + dev_dbg(dcp->dev, "firmware stopped for sleep: control %#x status %#x\n", + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL), + readl(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS)); + + ret = dcp_s2r_stop_streams(dcp); + if (ret) { + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +/* A failure before the firmware has stopped leaves it running, and refuses cleanly. */ +static int dcp_s2r_suspend(struct apple_dcp *dcp) +{ + bool was_off; + int ret; + + /* Still stopped from a suspend that never reached the commit. */ + if (READ_ONCE(dcp->s2r_fw_down)) { + dcp_s2r_stop_failed_start(dcp); + dcp_s2r_find_darts(dcp); + dcp->s2r_stopped = true; + ret = dcp_s2r_stop_streams(dcp); + if (ret) + return ret == -ETIMEDOUT ? -EBUSY : ret; + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + return 0; + } + + if (dcp_s2r_stops_crashed(dcp)) + return dcp_s2r_stop_crashed(dcp); + if (READ_ONCE(dcp->crashed) || IS_ERR_OR_NULL(dcp->rtk)) + return -EBUSY; + + dcp_s2r_answer_secure_port(dcp); + + dcp_drain_for_sleep(dcp); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + dev_err(dcp->dev, "display updates still pending, refusing sleep\n"); + return -EBUSY; + } + + /* Presentation stays paused if this fails: see dcp_s2r_quiesce(). */ + was_off = dcp->s2r_refused_off; + ret = dcp_s2r_stop_firmware(dcp); + if (ret) { + if (READ_ONCE(dcp->s2r_fw_down)) + dcp_s2r_restart_failed_stop(dcp); + return ret == -ETIMEDOUT ? -EBUSY : ret; + } + /* Dark since a refused idle, as after DPMS off: it stays so until DPMS on. */ + if (was_off) + dcp->dpms_powered_off = true; + + dcp_pmc_forget(dcp); + device_set_wakeup_path(dcp->dev); + + return 0; +} + +static void dcp_s2r_ps_lower(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & BIT(28)) { + val &= ~(BIT(28) | GENMASK(9, 8)); + writel(val, reg); + } + writel((val & ~GENMASK(3, 0)) | GENMASK(9, 8), reg); +} + +/* macOS's power-off, measured 3 of 3. The firmware is stopped and its DARTs are off. */ +static int dcp_s2r_domains_off(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + dcp->s2r_domains_up = false; + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + dcp_pmc_set(dcp, pmc, false); + val = readl(ps + 0x10000); + if (val & BIT(28)) + writel(val & ~(BIT(28) | GENMASK(9, 8)), ps + 0x10000); + writel(3, pwrgate + 0xc8c); + writel(0, pwrgate + 0xcd0); + dcp_s2r_ps_lower(ps + 0x10000); + dcp_s2r_ps_lower(ps + 0x318); + dcp_s2r_ps_lower(ps + 0x280); + + ret = readl_poll_timeout(ps + 0x10000, val, !FIELD_GET(GENMASK(7, 4), val), 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power off: %#x\n", val); + dev_dbg(dcp->dev, "display domains off: CPU %#x FE %#x SYS %#x\n", readl(ps + 0x10000), + readl(ps + 0x318), readl(ps + 0x280)); +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * Exact mirror of the writes in macOS's power-off, measured 3 of 3. DISP_SYS and + * DISP_FE first, then DISP_CPU with AUTO_ENABLE last. + */ +static int dcp_s2r_domains_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps = NULL, *pwrgate = NULL, *pmc = NULL; + u32 val; + int ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pwrgate = of_iomap(np, of_property_match_string(np, "reg-names", "pwrgate")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pwrgate || !pmc) + goto out; + + val = readl(ps + 0x280) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x280); + writel(val | 0xf | BIT(28), ps + 0x280); + val = readl(ps + 0x318) & ~(GENMASK(9, 8) | GENMASK(3, 0)); + writel(val | 0xf, ps + 0x318); + val = readl(ps + 0x10000) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + 0x10000); + writel(0, pwrgate + 0xc8c); + writel(1, pwrgate + 0xcd0); + writel(val | 0xf | BIT(28), ps + 0x10000); + dcp_pmc_set(dcp, pmc, true); + + ret = readl_poll_timeout(ps + 0x10000, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) + dev_err(dcp->dev, "DISP_CPU did not power on: %#x\n", val); + else + dcp->s2r_domains_up = true; +out: + if (ps) + iounmap(ps); + if (pwrgate) + iounmap(pwrgate); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * DISPEXT0 up as macOS does it after a wake: SYS and FE on, then the CPU with PS_MIN 4 first, + * since the wake reset it, then on, and pmc-dcp up once it reads on. + */ +static int dcp_s2r_ext_domains_on(struct apple_dcp *dcp) +{ + static const u16 sys_fe[] = { 0x288, 0x320 }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + u32 val; + int i, ret = -ENODEV; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + for (i = 0; i < ARRAY_SIZE(sys_fe); i++) { + val = readl(ps + sys_fe[i]) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28)); + writel(val | 0xf, ps + sys_fe[i]); + writel(val | 0xf | BIT(28), ps + sys_fe[i]); + } + val = readl(ps + 0x328) & ~(GENMASK(9, 8) | GENMASK(3, 0) | BIT(28) | DCP_PS_MIN); + val |= FIELD_PREP(DCP_PS_MIN, 4); + writel(val, ps + 0x328); + writel(val | 0xf, ps + 0x328); + writel(val | 0xf | BIT(28), ps + 0x328); + + ret = readl_poll_timeout(ps + 0x328, val, FIELD_GET(GENMASK(7, 4), val) == 0xf, + 100, 50000); + if (ret) { + dev_err(dcp->dev, "DISPEXT0_CPU did not power on: %#x\n", val); + } else { + dcp_pmc_set(dcp, pmc, true); + dcp->s2r_domains_up = true; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +/* + * A firmware stopped while crashed is about to start over: forget the calls and the + * transaction the old one left unanswered, and let the new one's messages through. A failed + * start marks it crashed again. + */ +static void dcp_s2r_forget_crash(struct apple_dcp *dcp) +{ + if (!READ_ONCE(dcp->crashed)) + return; + iomfb_reset_transport(dcp); + WRITE_ONCE(dcp->crashed, false); +} + +/* + * DCPEXT back after a real sleep, the same way in every macOS wake measured: its domains, + * its DART's streams, and a warm restart, with no secure port and no display-on call. After + * its own power cycle the start is the same, with its display DART lit again. + */ +static int dcp_s2r_ext_start(struct apple_dcp *dcp, bool cycled) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + dcp_s2r_forget_crash(dcp); + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (!ret) + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + /* After the domains, whose writes it would reject, the DART protects them again. */ + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + if (!cycled) + return apple_rtkit_sleep_return(dcp->rtk, control); + + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, false); + return apple_rtkit_power_cycled_return(dcp->rtk, control); +} + +/* DISPEXT0 off in its own order, CPU first. Only the CPU has to power-gate to start again. */ +static int dcp_s2r_ext_domains_off(struct apple_dcp *dcp) +{ + static const struct { + u16 offset; + u32 first, second; + bool member; + } steps[] = { + { 0x328, 0xf0400ff, 0xf0403f0 }, { 0x320, 0, 0x23f0, true }, { 0x288, 0, 0x23f0 }, + }; + struct device_node *np; + void __iomem *ps = NULL, *pmc = NULL; + int i, ret = -ENODEV; + u32 val; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return -ENODEV; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + pmc = of_iomap(np, of_property_match_string(np, "reg-names", "pmc-dcp")); + of_node_put(np); + if (!ps || !pmc) + goto out; + + /* + * Its display DART goes off with DISPEXT0, and every framebuffer is mapped into it: a + * map during the hold would reach its registers. Dark until the domains are back. + */ + if (dcp->s2r_disp_dart) + apple_dart_power_dark(dcp->s2r_disp_dart, true); + dcp->s2r_domains_up = false; + dcp_pmc_set(dcp, pmc, false); + ret = 0; + for (i = 0; i < ARRAY_SIZE(steps); i++) { + int gated; + + if (steps[i].first) + writel(steps[i].first, ps + steps[i].offset); + writel(steps[i].second, ps + steps[i].offset); + gated = readl_poll_timeout(ps + steps[i].offset, val, + !(val & GENMASK(7, 4)) && !(val & BIT(11)) && + (steps[i].member || (val & BIT(8))), 100, 50000); + dev_dbg(dcp->dev, "DISPEXT0 power word %#x reads %#x: %d\n", steps[i].offset, val, + gated); + if (i == 0 && gated) + ret = gated; + } +out: + if (ps) + iounmap(ps); + if (pmc) + iounmap(pmc); + + return ret; +} + +#define DCP_S2R_EXT_CYCLE_HOLD_MS 1000 + +static bool dcp_s2r_ext_cpu_on(struct apple_dcp *dcp) +{ + struct device_node *np; + void __iomem *ps; + bool on; + + np = of_parse_phandle(dcp->dev->of_node, "apple,display-power", 0); + if (!np) + return false; + ps = of_iomap(np, of_property_match_string(np, "reg-names", "ps")); + of_node_put(np); + if (!ps) + return false; + on = FIELD_GET(GENMASK(7, 4), readl(ps + 0x328)); + iounmap(ps); + + return on; +} + +/* + * After a refused sleep DCPEXT has idled with its power kept, and an idled firmware doesn't + * wake again: power-cycle it through DISPEXT0's own sequence, then start it as after a sleep. + * The sleep driver lowers DISPEXT0 before refusals of its own, and with it off the CPU and the + * display DART don't answer, so it's raised first. + */ +static int dcp_s2r_ext_cycle(struct apple_dcp *dcp) +{ + void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL; + int ret; + + if (!dcp->s2r_dcp_dart) + return -ENODEV; + ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart); + if (ret) + return ret; + if (!dcp_s2r_ext_cpu_on(dcp)) { + dev_info(dcp->dev, "DISPEXT0 lowered by the refused sleep: raised for the cycle\n"); + ret = dcp_s2r_ext_domains_on(dcp); + if (ret) + return ret; + } + + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + writel(readl(control) & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, control); + ret = dcp_s2r_ext_domains_off(dcp); + if (!ret) { + msleep(DCP_S2R_EXT_CYCLE_HOLD_MS); + ret = dcp_s2r_ext_start(dcp, true); + } + + return ret; +} + +/* Between the main mailbox's IOP report and its AP on, measured 2 of 2. */ +static void dcp_secure_port_map(void *data) +{ + struct apple_dcp *dcp = data; + ktime_t start = ktime_get(); + + dcp->secure_map_ret = apple_rtkit_port_map(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + dcp->secure_map_us = ktime_us_delta(ktime_get(), start); +} + +static int dcp_s2r_warm_restart(struct apple_dcp *dcp) +{ + int ret; + + dcp_s2r_forget_crash(dcp); + ret = dcp_s2r_domains_on(dcp); + if (ret) + return ret; + + /* After the domains, whose writes it would reject, the DART protects them again. */ + if (dcp->s2r_dcp_dart) + apple_dart_display_streams(dcp->s2r_dcp_dart, DCP_S2R_DCP_STREAMS, true, false); + + if (dcp->secure_rtk) { + dcp_secure_poll_stop(dcp); + ret = apple_rtkit_port_restart(dcp->secure_rtk); + if (ret) + return ret; + } + + ret = apple_rtkit_warm_restart(dcp->rtk); + if (ret) + return ret; + + writel(APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + ret = apple_rtkit_boot_with(dcp->rtk, dcp->secure_rtk ? dcp_secure_port_map : NULL, dcp); + if (ret) + return ret; + + if (dcp->secure_rtk) { + ret = dcp->secure_map_ret ?: + apple_rtkit_port_on(dcp->secure_rtk, DCP_SECURE_TIMEOUT_MS); + if (ret) + dev_warn(dcp->dev, "secure port handshake after restart: %d\n", ret); + else + dev_dbg(dcp->dev, "secure port up after restart, map took %lld us\n", + dcp->secure_map_us); + dcp_secure_poll_start(dcp); + } + + return 0; +} + +/* + * Display on after a sleep, and after DPMS off: the warm restart, then the named calls + * macOS sends first, A414(0), A478 x2, A476(1), A478, A412, then A473(1). + * + * Not sent: A439, which macOS overlaps with A473(1), A438 around swaps, A461 per frame + * and A352. They stay unnamed, and A439 and A438 hold macOS kernel pointers. Linux + * already lights the panel without A438 at boot. + */ +static int dcp_s2r_display_on(struct apple_dcp *dcp) +{ + int ret; + + ret = dcp_s2r_warm_restart(dcp); + if (ret) { + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + return ret; + } + + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + + /* Out of band; either channel answered on macOS. */ + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (ret) { + dev_err(dcp->dev, "no reply to set_power_state(1): %d\n", ret); + goto out; + } + if (!wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); +out: + WRITE_ONCE(dcp->s2r_active, false); + + return ret; +} + +/* Power kept: lowered as DPMS off does. A failed flush left the GAPF slice faulting that. */ +static int dcp_s2r_refused_domains_off(struct apple_dcp *dcp) +{ + int ret = dcp->s2r_dcp_dart ? apple_dart_gapf_ready(dcp->s2r_dcp_dart) : -ENODEV; + + return ret ?: dcp_s2r_domains_off(dcp); +} + +static void dcp_s2r_restart_done(struct apple_dcp *dcp, int ret) +{ + bool ext = dcp_s2r_ext_capable(dcp); + + dcp_s2r_put_darts(dcp); + /* An attached display's crossbar output; its link reconnects from the Type-C work. */ + if (ext) + dcp_typec_route_unpark(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_warn(dcp->dev, "firmware stopped for sleep, display stays off until reboot\n"); + return; + } + /* Back from a full restart, even after a failed start had marked it crashed. */ + WRITE_ONCE(dcp->crashed, false); + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + if (ext && !READ_ONCE(dcp->typec_work_enabled)) + dcp_enable_typec_work(dcp); +} + +/* + * If the display doesn't come back, the panel stays dark until reboot, and the driver + * must not talk to the stopped firmware, so treat it as a crashed DCP. Nothing else in + * the system depends on it. + */ +static int dcp_s2r_restart(struct apple_dcp *dcp, bool real) +{ + int ret = -EAGAIN; + + /* + * After a refused sleep the firmware has idled with its power kept, and an idled firmware + * doesn't wake again: the internal DCP comes back as DPMS on brings it, from a power + * cycle, and DCPEXT through its own. + */ + if (dcp_s2r_capable(dcp)) { + ret = real ? 0 : dcp_s2r_refused_domains_off(dcp); + if (!ret) + ret = dcp_s2r_display_on(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware restarted as for DPMS: %d\n", + ret); + } else if (dcp_s2r_ext_capable(dcp)) { + ret = real ? dcp_s2r_ext_start(dcp, false) : dcp_s2r_ext_cycle(dcp); + if (!real) + dev_info(dcp->dev, "sleep refused: firmware %s after a power cycle: %d\n", + ret ? "not back" : "back", ret); + else if (ret) + dev_err(dcp->dev, "warm restart failed: %d\n", ret); + } + dcp_s2r_restart_done(dcp, ret); + + return ret; +} + +static int dcp_s2r_resume(struct apple_dcp *dcp) +{ + bool real = pm_resume_via_firmware(); + + dcp->s2r_stopped = false; + /* A real sleep took the domains' power. */ + if (real) + dcp->s2r_domains_up = false; + /* A dark wake sleeps again with the panel never lit, as if DPMS had turned it off. */ + if (pm_dark_wake() && dcp_s2r_capable(dcp) && !dcp->dpms_powered_off) { + dcp->dpms_powered_off = true; + dev_info(dcp->dev, "dark wake: firmware and panel stay off\n"); + } + /* DCPEXT too, its crossbar output parked: dcp_s2r_dark_finish() if no resume follows. */ + if (pm_dark_wake() && dcp_s2r_ext_capable(dcp)) { + dcp->s2r_dark_held = true; + dev_info(dcp->dev, "dark wake: firmware and output stay off\n"); + return 0; + } + dcp->s2r_dark_held = false; + /* Off by DPMS before the sleep: it stays off until DPMS on brings it back. */ + if (dcp->dpms_powered_off) { + /* Off after a refused idle, its power may be on: DPMS on needs it lowered. */ + if (!real) + dcp_s2r_refused_domains_off(dcp); + dcp_s2r_put_darts(dcp); + return 0; + } + dcp_s2r_restart(dcp, real); + + return 0; +} + +/* + * The suspend after a dark wake failed before suspending DCPEXT, or before preparing it. It's + * still down from the real sleep, whose wake reset its CPU: start it as after one. + */ +static void dcp_s2r_dark_finish(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp->s2r_dark_held || pm_dark_wake() || IS_ERR_OR_NULL(dcp->rtk)) + return; + dcp->s2r_dark_held = false; + dcp->s2r_stopped = false; + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_ext_start(dcp, true); + dev_info(dcp->dev, "dark wake over without a resume: firmware %s: %d\n", + ret ? "not back" : "back", ret); + dcp_s2r_restart_done(dcp, ret); +} + +static int dcp_s2r_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct apple_dcp *dcp = container_of(nb, struct apple_dcp, s2r_pm_nb); + + if (action == PM_POST_SUSPEND) + dcp_s2r_dark_finish(dcp); + + return NOTIFY_DONE; +} + +static void dcp_s2r_pm_unregister(void *data) +{ + struct apple_dcp *dcp = data; + + unregister_pm_notifier(&dcp->s2r_pm_nb); +} + +static int dcp_s2r_pm_register(struct apple_dcp *dcp) +{ + int ret; + + if (!dcp_s2r_ext_capable(dcp)) + return 0; + dcp->s2r_pm_nb.notifier_call = dcp_s2r_pm_notify; + ret = register_pm_notifier(&dcp->s2r_pm_nb); + if (ret) + return ret; + + return devm_add_action_or_reset(dcp->dev, dcp_s2r_pm_unregister, dcp); +} + +/* + * Its own suspend failed with the CPU stopped, and the PM core won't call .resume: start it + * again as a refused sleep does, or the display stays dark until a later sleep gets through. + */ +static void dcp_s2r_restart_failed_stop(struct apple_dcp *dcp) +{ + dev_warn(dcp->dev, "stop failed with the CPU stopped: starting the firmware again\n"); + dcp_s2r_restart(dcp, false); + dcp->s2r_stopped = false; +} + +/* + * DPMS on an internal panel with hw.s2r as macOS does a display sleep: the backlight to 0, + * then the firmware stopped and the display domains off, and the reverse on wake. Not + * during a system sleep: that path stops the firmware itself. + */ +static bool dcp_dpms_power_cycles(struct apple_dcp *dcp) +{ + return dcp_s2r_capable(dcp) && pm_suspend_target_state == PM_SUSPEND_ON; +} + +static void dcp_dpms_power_off(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (!dcp_dpms_power_cycles(dcp) || READ_ONCE(dcp->s2r_fw_down)) + return; + + ret = dcp_s2r_suspend(dcp); + if (ret && !READ_ONCE(dcp->s2r_fw_down)) { + dev_warn(dcp->dev, "DPMS: firmware stop refused, backlight only: %d\n", ret); + return; + } + dcp->s2r_stopped = false; + if (!ret) + ret = dcp_s2r_domains_off(dcp); + dcp_s2r_put_darts(dcp); + dcp->dpms_powered_off = true; + dev_dbg(dcp->dev, "DPMS: display powered off in %lld ms: %d\n", + ktime_ms_delta(ktime_get(), start), ret); +} + +/* After a refused idle the firmware still runs with the display off: light it again there. */ +static void dcp_s2r_light_running(struct apple_dcp *dcp) +{ + int ret; + + dcp_s2r_find_darts(dcp); + reinit_completion(&dcp->s2r_dart_done); + WRITE_ONCE(dcp->s2r_active, true); + ret = iomfb_s2r_power_h17p(dcp, true, true, true); + if (!ret && !wait_for_completion_timeout(&dcp->s2r_dart_done, + msecs_to_jiffies(DCP_S2R_D572_TIMEOUT_MS))) + dev_warn(dcp->dev, "no D572 after set_power_state(1), scanout DART left off\n"); + WRITE_ONCE(dcp->s2r_active, false); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "display not back on the running firmware: %d\n", ret); + return; + } + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "display back on the running firmware\n"); +} + +static void dcp_dpms_power_on(struct apple_dcp *dcp) +{ + ktime_t start = ktime_get(); + int ret; + + if (dcp->s2r_refused_off) { + dcp->s2r_refused_off = false; + dcp_s2r_light_running(dcp); + return; + } + if (!dcp->dpms_powered_off) + return; + dcp->dpms_powered_off = false; + if (!READ_ONCE(dcp->s2r_fw_down)) + return; + + dcp_s2r_find_darts(dcp); + ret = dcp_s2r_display_on(dcp); + dcp_s2r_put_darts(dcp); + if (ret) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "DPMS: display power-on failed, it stays off: %d\n", ret); + return; + } + WRITE_ONCE(dcp->s2r_fw_down, false); + iomfb_pause_presentation(dcp, false); + dev_dbg(dcp->dev, "DPMS: display powered on in %lld ms\n", + ktime_ms_delta(ktime_get(), start)); +} + +bool dcp_s2r_firmware_stopped(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return READ_ONCE(dcp->s2r_fw_down); +} + +static int dcp_platform_suspend(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + /* Its power sequences may crash the DCP and blank the panel. */ + if (dcp_takeover_was_aborted(dcp)) + return -EBUSY; + + if (pm_suspend_via_firmware() && dcp_s2r_capable(dcp)) + return dcp_s2r_suspend(dcp); + if (pm_suspend_via_firmware() && dcp_s2r_ext_capable(dcp)) + return dcp_s2r_ext_suspend(dcp); + + /* + * T8132: setPowerState powers the display down; the coprocessor keeps + * running and its power domains stay on, as below, unless the internal + * display is blanked with appledrm.power_gate_ms set: then the + * coprocessor and its domains are powered down too (iomfb_m4_suspend()). + */ + if (iomfb_is_m4(dcp)) { + int ret = iomfb_m4_suspend(dcp); + + if (!ret) + device_set_wakeup_path(dev); + return ret; + } + if (READ_ONCE(dcp->ext_dead)) + return 0; + iomfb_v14_7_backlight_suspend(dcp); + + /* Serialize PM against explicit bring-up, including queued startup work. + * Before a startup attempt the device remains suspendable. After any + * attempt we cannot prove DMA/firmware quiescence, even on failure. + */ + if (dcp->external) { + const char *why; + + mutex_lock(&dcp->hpd_mutex); + why = dcp_external_sleep_vetoed(dcp); + if (why) { + mutex_unlock(&dcp->hpd_mutex); + if (dcp->external_native) + dev_warn(dev, "suspend refused: %s\n", why); + else + dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); + return -EBUSY; + } + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); + } + + if (dcp_is_external_h17p(dcp)) { + int ret; + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + device_set_wakeup_path(dev); + return 0; + } + + dcp_drain_for_sleep(dcp); + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + if (dcp->external_native) + disable_work_sync(&dcp->external_audio_work); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->hdmi_hpd_irq) { + disable_irq(dcp->hdmi_hpd_irq); + if (!dcp->active_typec_route) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); + /* Last: the steps above still talk to the coprocessor. */ + dcp_hold_for_sleep(dcp); + + return 0; +} + +static int dcp_platform_resume(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp_takeover_was_aborted(dcp)) + return 0; + + if (dcp->s2r_stopped) + return dcp_s2r_resume(dcp); if (iomfb_is_m4(dcp)) return iomfb_m4_resume(dcp); @@ -7097,6 +8403,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .firmware_display_windows = 5, .firmware_scratch = 0x20000, .firmware_request = 0x68000, + .s2r = true, }; /* Separate from the established internal-panel H17P profile. */ @@ -7110,6 +8417,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { /* Native PMGR BWR agent 2 and boost agent 3 for DISPEXT0. */ .firmware_scratch = 0x28000, .firmware_request = 0x70000, + .s2r_ext = true, }; /* diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index dd6665c8c283f..e1c1c7406f117 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -135,6 +135,7 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp); int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); +bool dcp_s2r_firmware_stopped(struct platform_device *pdev); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e9826e23b07c0..69b5a17a5e5de 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -365,7 +365,7 @@ static bool iomfb_uses_queue(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } -static bool iomfb_channels_idle(struct apple_dcp *dcp) +bool iomfb_channels_idle(struct apple_dcp *dcp) { return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && @@ -673,6 +673,38 @@ void iomfb_queue_init(struct apple_dcp *dcp) atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } +/* + * For firmware stopped cold and started again: the calls and the transaction the old + * one never answered went with it, and would hold the channels and the queue forever. + */ +void iomfb_reset_transport(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction; + + if (!iomfb_uses_queue(dcp)) + return; + + cancel_delayed_work_sync(&dcp->iomfb.timeout); + mutex_lock(&dcp->iomfb.lock); + transaction = dcp->iomfb.active; + dcp->iomfb.active = NULL; + if (transaction) { + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; + transaction->release(transaction); + } + iomfb_discard_pending(dcp); + dcp->ch_cmd.depth = 0; + dcp->ch_oobcmd.depth = 0; + dcp->ch_cb.depth = 0; + dcp->ch_oobcb.depth = 0; + dcp->ch_async.depth = 0; + dcp->ch_oobasync.depth = 0; + dcp->present_state_h17p.pending = false; + dcp->present_state_h17p.accepted = false; + mutex_unlock(&dcp->iomfb.lock); +} + void iomfb_queue_stop(struct apple_dcp *dcp) { struct iomfb_callback *item; @@ -3152,6 +3184,10 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct platform_device *pdev = apple_crtc->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + /* Can't be lit: a resume then restores its CRTC disabled, not failing every output. */ + if (READ_ONCE(dcp->crashed)) + return false; + if (iomfb_is_m4(dcp)) { struct drm_display_mode fixed; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 1021825209fb3..958b3b889ed10 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -196,6 +196,7 @@ enum dcpep_method { dcpep_pipe_query_478, dcpep_pipe_query_474, dcpep_pipe_set_476, + dcpep_unnamed_500, iomfbep_set_brightness_correction, dcpep_num_methods }; @@ -216,6 +217,7 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_channels_idle(struct apple_dcp *dcp); bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, u32 out_len); /* Desktop 26.6: does the record hold what the tag's handler reads and writes? */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 1e24d0e617a79..eb3e7e0e84c05 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -241,6 +241,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A500", dcpep_unnamed_500), /* (0x4, 0) */ IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ IOMFB_METHOD("A428", iomfbep_set_brightness_correction), /* (0x4, 0x4) 0x10000 */ IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ @@ -312,6 +313,21 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +/* + * D572: the firmware controls the display DART's power stage with this. The + * argument is 0 when it's done with the display and 1 when it wants it back. + */ +static bool trampoline_dart_power_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_packet_header *header = in - sizeof(*header); + + if (header->in_len >= sizeof(u32)) + dcp_s2r_dart_power(dcp, get_unaligned_le32(in)); + + return trampoline_zero(dcp, tag, out, in); +} + static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -691,7 +707,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [563] = trampoline_true, [565] = trampoline_true, [567] = trampoline_true, - [572] = trampoline_zero, /* powerUpDART */ + [572] = trampoline_dart_power_h17p, /* powerUpDART */ [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b400dc07cf168..4776b5ad31034 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -520,6 +520,7 @@ DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_IN(dcp_unnamed_500, dcpep_unnamed_500, u32); #endif DCP_THUNK_INOUT(dcp_set_brightness_correction, iomfbep_set_brightness_correction, u32, u32); @@ -2467,6 +2468,121 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } } +/* V27 shares the h17p template prefix, but does not own its sleep RPCs. */ +#if DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) +/* + * set_power_state around a firmware-driven suspend. What macOS sends before it in + * either direction: A414(0), A478 twice, A476(1), A478, A412. All are named in + * this firmware's table, and their arguments are measured from macOS. + * + * For display on, macOS also sends A439 at the same moment. Its name isn't known, + * it carries a macOS kernel pointer and its reply wasn't captured, so it isn't + * sent here. + */ +enum dcp_s2r_step { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A412, DCP_S2R_A473, +}; + +static const u8 dcp_s2r_seq[] = { + DCP_S2R_A414, DCP_S2R_A478, DCP_S2R_A478, DCP_S2R_A476, DCP_S2R_A478, + DCP_S2R_A412, DCP_S2R_A473, +}; + +static void dcp_s2r_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned int step = dcp->s2r_step++; + + switch (dcp_s2r_seq[step]) { + case DCP_S2R_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A478: + dcp_pipe_query_478(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A476: { + u32 val = 1; + + dcp_pipe_set_476(dcp, false, &val, dcp_s2r_run, cookie); + break; + } + case DCP_S2R_A412: + dcp_is_main_display(dcp, false, dcp_s2r_run, cookie); + break; + case DCP_S2R_A473: { + /* Measured: the state, then 00 00 00 00 00 aa aa aa. */ + struct DCP_FW_NAME(dcp_set_power_state_req) req = { + .unklong = dcp->s2r_power_on, + .padding = { 0xaa, 0xaa, 0xaa }, + }; + + dcp_set_power_state(dcp, dcp->s2r_power_oob, &req, + dcp->s2r_power_on ? dcp_on_final : complete_set_powerstate, + cookie); + break; + } + } +} + +static void dcp_s2r_500_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); +} + +/* Unnamed. macOS sends it after D572 on every display off, with these bytes. */ +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + u32 val = 0xaaaa0000; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_unnamed_500(dcp, false, &val, dcp_s2r_500_done, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} + +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob) +{ + struct dcp_wait_cookie *cookie; + unsigned long left; + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->s2r_power_on = on; + dcp->s2r_power_oob = oob; + dcp->s2r_step = named_calls ? 0 : ARRAY_SIZE(dcp_s2r_seq) - 1; + dcp_s2r_run(dcp, NULL, cookie); + + left = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + kref_put(&cookie->refcount, release_wait_cookie); + + return left ? 0 : -ETIMEDOUT; +} +#endif + static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { /* H17P does not confirm that scanout has stopped. */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 3bd2b0d2c214b..93e8895e9fb48 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -389,5 +389,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); +#if DCP_FW_H17(DCP_FW_VER) +int DCP_FW_NAME(iomfb_s2r_500)(struct apple_dcp *dcp); +int DCP_FW_NAME(iomfb_s2r_power)(struct apple_dcp *dcp, bool on, bool named_calls, bool oob); +#endif #endif diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index e01795b57364c..f8802d2067284 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 2b8bdac35f43d..fcdc7430b9e95 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -247,12 +247,80 @@ asahi_of_device_table!( #[cfg(CONFIG_DRM_ASAHI_G15)] asahi_of_device_table!(); +struct SleepOps(bindings::dev_pm_ops); + +// SAFETY: Immutable: the callbacks are plain functions. +unsafe impl Sync for SleepOps {} + +/// # Safety +/// +/// `dev` must be a device this driver is bound to, as for a PM core callback. +unsafe fn sleep_data<'a>(dev: *mut bindings::device) -> &'a AsahiPlatformDriverData<'a> { + // SAFETY: Probe stored a `Pin>` here, and the PM core keeps the + // device bound for the duration of a callback. + unsafe { &*bindings::dev_get_drvdata(dev).cast() } +} + +/// Runs before any device suspends, so the GPU goes first and work still in flight can finish. +unsafe extern "C" fn sleep_prepare(dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: Called by the PM core. + let data = unsafe { sleep_data(dev) }; + match data.g17.as_ref() { + Some(gpu) => kernel::error::from_result(|| gpu.sleep_prepare().map(|_| 0)), + None => EBUSY.to_errno(), + } +} + +/// Runs after every device resumed. Does nothing unless [`sleep_prepare`] stopped the firmware. +unsafe extern "C" fn sleep_complete(dev: *mut bindings::device) { + // SAFETY: Called by the PM core. + if let Some(gpu) = unsafe { sleep_data(dev) }.g17.as_ref() { + gpu.sleep_complete(); + } +} + +static SLEEP_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_prepare), + complete: Some(sleep_complete), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// For a driver with no firmware restart: a sleep that powers the SoC down would lose its firmware. +unsafe extern "C" fn sleep_refuse(_dev: *mut bindings::device) -> core::ffi::c_int { + // SAFETY: Reads a global flag word. + if unsafe { bindings::pm_suspend_via_firmware() } { + EBUSY.to_errno() + } else { + 0 + } +} + +static SLEEP_REFUSE_OPS: SleepOps = SleepOps(bindings::dev_pm_ops { + prepare: Some(sleep_refuse), + // SAFETY: All-zero is the absence of every other callback. + ..unsafe { core::mem::zeroed() } +}); + +/// The callbacks of a driver that refuses a sleep that powers the SoC down. +pub(crate) fn sleep_refuse_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_REFUSE_OPS.0) +} + /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { type IdInfo = ProbeConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&SLEEP_OPS.0) + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { // SAFETY: Takes no arguments, process context. Clears the T6001 AGX request in the PMP report if the pmp_report.agx_vote // parameter set it; a no-op everywhere else. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 861668429197d..4cb221fb0a5df 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -178,6 +178,10 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(driver) = &this._driver { driver.close_binding(); diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 38fd612d240da..41fdad4e4cdaa 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -145,6 +145,10 @@ impl platform::Driver for Driver { type Data<'bound> = Option>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static kernel::bindings::dev_pm_ops> { + crate::driver::sleep_refuse_ops() + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4e68d0bd74a47..89f8ebd85e666 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -61,6 +61,7 @@ mod recovery; mod replay; mod reports; mod runtime; +mod sleep; pub(crate) mod stats; pub(crate) mod status; mod teardown; @@ -235,6 +236,9 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, + /// The firmware's warm restart across a system sleep is known for this SoC. Otherwise the + /// driver refuses a sleep that powers the SoC down. + pub(crate) sleep_restart: bool, /// The SoC's values of region C; `None` where the firmware graph has its own (M5). pub(crate) region_c: Option<&'static fw::initdata::RegionCConfig>, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, @@ -577,7 +581,11 @@ impl Coprocessor { ) -> Result> { for attempt in 0..RECEIVER_ATTEMPTS { match rtkit::RtKit::::new(dev.as_ref().as_ref(), None, index, state.clone()) { - Ok(receiver) => return Ok(receiver), + Ok(receiver) => { + // Its OSLOG counters stay unanswered until a soak shows echoing them is safe. + receiver.set_oslog_echo(false); + return Ok(receiver); + } Err(error) if error == EINVAL || error == ETIMEDOUT => { if attempt + 1 < RECEIVER_ATTEMPTS { fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); @@ -619,6 +627,15 @@ impl Coprocessor { self.rtkit.send(EP_DOORBELL, msg) } + /// Starts the CPU of firmware that lost power with its state intact, and waits for RTKit to + /// come up. No endpoint starts and no initialization: it already has both. + fn restart(&mut self) -> Result { + self.rtkit.with(|rtkit| rtkit.warm_restart())?; + self.cpu.start()?; + self.running = true; + self.rtkit.with(|rtkit| rtkit.boot()) + } + /// Stops the CPU. The firmware keeps its state, but no longer runs. fn stop(&mut self) { if self.running { @@ -626,6 +643,14 @@ impl Coprocessor { self.running = false; } } + + /// The books for a CPU that the sleep's commit stopped, with no register access. + fn stopped_at_commit(&mut self) { + if self.running { + self.cpu.stopped_elsewhere(); + self.running = false; + } + } } impl Drop for Coprocessor { @@ -658,6 +683,44 @@ impl Bridge { } } + /// The bridge across a power-off of the GPU's domain. Nothing on a target that runs without + /// it. Teardown still clears it afterwards. + fn set(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + return self.regs.enable_g17_bridge(); + } + self.regs.disable_g17_bridge(); + Ok(()) + } + + /// Core gating across a power-off of the GPU's domain. Nothing on a target that runs without + /// the bridge. + fn set_gating(&self, enable: bool) -> Result { + if !self.enabled { + return Ok(()); + } + if enable { + self.regs.enable_g17_gating() + } else { + self.regs.hold_g17_gating() + } + } + + /// Both bridge words and the gating word, on a target that has them. + fn words(&self) -> Option<[u32; 3]> { + if !self.enabled { + return None; + } + self.regs.g17_fabric_words().ok() + } + + fn forget(&mut self) { + self.enabled = false; + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -746,7 +809,10 @@ impl Firmware { /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { use recovery::Memory; - if self.recovery.pending() || self.init.recovery_state()? != 0 { + if self.primary.state.shared.sleeping() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + { return Err(EAGAIN); } let target = channel::Rings::publish_control(&self.init, record)?; @@ -896,7 +962,7 @@ impl Firmware { /// Observe a telemetry epoch boundary without changing GPU power policy. fn observe_statistics_power(&mut self) { - if self.rings.statistics_suspended() { + if self.rings.statistics_suspended() || self.primary.state.shared.sleeping() { return; } let roles = self.roles(); @@ -1073,6 +1139,14 @@ struct Shared { /// The M5 shared-queue resource policy (frozen device topology). render_policy: Option, crashed: AtomicBool, + /// The firmware is stopped, or about to be: nothing may read or publish to it. + sleeping: AtomicBool, + /// A sleep holds preparations closed, so a recovery settling meanwhile doesn't reopen them. + sleep_gate: AtomicBool, + /// The power state word as it was before the firmware stopped. + sleep_ps: AtomicU32, + /// The stop the sleep's commit makes, from `.prepare` to `.complete`; null otherwise. + sleep_commit: core::sync::atomic::AtomicPtr, feed: Arc, #[pin] feed_work: Work, @@ -1189,6 +1263,9 @@ impl_has_work! { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } if let Err(error) = this.grow_render() { if this.state.lock().is_some() { dev_err!( @@ -1212,6 +1289,9 @@ impl WorkItem for Shared { impl WorkItem for Shared { type Pointer = Arc; fn run(this: Arc) { + if this.sleeping() { + return; + } let result = { let mut state = this.state.lock(); match (*state).as_deref_mut() { @@ -1589,6 +1669,10 @@ impl Gpu { firmware_test: !submits, render_policy, crashed: AtomicBool::new(false), + sleeping: AtomicBool::new(false), + sleep_gate: AtomicBool::new(false), + sleep_ps: AtomicU32::new(0), + sleep_commit: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index be27fedaca625..f9bd31f13bc7d 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -541,6 +541,11 @@ impl Port { self.demand.set_active(full); } + /// The next statistics drain and reader discard what the decoder holds. + pub(super) fn statistics_epoch_bump(&self) { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } + pub(super) fn statistics_epoch(&self) -> u64 { self.statistics_epoch.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index ca504ed87db91..9cee6fdbe7aab 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2663,6 +2663,10 @@ impl Pair { } } } + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } + /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. /// The steady state in which a command may publish without the device diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 83a4bcb2f3022..4b8a2d2f64ee8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -292,6 +292,12 @@ impl ParkedPair { } } +impl ParkedPair { + pub(crate) fn forget_registration(&mut self) { + self.registered = None; + } +} + impl Prepared { /// Called after the registry revalidates its preparation epoch and parked pair identity. pub(crate) fn finish_install( diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index a3bafd969981d..f484baa36f161 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -109,7 +109,7 @@ fn write_power( firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>, ) -> kernel::fmt::Result { - if firmware.rings.statistics_suspended() { + if firmware.rings.statistics_suspended() || firmware.primary.state.shared.sleeping() { return writeln!(f, "power_management=firmware suspended=1"); } let core_power = firmware.bridge.regs.g17_core_power(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 5114465145081..c17680448bd71 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -345,6 +345,59 @@ impl Firmware { } } + /// A step of the event pass is waiting on the firmware with no event to wake it. + fn runtime_polling(&self, now: u64) -> bool { + self.render_control_backpressured() + || self.init.outer_waiting() + || self.queues.replay_outer_wait.is_some() + || self.queues.replay_consumer_wait.is_some() + || self.queues.render.replay_polling() + || self.queues.teardown.polling(now) + || self.compute_exit_polling(now) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()) + } + + /// Nothing is published that hasn't retired, nothing is waiting on the firmware, its rings + /// are empty, and the idle check has taken the effort and power holds down. + pub(in crate::g17) fn sleep_quiet(&self, now: u64) -> Result { + use crate::g17::recovery::Memory; + let compute = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + if compute + || self.queues.render_in_flight() + || !self.queues.render_settled() + || self.recovery.pending() + || self.init.recovery_state()? != 0 + || self.runtime_polling(now) + || self.hold.active() + || self.registration_power_release_pending + || self.effort.applied() != 0 + { + return Ok(false); + } + let [consumer, unknown, producer] = self.init.control_counters()?; + if consumer != producer || unknown != producer { + return Ok(false); + } + for &role in self.roles() { + if channel::Events::pending(&self.init, role)? { + return Ok(false); + } + } + Ok(true) + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -421,6 +474,10 @@ impl crate::g17::Shared { /// callbacks are finished. Shutdown can detach Firmware during an off-lock /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { + // The way back queues a pass, which also handles a crash noticed meanwhile. + if self.sleeping() { + return; + } if self.crashed.load(core::sync::atomic::Ordering::Acquire) { self.stop_runtime(); return; @@ -490,7 +547,9 @@ impl crate::g17::Shared { } if recover { firmware.service_runtime_recovery(deferred)?; - if !firmware.recovery.pending() { + if !firmware.recovery.pending() + && !self.sleep_gate.load(core::sync::atomic::Ordering::Acquire) + { if !self.preparations.reopen() { return Ok(()); } @@ -569,21 +628,7 @@ impl crate::g17::Shared { // The firmware may consume M5 work-ring slots without a completion interrupt: while // a preparation waits for unclaimed room, every pass polls the consumers. let outer = firmware.init.poll_outer_progress()?; - let backpressured = firmware.render_control_backpressured(); - let polling = backpressured - || firmware.init.outer_waiting() - || firmware.queues.replay_outer_wait.is_some() - || firmware.queues.replay_consumer_wait.is_some() - || firmware.queues.render.replay_polling() - || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now) - || firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); + let polling = firmware.runtime_polling(now); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 4b62d96d7cd3c..147f016caab3c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -1110,6 +1110,28 @@ impl Registry { } Ok(()) } + /// No graph is being built or grown. + pub(in crate::g17) fn render_settled(&self) -> bool { + self.render + .entries + .iter() + .flatten() + .all(|entry| entry.preparing.is_none() && !entry.growing) + } + + /// The firmware restarted with no power on the cores: nothing it was told about a pair through + /// their registers can be relied on, so the next command installs it again. + pub(in crate::g17) fn forget_render_registrations(&mut self) { + for entry in self.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + pair.forget_registration(); + } + if let Some(parked) = entry.parked.as_mut() { + parked.forget_registration(); + } + } + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c new file mode 100644 index 0000000000000..6207c99613172 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The GPU's power state word across a system sleep. The domain is always on as far + * as genpd knows, so nothing else lowers it. The word and its parent come from the + * device tree. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sleep.h" + +#define PS_AUTO_ENABLE BIT(28) +#define PS_AUTO GENMASK(27, 24) +#define PS_BUSY BIT(11) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTIVE 0xf +#define PS_POLL_US 100 +#define PS_TIMEOUT_US 50000 + +/* In the "sgx" range, as regs.rs has them. */ +#define SGX_GATING 0xd06030 +#define SGX_GATING_HOLD BIT(2) +#define SGX_BRIDGE 0x1000104 +#define SGX_BRIDGE_WORDS 2 +#define SGX_BRIDGE_ENABLE BIT(0) + +#define ASC_CPU_STATUS 0x4 + +/* The PMP's request for the GPU, where the device tree names one. */ +int asahi_g17_pmp_request(struct device *dev, bool enable) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (!np) + return 0; + ret = apple_pmp_report_entry_request(np, enable); + of_node_put(np); + + return ret; +} + +static int asahi_g17_ps(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -EINVAL; + ret = of_property_read_u32(np, "reg", offset); + if (!ret) { + *regmap = np->parent ? syscon_node_to_regmap(np->parent) : ERR_PTR(-EINVAL); + ret = PTR_ERR_OR_ZERO(*regmap); + } + of_node_put(np); + + return ret; +} + +/* Automatic power management off. Returns the word as it was in @saved. */ +int asahi_g17_power_auto_off(struct device *dev, u32 *saved) +{ + struct regmap *regmap; + u32 offset; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, saved); + if (ret) + return ret; + + return regmap_write(regmap, offset, *saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS)); +} + +/* The target off, then wait for the domain to finish going off. */ +int asahi_g17_power_lower(struct device *dev) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + regmap_write(regmap, offset, + (val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS); + + ret = regmap_read_poll_timeout(regmap, offset, cur, + !FIELD_GET(PS_ACTUAL, cur) && !(cur & PS_BUSY), + PS_POLL_US, PS_TIMEOUT_US); + dev_dbg(dev, "GPU domain %#x, now %#x\n", val, cur); + + return ret; +} + +static u32 asahi_g17_ps_on(u32 val, u32 saved) +{ + val &= ~(PS_AUTO_ENABLE | PS_AUTO | PS_BUSY | PS_FLAGS | PS_ACTUAL | PS_TARGET); + + return val | (saved & PS_AUTO) | FIELD_PREP(PS_TARGET, PS_ACTIVE); +} + +/* The target on, and wait for it. */ +int asahi_g17_power_raise(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val, cur; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + dev_dbg(dev, "GPU domain before raising it: %#x\n", val); + + regmap_write(regmap, offset, asahi_g17_ps_on(val, saved)); + + return regmap_read_poll_timeout(regmap, offset, cur, + FIELD_GET(PS_ACTUAL, cur) == PS_ACTIVE, + PS_POLL_US, PS_TIMEOUT_US); +} + +/* Automatic power management as it was. */ +int asahi_g17_power_auto_on(struct device *dev, u32 saved) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = asahi_g17_ps(dev, ®map, &offset); + if (ret) + return ret; + ret = regmap_read(regmap, offset, &val); + if (ret) + return ret; + + return regmap_write(regmap, offset, asahi_g17_ps_on(val, saved) | (saved & PS_AUTO_ENABLE)); +} + +/* + * What the sleep's commit needs to stop the GPU with interrupts off, taken in .prepare, and + * what it saw, for the log after the sleep. + */ +struct asahi_g17_commit { + struct device *dev; + unsigned int count; + struct apple_rtkit *rtk[2]; + struct apple_mbox *mbox[2]; + void __iomem *cpu[2]; + void __iomem *gating; + void __iomem *bridge; + void __iomem *ps; + struct device_node *request; + + bool ran; + int idle[2]; + unsigned int idle_us[2]; + u32 status[2]; + u32 words[SGX_BRIDGE_WORDS + 1]; + int request_ret; + u32 saved; + u32 lowered; + int lower_ret; +}; + +/* + * down() at the commit, every refusal behind it: both instances idled by polling, mailboxes + * and run bits off, the gating hold, the PMP request withdrawn while the PMP still runs, then + * the bridge and the domain. Told to idle, the firmware can't carry on, so the sleep goes on + * whatever it answers. + */ +static int asahi_g17_commit_stop(void *data) +{ + struct asahi_g17_commit *c = data; + unsigned int i; + u32 val; + + c->ran = true; + for (i = 0; i < c->count; i++) + c->idle[i] = apple_rtkit_sleep_idle_polled(c->rtk[i], c->cpu[i], &c->idle_us[i]); + for (i = 0; i < c->count; i++) { + c->status[i] = readl(c->cpu[i] + ASC_CPU_STATUS); + apple_rtkit_mailbox_recv_off(c->rtk[i]); + apple_mbox_ascwrap_v6_stop_cpu_atomic(c->mbox[i]); + if (!i && c->bridge) + writel(readl(c->gating) | SGX_GATING_HOLD, c->gating); + } + c->request_ret = c->request ? apple_pmp_report_entry_withdraw_polled(c->request) : 0; + + c->saved = readl(c->ps); + writel(c->saved & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS), c->ps); + if (c->bridge) { + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + c->words[i] = readl(c->bridge + 4 * i); + c->words[i] = readl(c->gating); + for (i = 0; i < SGX_BRIDGE_WORDS; i++) + writel(c->words[i] & ~SGX_BRIDGE_ENABLE, c->bridge + 4 * i); + } + val = readl(c->ps); + writel((val & ~(PS_AUTO_ENABLE | PS_BUSY | PS_FLAGS | PS_TARGET)) | PS_FLAGS, c->ps); + c->lower_ret = readl_poll_timeout_atomic(c->ps, c->lowered, + !FIELD_GET(PS_ACTUAL, c->lowered) && + !(c->lowered & PS_BUSY), + 10, PS_TIMEOUT_US); + + return (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) ? 1 : 0; +} + +static void __iomem *asahi_g17_map(struct device_node *np, const char *name, int index, + u32 offset, u32 size) +{ + struct resource res; + + if (name) + index = of_property_match_string(np, "reg-names", name); + if (index < 0 || of_address_to_resource(np, index, &res) || + offset + size > resource_size(&res)) + return NULL; + + return ioremap(res.start + offset, size); +} + +static void asahi_g17_commit_free(struct asahi_g17_commit *c) +{ + if (c->request) { + apple_pmp_report_entry_commit_owned(c->request, false); + of_node_put(c->request); + } + if (c->ps) + iounmap(c->ps); + if (c->bridge) + iounmap(c->bridge); + if (c->gating) + iounmap(c->gating); + kfree(c); +} + +/* + * Leaves the GPU's whole stop to the sleep's commit: maps what the stop touches, keeps the + * PMP request for it, and registers it with the sleep driver, ready. Nothing is stopped + * here. @secondary_rtk is NULL on a single instance, and @bridge false on a target that runs + * without the bridge. + */ +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge) +{ + struct asahi_g17_commit *c; + struct device_node *np; + unsigned int i; + u32 offset; + int ret; + + c = kzalloc_obj(*c); + if (!c) + return ERR_PTR(-ENOMEM); + c->dev = dev; + c->rtk[0] = primary_rtk; + c->mbox[0] = primary_mbox; + c->rtk[1] = secondary_rtk; + c->mbox[1] = secondary_mbox; + c->count = secondary_rtk ? 2 : 1; + + ret = -EOPNOTSUPP; + for (i = 0; i < c->count; i++) { + c->cpu[i] = apple_mbox_ascwrap_v6_cpu_control(c->mbox[i]); + if (!c->rtk[i] || !c->cpu[i]) + goto err; + } + + ret = -ENODEV; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + goto err; + if (!of_property_read_u32(np, "reg", &offset) && np->parent) + c->ps = asahi_g17_map(np->parent, NULL, 0, offset, 4); + of_node_put(np); + c->gating = asahi_g17_map(dev->of_node, "sgx", 0, SGX_GATING, 4); + if (bridge) + c->bridge = asahi_g17_map(dev->of_node, "sgx", 0, SGX_BRIDGE, + 4 * SGX_BRIDGE_WORDS); + if (!c->ps || !c->gating || (bridge && !c->bridge)) + goto err; + + c->request = of_parse_phandle(dev->of_node, "apple,pmp-report-entry", 0); + if (c->request) + apple_pmp_report_entry_commit_owned(c->request, true); + + ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + if (ret) + goto err; + apple_sleep_coproc_set_ready(dev, true); + + return c; + +err: + asahi_g17_commit_free(c); + return ERR_PTR(ret); +} + +/* Whether only the sleep's commit may stop the GPU: then nothing else may. */ +bool asahi_g17_commit_required(void) +{ + return apple_sleep_commit_stops_coprocs(); +} + +/* + * After the sleep, refused or not: whether the commit stopped the GPU, and in @saved the + * domain's word from before, if it did. Frees @c. + */ +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved) +{ + bool stopped = apple_sleep_coproc_stopped(c->dev) || c->ran; + + apple_sleep_coproc_unregister(c->dev); + if (c->ran) { + if (c->idle[0] || c->idle[1] || c->request_ret || c->lower_ret) + dev_warn(c->dev, "GPU stop at the commit: idle %d %d, PMP request %d, domain %#x (%d)\n", + c->idle[0], c->idle[1], c->request_ret, c->lowered, c->lower_ret); + dev_dbg(c->dev, "GPU stopped at the commit: idle in %u %u us, status %#x %#x\n", + c->idle_us[0], c->idle_us[1], c->status[0], c->status[1]); + dev_dbg(c->dev, "GPU at the commit: bridge %#x %#x gating %#x, domain %#x now %#x\n", + c->words[0], c->words[1], c->words[2], c->saved, c->lowered); + *saved = c->saved; + } + asahi_g17_commit_free(c); + + return stopped; +} diff --git a/drivers/gpu/drm/asahi/g17/sleep.h b/drivers/gpu/drm/asahi/g17/sleep.h new file mode 100644 index 0000000000000..e4473c73e19d3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SLEEP_H +#define ASAHI_G17_SLEEP_H +#include + +struct apple_mbox; +struct apple_rtkit; +struct asahi_g17_commit; +struct device; + +int asahi_g17_pmp_request(struct device *dev, bool enable); +struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, + struct apple_rtkit *primary_rtk, + struct apple_mbox *primary_mbox, + struct apple_rtkit *secondary_rtk, + struct apple_mbox *secondary_mbox, bool bridge); +bool asahi_g17_commit_finish(struct asahi_g17_commit *c, u32 *saved); +bool asahi_g17_commit_required(void); +int asahi_g17_power_auto_off(struct device *dev, u32 *saved); +int asahi_g17_power_lower(struct device *dev); +int asahi_g17_power_raise(struct device *dev, u32 saved); +int asahi_g17_power_auto_on(struct device *dev, u32 saved); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sleep.rs b/drivers/gpu/drm/asahi/g17/sleep.rs new file mode 100644 index 0000000000000..fe5779ee5d5e1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sleep.rs @@ -0,0 +1,820 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! System sleep: the firmware goes idle, its domain goes off, and it restarts warm. +//! +//! The firmware isn't initialized again. It keeps its rings and state in DRAM, so a restart is +//! the RTKit handshake and the end of the power-on transition. Publication stays closed from +//! before the idle until after the restart, and nothing is in flight across it: work that doesn't +//! finish in time refuses the sleep. +//! +//! Order, on both instances, is what macOS does on the chip whose [`super::Config`] sets +//! `sleep_restart`. Other chips refuse. +//! +//! Only a real wake restarts the firmware. A sleep refused after the firmware stopped leaves it +//! stopped and publication closed until the next real sleep, with the domain on so that everything +//! behind it stays reachable. That sleep then only has the domain left to lower. +//! +//! Where the sleep driver takes it, `.prepare` only drains and checks for quiet, and the sleep's +//! commit makes the whole stop in C, with interrupts off and every refusal behind it. A sleep +//! refused before then finds the firmware never stopped. Elsewhere `.prepare` stops it. + +use super::{Firmware, Role, Shared}; +use core::sync::atomic::Ordering; +use kernel::{ + bindings, + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, + workqueue::{DelayedWork, Work}, +}; + +/// Time work in flight gets to finish before the sleep is refused. +const DRAIN_NS: u64 = 3_000_000_000; +const DRAIN_TICK_MS: i64 = 2; + +unsafe extern "C" { + fn asahi_g17_pmp_request(dev: *mut core::ffi::c_void, enable: bool) -> core::ffi::c_int; + fn asahi_g17_power_auto_off(dev: *mut core::ffi::c_void, saved: *mut u32) -> core::ffi::c_int; + fn asahi_g17_power_lower(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_g17_power_raise(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_power_auto_on(dev: *mut core::ffi::c_void, saved: u32) -> core::ffi::c_int; + fn asahi_g17_commit_prepare( + dev: *mut core::ffi::c_void, + primary_rtk: *mut bindings::apple_rtkit, + primary_mbox: *mut core::ffi::c_void, + secondary_rtk: *mut bindings::apple_rtkit, + secondary_mbox: *mut core::ffi::c_void, + bridge: bool, + ) -> *mut core::ffi::c_void; + fn asahi_g17_commit_finish(commit: *mut core::ffi::c_void, saved: *mut u32) -> bool; + fn asahi_g17_commit_required() -> bool; +} + +/// What `.complete` does after a sleep whose stop was left to the commit. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum AfterCommit { + /// Refused before the commit: the firmware never stopped, so only publication reopens. + Running, + /// Stopped at the commit, or its power lost anyway: as after [`down`]. + Stopped, +} + +fn after_commit(stopped: bool, real: bool) -> AfterCommit { + if stopped || real { + AfterCommit::Stopped + } else { + AfterCommit::Running + } +} + +/// What stopping and restarting the firmware touches, in the order [`down`] and [`up`] use it. +pub(super) trait Hardware { + fn has_secondary(&self) -> bool; + /// RTKit AP and IOP idle, each acknowledged. + fn idle(&mut self, role: Role) -> Result; + /// Back from idle with no restart, after a refusal. One that fails loses the device. + fn wake(&mut self, role: Role); + /// Mailbox off, then run bit off, for an instance that acknowledged idle. + fn stop(&mut self, role: Role); + /// The PMP's request for the GPU. It may use a device it's told is in use, so the request is + /// only ever set with the domain on. + fn request(&mut self, enable: bool); + /// Core gating as it is while the firmware runs, or held for a domain about to go off. + fn gating(&mut self, enable: bool) -> Result; + /// The GPU-to-fabric bridge. + fn bridge(&mut self, enable: bool) -> Result; + /// Automatic power management off, keeping the word for the way back. + fn auto_off(&mut self) -> Result; + /// The domain off, waiting until it has settled. + fn lower(&mut self) -> Result; + /// The domain on, waiting until it is. + fn raise(&mut self) -> Result; + /// Automatic power management as it was. + fn auto_on(&mut self) -> Result; + /// Run bit on, RTKit handshake to AP on. + fn restart(&mut self, role: Role) -> Result; + /// Ends the power-on transition, on every instance. + fn power_on_done(&mut self) -> Result; +} + +/// Stops the firmware and lowers the domain. An error leaves the firmware running, so the caller +/// refuses the sleep. Both instances idle before either stops, so that a refusal by the second +/// still has a way back. A domain that is slow to go off isn't an error here: the sleep driver +/// checks it at the commit. +pub(super) fn down(hw: &mut H) -> Result { + if let Err(error) = hw.idle(Role::Primary) { + hw.wake(Role::Primary); + return Err(error); + } + if hw.has_secondary() { + if let Err(error) = hw.idle(Role::Secondary) { + hw.wake(Role::Secondary); + hw.wake(Role::Primary); + return Err(error); + } + } + hw.stop(Role::Primary); + let _ = hw.gating(false); + if hw.has_secondary() { + hw.stop(Role::Secondary); + } + hw.request(false); + lower_domain(hw); + Ok(()) +} + +/// Lowers the domain, with the firmware stopped. +fn lower_domain(hw: &mut H) { + let _ = hw.auto_off(); + let _ = hw.bridge(false); + let _ = hw.lower(); +} + +/// Brings the domain back for firmware that stays stopped after a refused sleep. +fn hold(hw: &mut H) -> Result { + hw.raise()?; + hw.auto_on() +} + +/// Restarts the firmware. Any error loses the device: there is no other way back. +pub(super) fn up(hw: &mut H) -> Result { + hw.raise()?; + hw.bridge(true)?; + hw.auto_on()?; + hw.request(true); + hw.gating(true)?; + if hw.has_secondary() { + hw.restart(Role::Secondary)?; + } + hw.restart(Role::Primary)?; + hw.power_on_done() +} + +impl Hardware for Firmware { + fn has_secondary(&self) -> bool { + self.secondary.is_some() + } + + fn idle(&mut self, role: Role) -> Result { + self.coprocessor(role)?.rtkit.with(|rtkit| rtkit.idle()) + } + + fn wake(&mut self, role: Role) { + let result = match self.coprocessor(role) { + Ok(coprocessor) => coprocessor.rtkit.with(|rtkit| rtkit.wake()), + Err(error) => Err(error), + }; + // EINVAL: it never left the on state. + if result.is_err_and(|error| error != EINVAL) { + self.primary.state.shared.lose_device_quietly(); + } + } + + fn stop(&mut self, role: Role) { + if let Ok(coprocessor) = self.coprocessor(role) { + coprocessor.rtkit.with(|rtkit| rtkit.mailbox_off()); + coprocessor.stop(); + } + } + + fn request(&mut self, enable: bool) { + self.primary.state.shared.pmp_request(enable); + } + + fn gating(&mut self, enable: bool) -> Result { + self.bridge.set_gating(enable) + } + + fn bridge(&mut self, enable: bool) -> Result { + if let Some([first, second, gating]) = self.bridge.words() { + dev_dbg!( + self.primary.state.shared.dev.as_ref(), + "GPU before bridge {}: {:#x} {:#x} gating {:#x}\n", + if enable { "on" } else { "off" }, + first, + second, + gating + ); + } + self.bridge.set(enable) + } + + fn auto_off(&mut self) -> Result { + self.primary.state.shared.power_auto_off() + } + + fn lower(&mut self) -> Result { + self.primary.state.shared.power_lower() + } + + fn raise(&mut self) -> Result { + let result = self.primary.state.shared.power_raise(); + if result.is_err() { + // The registers aren't reachable, so teardown can't clear the bridge. + self.bridge.forget(); + } + result + } + + fn auto_on(&mut self) -> Result { + self.primary.state.shared.power_auto_on() + } + + fn restart(&mut self, role: Role) -> Result { + let shared = self.primary.state.shared.clone(); + let coprocessor = self.coprocessor(role)?; + let result = coprocessor.restart(); + if let (Err(_), Ok((control, status))) = (&result, coprocessor.cpu.state()) { + dev_err!( + shared.dev.as_ref(), + "GPU firmware {:?}: control {:#x} status {:#x}\n", + role, + control, + status + ); + } + result + } + + fn power_on_done(&mut self) -> Result { + if !self.variant.power_on_done { + return Ok(()); + } + self.primary.notify(super::MSG_POWER_ON_DONE)?; + if let Some(secondary) = self.secondary.as_ref() { + secondary.notify(super::MSG_POWER_ON_DONE)?; + } + Ok(()) + } +} + +impl Firmware { + /// Hands the sleep's commit what its stop needs. Nothing stops here. + fn leave_for_commit(&mut self) -> Result<*mut core::ffi::c_void> { + let shared = self.primary.state.shared.clone(); + let primary_rtk = self.primary.rtkit.with(|rtkit| rtkit.as_raw()); + let primary_mbox = self.primary.cpu.as_raw().cast(); + let (secondary_rtk, secondary_mbox) = match self.secondary.as_ref() { + Some(secondary) => ( + secondary.rtkit.with(|rtkit| rtkit.as_raw()), + secondary.cpu.as_raw().cast(), + ), + None => (core::ptr::null_mut(), core::ptr::null_mut()), + }; + // SAFETY: `dev` is the live GPU device. The RTKit and mailbox pointers stay valid while + // the firmware exists, and `.complete` frees the result before anything drops them. + let commit = unsafe { + asahi_g17_commit_prepare( + shared.dev.as_raw().cast(), + primary_rtk, + primary_mbox, + secondary_rtk, + secondary_mbox, + self.bridge.enabled, + ) + }; + kernel::error::from_err_ptr(commit) + } + + /// The books for both instances, which the commit stopped. + fn stopped_at_commit(&mut self) { + if let Some(secondary) = self.secondary.as_mut() { + secondary.stopped_at_commit(); + } + self.primary.stopped_at_commit(); + } + + fn coprocessor(&mut self, role: Role) -> Result<&mut super::Coprocessor> { + match role { + Role::Primary => Ok(&mut self.primary), + Role::Secondary => self.secondary.as_mut().ok_or(ENODEV), + } + } +} + +impl Shared { + pub(super) fn sleeping(&self) -> bool { + self.sleeping.load(Ordering::Acquire) + } + + /// A PMP that doesn't answer in time only gets a warning: nothing better is left to do here. + fn pmp_request(&self, enable: bool) { + // SAFETY: `dev` is the live GPU device. + let result = kernel::error::to_result(unsafe { + asahi_g17_pmp_request(self.dev.as_raw().cast(), enable) + }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "PMP request for the GPU, {}: {:?}\n", enable, error); + } + } + + fn power_auto_off(&self) -> Result { + let mut saved = 0; + // SAFETY: `dev` is the live GPU device and `saved` is writable. + let result = kernel::error::to_result(unsafe { + asahi_g17_power_auto_off(self.dev.as_raw().cast(), &mut saved) + }); + self.sleep_ps.store(saved, Ordering::Relaxed); + result + } + + fn power_lower(&self) -> Result { + // SAFETY: `dev` is the live GPU device. + let result = + kernel::error::to_result(unsafe { asahi_g17_power_lower(self.dev.as_raw().cast()) }); + if let Err(error) = result { + dev_warn!(self.dev.as_ref(), "GPU domain slow to go off: {:?}\n", error); + } + result + } + + fn power_raise(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { asahi_g17_power_raise(self.dev.as_raw().cast(), saved) }) + } + + fn power_auto_on(&self) -> Result { + let saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `dev` is the live GPU device. + kernel::error::to_result(unsafe { + asahi_g17_power_auto_on(self.dev.as_raw().cast(), saved) + }) + } + + fn cancel_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: As in `disable_timer`: a pending callback owned one Arc that it can no longer + // reclaim. Future enqueues stay possible and run into the sleeping check. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, bindings::delayed_work, work); + bindings::cancel_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: See above. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// With the sleeping flag set, joins whatever may be reading firmware state. The growth work + /// isn't joined: it may be waiting on the closed gate, and can't publish while it is closed. + fn join_event_workers(self: &Arc) { + self.cancel_timer(&self.poll); + self.cancel_timer(&self.idle); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + } + + /// Waits for what's been published to retire. The event worker keeps running meanwhile, and + /// its own idle check takes the power and effort holds down. + fn wait_quiet(self: &Arc, deadline: u64) -> Result { + loop { + if self.crashed.load(Ordering::Acquire) { + return Err(ENODEV); + } + let now = super::now_ns(); + { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + // Nothing else retries this with no render publication coming. + if firmware.registration_power_release_pending { + let _ = firmware.release_registration_power(); + } + if firmware.sleep_quiet(now)? { + return Ok(()); + } + } + if now >= deadline { + return Err(EBUSY); + } + self.queue_events(); + fsleep(Delta::from_millis(DRAIN_TICK_MS)); + } + } + + /// Undoes everything short of a stopped firmware. A builder that never finishes would keep + /// publication closed for good, and that is a lost device. + fn sleep_cancel(self: &Arc) { + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + let drained = self + .preparations + .wait_drained_until(super::now_ns().saturating_add(DRAIN_NS)); + if (drained.is_err() || !self.preparations.reopen()) + && !self.crashed.load(Ordering::Acquire) + { + self.lose_device_quietly(); + } + self.queue_events(); + } + + /// Keeps the firmware stopped after a sleep that never reached the hardware. False if there's + /// no firmware to keep, or no domain to keep it in. + fn sleep_hold(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { + return false; + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return false; + }; + if hold(firmware).is_err() { + return false; + } + dev_warn!( + self.dev.as_ref(), + "sleep refused: GPU firmware stays stopped until a real sleep\n" + ); + true + } + + /// Leaves the stop to the commit. False, with the reason logged, if it can't be. Where only + /// the commit may stop it, that refuses the sleep instead: a refusal after an earlier stop + /// would leave it stopped. + fn sleep_leave_for_commit(&self, firmware: &mut Firmware) -> Result { + match firmware.leave_for_commit() { + Ok(commit) => { + self.sleep_commit.store(commit, Ordering::Release); + dev_dbg!(self.dev.as_ref(), "GPU firmware left running for the sleep's commit\n"); + Ok(true) + } + // SAFETY: Reads the sleep driver's state. + Err(error) if unsafe { asahi_g17_commit_required() } => { + dev_err!( + self.dev.as_ref(), + "no GPU stop at the commit ({:?}): refusing the sleep\n", + error + ); + Err(EBUSY) + } + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "no GPU stop at the commit ({:?}): stopping it now\n", + error + ); + Ok(false) + } + } + } + + /// Whether the commit stopped the firmware, if the stop was left to it. + fn sleep_commit_finish(&self) -> Option { + let commit = self.sleep_commit.swap(core::ptr::null_mut(), Ordering::AcqRel); + if commit.is_null() { + return None; + } + let mut saved = self.sleep_ps.load(Ordering::Relaxed); + // SAFETY: `commit` came from `asahi_g17_commit_prepare` and is finished only once. + let stopped = unsafe { asahi_g17_commit_finish(commit, &mut saved) }; + self.sleep_ps.store(saved, Ordering::Relaxed); + Some(stopped) + } + + /// After the commit stopped the firmware: the books it left, as [`down`] would have them. + fn sleep_settle_commit(&self) { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.stopped_at_commit(); + } + } + + /// Closes publication, waits for quiet, and leaves the stop to the sleep's commit where the + /// sleep driver takes it, or stops the firmware. A refusal changes nothing. + fn sleep_down(self: &Arc) -> Result { + if self.sleeping() { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + lower_domain(firmware); + } + return Ok(()); + } + let deadline = super::now_ns().saturating_add(DRAIN_NS); + self.sleep_gate.store(true, Ordering::Release); + self.preparations.close(); + let result = self.preparations.wait_drained_until(deadline).map_err(|_| EBUSY) + .and_then(|_| self.wait_quiet(deadline)) + .and_then(|_| { + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !firmware.sleep_quiet(super::now_ns())? { + return Err(EBUSY); + } + if self.sleep_leave_for_commit(firmware)? { + return Ok(()); + } + down(firmware) + }); + match result { + Ok(()) => Ok(()), + Err(error) if error == ENODEV || self.crashed.load(Ordering::Acquire) => { + // Nothing to save: let the worker finish detaching what's left, and then the + // domain only has to be off for the sleep. + self.sleeping.store(false, Ordering::Release); + self.queue_events(); + // SAFETY: An initialized, pinned work field of this Shared. + unsafe { bindings::flush_work(Work::raw_get(&self.work)) }; + self.sleeping.store(true, Ordering::Release); + self.join_event_workers(); + self.pmp_request(false); + let _ = self.power_auto_off(); + let _ = self.power_lower(); + Ok(()) + } + Err(error) => { + self.sleep_cancel(); + Err(error) + } + } + } + + /// Restarts the firmware and reopens publication. A failure loses the device. + fn sleep_up(self: &Arc) -> Result { + let started = super::now_ns(); + let result = { + let mut state = self.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if !self.crashed.load(Ordering::Acquire) => { + let result = up(firmware); + if result.is_ok() { + // The cores lost power, so the next command on each pair installs it again. + firmware.queues.forget_render_registrations(); + firmware.rings.invalidate_statistics(); + firmware.port.statistics_epoch_bump(); + } + result + } + _ => self + .power_raise() + .and_then(|_| self.power_auto_on()) + .inspect(|_| self.pmp_request(true)), + } + }; + self.sleeping.store(false, Ordering::Release); + self.sleep_gate.store(false, Ordering::Release); + match result { + Ok(()) => { + if !self.crashed.load(Ordering::Acquire) { + dev_dbg!( + self.dev.as_ref(), + "GPU firmware back in {} us\n", + super::now_ns().saturating_sub(started) / 1000 + ); + } + self.preparations.reopen(); + self.queue_events(); + Ok(()) + } + Err(error) => { + dev_err!(self.dev.as_ref(), "GPU firmware did not come back: {:?}\n", error); + self.lose_device_quietly(); + Err(error) + } + } + } +} + +impl super::Gpu { + /// Called before any device suspends. Only a sleep that powers the SoC down gets here. + pub(crate) fn sleep_prepare(&self) -> Result { + if !self.cfg.sleep_restart { + dev_warn!( + self.shared.dev.as_ref(), + "no firmware restart on this chip, refusing sleep\n" + ); + return Err(EBUSY); + } + let result = self.shared.sleep_down(); + if let Err(error) = result { + dev_warn!(self.shared.dev.as_ref(), "GPU refused sleep: {:?}\n", error); + } + result + } + + /// Called after every device resumed, whether or not the sleep reached the hardware. + pub(crate) fn sleep_complete(&self) { + if !self.shared.sleeping() { + return; + } + // SAFETY: Reads a global flag word. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if let Some(stopped) = self.shared.sleep_commit_finish() { + if after_commit(stopped, real) == AfterCommit::Running { + dev_info!( + self.shared.dev.as_ref(), + "sleep refused before the commit: GPU firmware never stopped\n" + ); + self.shared.sleep_cancel(); + return; + } + self.shared.sleep_settle_commit(); + } + if !real && self.shared.sleep_hold() { + return; + } + let _ = self.shared.sleep_up(); + } +} + +#[kernel::macros::kunit_tests(asahi_g17_sleep)] +mod tests { + use super::*; + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + enum Call { + Idle(Role), + Wake(Role), + Stop(Role), + Request(bool), + Gating(bool), + Bridge(bool), + AutoOff, + Lower, + Raise, + AutoOn, + Restart(Role), + Done, + } + + struct Fake { + secondary: bool, + fail: Option, + calls: [Option; 14], + count: usize, + } + + impl Fake { + fn new(secondary: bool, fail: Option) -> Self { + Self { secondary, fail, calls: [None; 14], count: 0 } + } + + fn note(&mut self, call: Call) -> Result { + self.calls[self.count] = Some(call); + self.count += 1; + if self.fail == Some(call) { Err(EIO) } else { Ok(()) } + } + + fn called(&self, expected: &[Call]) -> bool { + self.count == expected.len() + && expected.iter().zip(self.calls.iter()).all(|(a, b)| Some(*a) == *b) + } + } + + impl Hardware for Fake { + fn has_secondary(&self) -> bool { self.secondary } + fn idle(&mut self, role: Role) -> Result { self.note(Call::Idle(role)) } + fn wake(&mut self, role: Role) { let _ = self.note(Call::Wake(role)); } + fn stop(&mut self, role: Role) { let _ = self.note(Call::Stop(role)); } + fn request(&mut self, enable: bool) { let _ = self.note(Call::Request(enable)); } + fn gating(&mut self, enable: bool) -> Result { self.note(Call::Gating(enable)) } + fn bridge(&mut self, enable: bool) -> Result { self.note(Call::Bridge(enable)) } + fn auto_off(&mut self) -> Result { self.note(Call::AutoOff) } + fn lower(&mut self) -> Result { self.note(Call::Lower) } + fn raise(&mut self) -> Result { self.note(Call::Raise) } + fn auto_on(&mut self) -> Result { self.note(Call::AutoOn) } + fn restart(&mut self, role: Role) -> Result { self.note(Call::Restart(role)) } + fn power_on_done(&mut self) -> Result { self.note(Call::Done) } + } + + use Role::{Primary as P, Secondary as S}; + + #[test] + fn down_idles_the_primary_first_and_lowers_the_domain_last() { + let mut hw = Fake::new(true, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Idle(S), + Call::Stop(P), + Call::Gating(false), + Call::Stop(S), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + } + + #[test] + fn up_restarts_the_secondary_first_and_ends_the_transition_last() { + let mut hw = Fake::new(true, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_single_instance_never_names_a_secondary() { + let mut hw = Fake::new(false, None); + assert_eq!(down(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Idle(P), + Call::Stop(P), + Call::Gating(false), + Call::Request(false), + Call::AutoOff, + Call::Bridge(false), + Call::Lower, + ])); + let mut hw = Fake::new(false, None); + assert_eq!(up(&mut hw), Ok(())); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(P), + Call::Done, + ])); + } + + #[test] + fn a_refused_idle_touches_nothing_past_the_firmware() { + for call in [Call::Idle(P), Call::Idle(S)] { + let mut hw = Fake::new(true, Some(call)); + assert_eq!(down(&mut hw), Err(EIO)); + let past = |c: &Call| !matches!(c, Call::Idle(_) | Call::Wake(_)); + assert!(!hw.calls.iter().flatten().any(past)); + } + } + + #[test] + fn a_primary_that_refuses_idle_is_woken_and_nothing_stops() { + let mut hw = Fake::new(true, Some(Call::Idle(P))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Wake(P)])); + } + + #[test] + fn a_secondary_that_refuses_idle_wakes_both_and_stops_nothing() { + let mut hw = Fake::new(true, Some(Call::Idle(S))); + assert_eq!(down(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Idle(P), Call::Idle(S), Call::Wake(S), Call::Wake(P)])); + } + + #[test] + fn a_refused_sleep_only_brings_the_domain_back() { + let mut hw = Fake::new(true, None); + assert_eq!(hold(&mut hw), Ok(())); + assert!(hw.called(&[Call::Raise, Call::AutoOn])); + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(hold(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + } + + #[test] + fn the_sleep_after_a_refusal_only_lowers_the_domain() { + let mut hw = Fake::new(true, None); + lower_domain(&mut hw); + assert!(hw.called(&[Call::AutoOff, Call::Bridge(false), Call::Lower])); + } + + #[test] + fn a_slow_domain_does_not_fail_down() { + let mut hw = Fake::new(false, Some(Call::Lower)); + assert_eq!(down(&mut hw), Ok(())); + } + + #[test] + fn up_stops_at_the_first_failure() { + let mut hw = Fake::new(true, Some(Call::Restart(S))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[ + Call::Raise, + Call::Bridge(true), + Call::AutoOn, + Call::Request(true), + Call::Gating(true), + Call::Restart(S), + ])); + // No domain, so no bridge and no request either. + let mut hw = Fake::new(true, Some(Call::Raise)); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise])); + // No bridge, so the PMP isn't told about the GPU. + let mut hw = Fake::new(true, Some(Call::Bridge(true))); + assert_eq!(up(&mut hw), Err(EIO)); + assert!(hw.called(&[Call::Raise, Call::Bridge(true)])); + } + + #[test] + fn a_stop_left_to_the_commit_is_undone_only_if_it_never_ran() { + assert_eq!(after_commit(false, false), AfterCommit::Running); + assert_eq!(after_commit(true, false), AfterCommit::Stopped); + assert_eq!(after_commit(true, true), AfterCommit::Stopped); + // No stop, but the power went anyway: the firmware has to restart. + assert_eq!(after_commit(false, true), AfterCommit::Stopped); + } + +} diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 102e8d813a83f..f9321745cecea 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -58,6 +58,7 @@ pub(crate) const CONFIG: Config = Config { m5: None, adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on // the firmware's own blame. recovery_slot_keys: None, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 9d61935a60807..98a36216528e2 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -49,6 +49,7 @@ pub(super) const BASE: Config = Config { m5: None, adopt_root_entries: false, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G16, firmware: fw::variant::Firmware::Identified(&[]), diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index ae2b48d9184d1..949bd81d00edd 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -40,6 +40,7 @@ pub(crate) const CONFIG: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J714), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, // The firmware in `fw-text` must be this build. iBoot loads the G17S build of both roles @@ -110,6 +111,7 @@ pub(crate) const CONFIG_MAX: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J716C), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, // As on the M5 Ultra, the firmware in `fw-text` must be this build. The boot loader loads @@ -191,6 +193,7 @@ pub(crate) const CONFIG_ULTRA: Config = Config { m5: Some(&crate::g17::fw::m5::initdata::J775D), adopt_root_entries: true, fabric_bridge: false, + sleep_restart: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, // On bare metal nothing vouches for the firmware the boot loader left in `fw-text`: it diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 891bb757db9e4..1ade3df12c2cb 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -56,6 +56,7 @@ pub(crate) const CONFIG: Config = Config { m5: None, adopt_root_entries: true, fabric_bridge: true, + sleep_restart: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 60e485f29f06b..b0e37fa1d8069 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -80,6 +80,7 @@ const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) /// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. const G17_GATING_CONTROL: usize = 0xd06030; const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) +const G17_GATING_HOLD: u32 = 0x4; // BIT(2) /// These performance selectors must be sampled before firmware can gate the core registers. const G17_PERF_CONTROL: usize = 0xe0141c; @@ -281,6 +282,37 @@ impl CpuControl { } } } + + /// The provider, for a system sleep's last step to stop the CPU with interrupts off. + pub(crate) fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + + /// Settles the provider's books for a CPU that a system sleep's last step stopped. + pub(crate) fn stopped_elsewhere(&self) { + // SAFETY: The constructor established the live, role-checked provider. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_cpu_stopped(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!( + self.dev.as_ref(), + "Could not settle a stopped GPU coprocessor: {:?}\n", + error + ); + } + } + } + + /// The CPU's control and status words. + pub(crate) fn state(&self) -> Result<(u32, u32)> { + let (mut control, mut status) = (0, 0); + // SAFETY: The constructor established the live, role-checked provider, and both pointers + // are writable. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_cpu_state(self.mbox, &mut control, &mut status) + })?; + Ok((control, status)) + } } /// Device resources for this GPU instance. @@ -892,6 +924,25 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } + + /// Hold part of the gating off, for a domain about to go off. The firmware must be idle. + pub(crate) fn hold_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val | G17_GATING_HOLD, G17_GATING_CONTROL) + } + + /// Both bridge words and the gating word. + pub(crate) fn g17_fabric_words(&self) -> Result<[u32; 3]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + Ok([ + sgx.try_read32(G17_BRIDGE_CONTROL[0])?, + sgx.try_read32(G17_BRIDGE_CONTROL[1])?, + sgx.try_read32(G17_GATING_CONTROL)?, + ]) + } } /// Split the packed per-cluster core masks into one mask per cluster. diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index a752c224959de..68a31b88c23dc 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include "../hid-ids.h" @@ -86,6 +87,9 @@ struct dchid_init_hdr { #define CMD_REGISTER_RING 0x91 #define CMD_SEND_FIRMWARE 0x95 #define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_HOST_STATE 0xc1 +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 #define CMD_ACK_GPIO_CMD 0xa1 struct dchid_init_block_hdr { @@ -202,6 +206,8 @@ struct dockchannel_hid { bool inherited_mtp; /* Between the system sleep suspend and resume callbacks */ bool suspended; + /* The firmware was told of a sleep and has not been told of the wake */ + bool host_asleep; /* Held from probe on when the multi-touch interface uses power method 2 */ struct gpio_desc *afe_reset; @@ -447,6 +453,14 @@ static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); } +static int dchid_host_state(struct dockchannel_hid *dchid, u8 state) +{ + u8 msg[] = { CMD_HOST_STATE, state }; + int ret = dchid_comm_cmd(dchid, msg, sizeof(msg)); + + return ret < 0 ? ret : 0; +} + static int dchid_enable_interface(struct dchid_iface *iface) { u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; @@ -513,7 +527,7 @@ static int dchid_register_ring(struct dockchannel_hid *dchid) .size = DCHID_RING_SIZE, }; /* Observed in both macOS and standalone T8132 bringup before 0x91. */ - u8 prepare[] = { 0xc1, 0x02 }; + u8 prepare[] = { CMD_HOST_STATE, HOST_STATE_AWAKE }; int ret = 0; if (!dchid->use_ring) @@ -1768,10 +1782,13 @@ static void dockchannel_hid_remove(struct platform_device *pdev) } /* - * The coprocessor keeps running across suspend-to-idle: there is no known - * sleep command for it, and its RTKit session and the trackpad AFE power - * sequence are not restartable. Only mark the sleep window, in which input - * may wake the system and requests from the HID core fail fast. + * The coprocessor keeps running across suspend-to-idle, and its RTKit session + * and the trackpad AFE power sequence are not restartable. Mark the sleep + * window, in which input may wake the system and requests from the HID core + * fail fast. A firmware-driven suspend also sets the host state, after the idle + * notice the helper sends in its .prepare, and the wake sets it back after the + * helper's .resume has sent the on notice, the device link ordering the two. That + * is macOS's order. */ static int dockchannel_hid_suspend(struct device *dev) { @@ -1788,6 +1805,13 @@ static int dockchannel_hid_suspend(struct device *dev) if (ret) goto unwind; } + if (pm_suspend_via_firmware()) { + /* A firmware that doesn't answer isn't a reason to stay awake. */ + ret = dchid_host_state(dchid, HOST_STATE_SLEEP); + if (ret) + dev_warn(dev, "host state not set: %d\n", ret); + dchid->host_asleep = !ret; + } WRITE_ONCE(dchid->suspended, true); return 0; @@ -1807,6 +1831,14 @@ static int dockchannel_hid_resume(struct device *dev) struct dockchannel_hid *dchid = dev_get_drvdata(dev); int i, ret = 0; + if (dchid->host_asleep) { + dchid->host_asleep = false; + ret = dchid_host_state(dchid, HOST_STATE_AWAKE); + if (ret) + dev_warn(dev, "host state not restored: %d\n", ret); + ret = 0; + } + /* Allow feature requests again before notifying any HID driver. */ WRITE_ONCE(dchid->suspended, false); for (i = 0; i < MAX_INTERFACES; i++) { diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index adacda51cc2e5..0971ac3085c29 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -17,6 +17,7 @@ #include #include #include +#include #define LID_WAKE_SETTLE_MS_J716S 2000 @@ -31,22 +32,26 @@ MODULE_PARM_DESC(lid_wake_settle_ms, * @smc: Pointer to apple_smc struct of the mfd parent * @input: Allocated input_dev; devres managed * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) - * @lid_lock: Serialises lid reports with the system sleep callbacks - * @lid_settle_ms: Window after system sleep starts in which a lid-open report is held back - * @wakeup_mode: Set to true when system is suspended and power button events should wake it - * @lid_held: A lid-open report was not passed on during this system sleep - * @sleep_start: CLOCK_BOOTTIME at which this system sleep started + * @pm_nb: Notifier for the whole suspend window, including early aborts + * @lid_lock: Serialises lid reports, the notifier and device sleep callbacks + * @lid_settle_ms: Early lid-open confirmation window for affected boards + * @wakeup_mode: Power/lid events are wake events during system sleep + * @lid_held: A lid-open report was held until a full resume + * @sleep_start: CLOCK_BOOTTIME at the start of the current sleep attempt + * @dark: A dark wake must recheck the lid before sleeping again */ struct macsmc_input { struct device *dev; struct apple_smc *smc; struct input_dev *input; struct notifier_block nb; + struct notifier_block pm_nb; unsigned int lid_settle_ms; - struct mutex lid_lock; /* protects the three fields below */ + struct mutex lid_lock; /* protects wakeup_mode, lid_held, sleep_start and dark */ bool wakeup_mode; bool lid_held; ktime_t sleep_start; + bool dark; }; #define SMC_EV_BTN 0x7201 @@ -59,18 +64,19 @@ struct macsmc_input { static void macsmc_input_event_button(struct macsmc_input *smcin, unsigned long event) { + bool wakeup_mode = READ_ONCE(smcin->wakeup_mode); u8 button = (event >> 8) & 0xff; u8 state = !!(event & 0xff); switch (button) { case BTN_POWER: case BTN_TOUCHID: - pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && state)); + pm_wakeup_dev_event(smcin->dev, 0, (wakeup_mode && state)); /* * Suppress KEY_POWER reports when suspended to avoid powering down * immediately after waking from s2idle. * */ - if (smcin->wakeup_mode) + if (wakeup_mode) return; input_report_key(smcin->input, KEY_POWER, state); @@ -151,6 +157,10 @@ static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long eve } pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + /* Through a sleep, .complete reports where the lid ended up. */ + if (smcin->wakeup_mode) + return; + input_report_switch(smcin->input, SW_LID, lid_state); input_sync(smcin->input); } @@ -173,6 +183,44 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +/* + * A suspend's first steps run before .prepare, and the SMC reports the lid and the button as + * plain events until its AP idle notice. Treat them as wakes from the suspend's start, so a lid + * opened or a button pressed while the machine goes down ends the suspend, as on macOS. + */ +static int macsmc_input_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, pm_nb); + + guard(mutex)(&smcin->lid_lock); + if (action == PM_SUSPEND_PREPARE) { + smcin->sleep_start = ktime_get_boottime(); + smcin->lid_held = false; + WRITE_ONCE(smcin->wakeup_mode, true); + } else if (action == PM_POST_SUSPEND) { + u8 val; + + WRITE_ONCE(smcin->wakeup_mode, false); + smcin->dark = false; + smcin->lid_held = false; + /* A suspend that ended before .prepare held its lid events back too. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0) { + input_report_switch(smcin->input, SW_LID, val); + input_sync(smcin->input); + } + } + + return NOTIFY_DONE; +} + +static void macsmc_input_unregister_pm_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + unregister_pm_notifier(&smcin->pm_nb); +} + static void macsmc_input_unregister_notifier(void *data) { struct macsmc_input *smcin = data; @@ -275,41 +323,64 @@ static int macsmc_input_probe(struct platform_device *pdev) if (error) return error; - return 0; + smcin->pm_nb.notifier_call = macsmc_input_pm_notify; + error = register_pm_notifier(&smcin->pm_nb); + if (error) + return error; + + return devm_add_action_or_reset(&pdev->dev, macsmc_input_unregister_pm_notifier, smcin); } static int macsmc_input_pm_prepare(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); + u8 val; guard(mutex)(&smcin->lid_lock); smcin->sleep_start = ktime_get_boottime(); smcin->lid_held = false; - smcin->wakeup_mode = true; + WRITE_ONCE(smcin->wakeup_mode, true); + /* A lid opened as the dark wake ended may have sent its event already. */ + if (smcin->dark) { + smcin->dark = false; + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD)) && + apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &val) >= 0 && !val) { + dev_info(dev, "the lid opened during a dark wake: not sleeping again\n"); + return -EBUSY; + } + } return 0; } static void macsmc_input_pm_complete(struct device *dev) { struct macsmc_input *smcin = dev_get_drvdata(dev); - u8 closed; + u8 lid; + bool held; int error; guard(mutex)(&smcin->lid_lock); - smcin->wakeup_mode = false; - if (!smcin->lid_held) + /* Keep the wake window and held lid event through an automatic dark re-sleep. */ + if (pm_dark_wake()) { + smcin->dark = apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &lid) < 0 || lid; return; + } + WRITE_ONCE(smcin->wakeup_mode, false); + held = smcin->lid_held; smcin->lid_held = false; - /* An open report was held back: report what the SMC says now. */ - error = apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &closed); - if (error < 0) { - dev_warn(dev, "Failed to read the lid state after sleep: %d\n", error); - return; + /* Full resume reports the actual state, including an open held for confirmation. */ + if (apple_smc_key_exists(smcin->smc, SMC_KEY(MSLD))) { + error = apple_smc_read_u8(smcin->smc, SMC_KEY(MSLD), &lid); + if (error < 0) { + dev_warn(dev, "Failed to read the lid state after sleep: %d\n", error); + return; + } + if (held) + dev_info(dev, "lid after sleep: MSLD=%u\n", lid); + input_report_switch(smcin->input, SW_LID, !!lid); + input_sync(smcin->input); } - dev_info(dev, "lid after sleep: MSLD=%u\n", closed); - input_report_switch(smcin->input, SW_LID, !!closed); - input_sync(smcin->input); } static const struct dev_pm_ops macsmc_input_pm_ops = { diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index c34aa47b9833f..7c7c2531868a1 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -306,6 +306,11 @@ struct apple_dart_fw_root { u64 *accepted; }; +struct apple_dart_sleep_list { + const u32 *pairs; + unsigned int count; +}; + /* * The subtrees behind the slots firmware holds in a four-level locked root * (fw_mirror), recorded when the root is mapped: per slot, the root entry and @@ -337,11 +342,29 @@ struct apple_dart_fw_tree { * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ +#define APPLE_DART_GAPF_PROTECT BIT(4) + struct apple_dart { struct device *dev; const struct apple_dart_hw *hw; void __iomem *regs; + void __iomem *gapf; + /* The GAPF slice as it read at probe; a deep sleep resets it to 0 */ + u32 gapf_boot; + bool sleep_flushed; + /* + * A display DART whose streams are off for a power-down: its registers may be + * unpowered, so maps use TCR as it was, and invalidations wait for the streams. + */ + bool display_dark; + bool display_tlb_stale; + DECLARE_BITMAP(display_tcr_four, DART_MAX_STREAMS); + /* The address filter the loader programs, from the device tree, replayed after a sleep. */ + void __iomem *dapf; + u64 *dapf_ranges; + u32 *dapf_r20; + unsigned int dapf_count; int irq; struct clk_bulk_data *clks; @@ -360,6 +383,10 @@ struct apple_dart { u32 per_stream : 1; u32 isp_host : 1; u32 power_retained : 1; + /* State a consumer saved for a firmware-driven suspend and will restore. */ + u32 sleep_saved : 1; + /* Its prepare list stopped the streams, and TLB commands wait for the restore. */ + u32 sleep_gated : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; /* @@ -374,10 +401,16 @@ struct apple_dart { u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; + /* Clocked by its host's port: the host resumes it after a real sleep. */ + u32 host_resumes : 1; + /* Reset and restored, its address filter written, at every power-up. */ + u32 restore_on_power_up : 1; bool commands_gated; /* Resume restored the translations, but the last steps of its reset failed. */ bool reset_incomplete; struct apple_tunable *tunables; + /* The bootloader sets these; written back where a power-off lost them. */ + struct apple_tunable *resume_tunables; u16 version; /* @@ -432,6 +465,14 @@ struct apple_dart { unsigned int pm_users; wait_queue_head_t pm_wq; u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ + /* The register writes around a firmware-driven suspend, from the device tree. */ + struct apple_dart_sleep_list sleep_prepare; + struct apple_dart_sleep_list sleep_restore; + struct apple_dart_sleep_list sleep_restore_late; + u32 sleep_enable[BITS_TO_U32(DART_MAX_STREAMS)]; + /* Its own config words at the sleep, which the restore keeps. */ + u32 sleep_cfg[20]; + /* Retained radio/display state left in place across suspend-to-idle. */ bool s2idle_kept; /* Resources reachable by a retained domain outlive device removal. */ @@ -665,21 +706,27 @@ static inline u32 apple_dart_readl(struct apple_dart *dart, u32 offset) return value; } -static void apple_dart_apply_tunables(struct apple_dart *dart) +/* Returns how many words it had to change. */ +static unsigned int apple_dart_apply_tunables(struct apple_dart *dart, + const struct apple_tunable *tunables) { + unsigned int changed = 0; size_t i; - for (i = 0; i < dart->tunables->sz; ++i) { + for (i = 0; i < tunables->sz; ++i) { u32 old_value, value; - old_value = apple_dart_readl(dart, - dart->tunables->values[i].offset); - value = old_value & ~dart->tunables->values[i].mask; - value |= dart->tunables->values[i].value; - if (value != old_value) + old_value = apple_dart_readl(dart, tunables->values[i].offset); + value = old_value & ~tunables->values[i].mask; + value |= tunables->values[i].value; + if (value != old_value) { apple_dart_writel(dart, value, - dart->tunables->values[i].offset); + tunables->values[i].offset); + changed++; + } } + + return changed; } /* @@ -2460,6 +2507,17 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) continue; + /* + * Its display is powered off: publish to memory only. With its streams off + * nothing walks the tables, and the TLB is invalidated before they're on. + */ + if (READ_ONCE(stream.dart->display_dark)) { + ret = stream.dart->locked ? + apple_dart_hw_sync_locked(cfg, &stream, true) : 0; + WRITE_ONCE(stream.dart->display_tlb_stale, true); + err = err ?: ret; + continue; + } ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { dev_err_ratelimited(stream.dart->dev, @@ -2590,7 +2648,10 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, } continue; } - four = apple_dart_readl(dart, DART_TCR(dart, sid)) & + if (READ_ONCE(dart->display_dark)) + four = test_bit(sid, dart->display_tcr_four); + else + four = apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level; idx = four ? 0 : addr >> (pgshift + 2 * bits); slot = (addr >> (pgshift + (four ? 2 : 1) * bits)) & @@ -4584,6 +4645,146 @@ static bool apple_dart_has_live_rtkit_client(struct device_node *np) return false; } +static void apple_dart_parse_sleep_list(struct apple_dart *dart, const struct resource *res, + const char *name, struct apple_dart_sleep_list *list) +{ + struct device_node *np = dart->dev->of_node; + unsigned int i; + u32 *pairs; + int count; + + count = of_property_count_u32_elems(np, name); + if (count <= 0) + return; + if (count % 2) { + dev_err(dart->dev, "%s: not a list of offset and value\n", name); + return; + } + pairs = devm_kcalloc(dart->dev, count, sizeof(*pairs), GFP_KERNEL); + if (!pairs || of_property_read_u32_array(np, name, pairs, count)) + return; + for (i = 0; i < count; i += 2) { + if (pairs[i] % 4 || pairs[i] + 4 > resource_size(res)) { + dev_err(dart->dev, "%s: bad offset %#x\n", name, pairs[i]); + return; + } + } + list->pairs = pairs; + list->count = count / 2; +} + +/* The config words a sleep list writes: 0x200 to 0x22c and 0x300 to 0x31c. */ +#define DART_SLEEP_CFG_WORDS 20 + +static unsigned int apple_dart_sleep_cfg_off(unsigned int i) +{ + return i < 12 ? 0x200 + 4 * i : 0x300 + 4 * (i - 12); +} + +static void apple_dart_sleep_cfg_read(struct apple_dart *dart, u32 *cfg) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + cfg[i] = readl(dart->regs + apple_dart_sleep_cfg_off(i)); +} + +/* + * The DART's address filter, which the loader programs and a real sleep clears: one entry + * per range, 0x40 apart, written in the order the loader writes them. + * apple,dapf-ranges gives start and end pairs from the ADT's dapf-instance-0, and the + * optional apple,dapf-r20 each entry's last word, 1 if absent. + */ +#define DART_DAPF_LOCK 0x208 +#define DART_DAPF_LOCKED BIT(2) +#define DART_DAPF_STRIDE 0x40 +#define DART_DAPF_CONFIG 0x31 + +static int apple_dart_dapf_parse(struct platform_device *pdev, struct apple_dart *dart) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf"); + int n = of_property_count_u64_elems(np, "apple,dapf-ranges"); + unsigned int i; + + /* A named DAPF window also exists on inherited cold-boot paths that + * provide no replay description. Keep their loader setup untouched. + * An explicit replay request or orphan metadata still requires data. + */ + if (!of_property_present(np, "apple,dapf-ranges")) { + if (of_property_present(np, "apple,dapf-r20") || + of_property_read_bool(np, "apple,restore-on-power-up")) + return dev_err_probe(dev, -EINVAL, + "DAPF replay requested without apple,dapf-ranges\n"); + return 0; + } + if (n <= 0 || n % 2) + return dev_err_probe(dev, -EINVAL, "apple,dapf-ranges needs start and end pairs\n"); + dart->dapf_count = n / 2; + if (resource_size(res) < dart->dapf_count * DART_DAPF_STRIDE) + return dev_err_probe(dev, -EINVAL, "DAPF region too small for %u ranges\n", + dart->dapf_count); + + dart->dapf = devm_platform_ioremap_resource_byname(pdev, "dapf"); + if (IS_ERR(dart->dapf)) + return PTR_ERR(dart->dapf); + dart->dapf_ranges = devm_kcalloc(dev, n, sizeof(*dart->dapf_ranges), GFP_KERNEL); + dart->dapf_r20 = devm_kcalloc(dev, dart->dapf_count, sizeof(*dart->dapf_r20), GFP_KERNEL); + if (!dart->dapf_ranges || !dart->dapf_r20) + return -ENOMEM; + of_property_read_u64_array(np, "apple,dapf-ranges", dart->dapf_ranges, n); + for (i = 0; i < dart->dapf_count; i++) + dart->dapf_r20[i] = 1; + if (of_property_count_u32_elems(np, "apple,dapf-r20") == dart->dapf_count) + of_property_read_u32_array(np, "apple,dapf-r20", dart->dapf_r20, dart->dapf_count); + + return 0; +} + +/* Entry 0 as written: the end reads back with its low two bits set, the last word's bytes. */ +static bool apple_dart_dapf_holds(struct apple_dart *dart) +{ + return readq(dart->dapf + 0x08) == dart->dapf_ranges[0] && + (readq(dart->dapf + 0x10) | 3) == (dart->dapf_ranges[1] | 3); +} + +/* + * After a power-off, before any consumer's firmware starts. Not while it's kept, unless + * @always, and not while it's locked. Each entry's config word goes last, after its + * streams, as macOS writes it. + */ +static void apple_dart_dapf_replay(struct apple_dart *dart, bool always) +{ + unsigned int i; + + if (!dart->dapf_count) + return; + if (!always && apple_dart_dapf_holds(dart)) + return; + if (readl(dart->regs + DART_DAPF_LOCK) & DART_DAPF_LOCKED) { + dev_warn(dart->dev, "DAPF locked after the sleep: not replayed\n"); + return; + } + + for (i = 0; i < dart->dapf_count; i++) { + void __iomem *entry = dart->dapf + i * DART_DAPF_STRIDE; + + writel(0, entry + 0x04); + writeq(dart->dapf_ranges[2 * i], entry + 0x08); + writeq(dart->dapf_ranges[2 * i + 1], entry + 0x10); + writel(dart->dapf_r20[i], entry + 0x20); + writel(DART_DAPF_CONFIG, entry + 0x00); + } + /* Its consumer's firmware faults on its first access without it: say so plainly. */ + if (!apple_dart_dapf_holds(dart)) { + dev_err(dart->dev, "DAPF write-back didn't take: entry 0 reads %#llx..%#llx\n", + readq(dart->dapf + 0x08), readq(dart->dapf + 0x10)); + return; + } + dev_dbg(dart->dev, "DAPF: %u ranges written back after the sleep\n", dart->dapf_count); +} + /* * The setup a T8110 DART needs when nothing initialised it before Linux: * apple,tunable (applied at every reset), apple,bypass-sids (streams kept in @@ -4868,6 +5069,19 @@ static int apple_dart_probe(struct platform_device *pdev) dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "gapf")) { + dart->gapf = devm_platform_ioremap_resource_byname(pdev, "gapf"); + if (IS_ERR(dart->gapf)) + return PTR_ERR(dart->gapf); + dart->gapf_boot = readl(dart->gapf); + } + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "dapf")) { + ret = apple_dart_dapf_parse(pdev, dart); + if (ret) + return ret; + } + dart->restore_on_power_up = of_property_read_bool(dev->of_node, + "apple,restore-on-power-up"); if (tunneled) { dart->tunables = devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable", res); @@ -4888,6 +5102,14 @@ static int apple_dart_probe(struct platform_device *pdev) } } } + dart->resume_tunables = devm_apple_tunable_parse(dev, dev->of_node, + "apple,resume-tunable", res); + if (IS_ERR(dart->resume_tunables)) { + if (PTR_ERR(dart->resume_tunables) != -ENOENT) + return dev_err_probe(dev, PTR_ERR(dart->resume_tunables), + "failed to parse resume tunables\n"); + dart->resume_tunables = NULL; + } if (j713_dcpext || j713_dispext) { /* @@ -4934,7 +5156,7 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled && dart->tunables) { dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", dart->tunables->sz); - apple_dart_apply_tunables(dart); + apple_dart_apply_tunables(dart, dart->tunables); } if (tunneled) { @@ -5108,6 +5330,10 @@ static int apple_dart_probe(struct platform_device *pdev) ret = -EBUSY; goto err_clk_disable; } + apple_dart_parse_sleep_list(dart, res, "apple,sleep-prepare", &dart->sleep_prepare); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore", &dart->sleep_restore); + apple_dart_parse_sleep_list(dart, res, "apple,sleep-restore-late", + &dart->sleep_restore_late); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); /* @@ -5619,11 +5845,191 @@ static void apple_dart_save_state(struct apple_dart *dart) } } +static void apple_dart_sleep_write(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, list->pairs[2 * i + 1], list->pairs[2 * i]); +} + +/* + * A sleep list's config words go back as this DART had them before the sleep, not as + * the list has them: Linux's own setup of those words can differ from macOS's. + */ +static u32 apple_dart_sleep_value(struct apple_dart *dart, u32 off, u32 val) +{ + unsigned int i; + + for (i = 0; i < DART_SLEEP_CFG_WORDS; i++) + if (apple_dart_sleep_cfg_off(i) == off) + return dart->sleep_cfg[i]; + return val; +} + +/* A sleep list, with its config words as the DART had them before the sleep. */ +static void apple_dart_sleep_write_kept(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + for (i = 0; i < list->count; i++) + apple_dart_writel(dart, + apple_dart_sleep_value(dart, list->pairs[2 * i], + list->pairs[2 * i + 1]), + list->pairs[2 * i]); +} + +static void apple_dart_sleep_wait_tlb(struct apple_dart *dart) +{ + u32 status; + + if (readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT)) + dev_err(dart->dev, "TLB command still busy around a sleep: %#x\n", status); +} + +/* Whether a sleep list stops streams, which the restore then has to start again. */ +static bool apple_dart_sleep_list_stops_streams(struct apple_dart *dart, + const struct apple_dart_sleep_list *list) +{ + unsigned int i; + + if (dart->hw->type != DART_T8110) + return false; + for (i = 0; i < list->count; i++) + if (list->pairs[2 * i] >= DART_T8110_DISABLE_STREAMS && + list->pairs[2 * i] < DART_T8110_DISABLE_STREAMS + 0x20) + return true; + return false; +} + +/** + * apple_dart_sleep_listed() - Whether a DART's consumer saves and restores it for a sleep + * @dev: the DART + * + * Return: true if its node has apple,sleep-restore and the DART driver is bound. + */ +bool apple_dart_sleep_listed(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + return dart && dart->sleep_restore.count; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_listed); + +/** + * apple_dart_sleep_prepare() - Save a DART whose consumer is about to gate itself + * @dev: the DART + * + * For a DART with apple,sleep-restore in its node. A consumer calls this before + * telling its firmware to sleep, while the DART's state can be read, and + * apple_dart_sleep_restore() before starting that firmware again. The DART's own + * callbacks then leave it alone. Without the property nothing happens here, and + * they do what they always do. The state is saved before apple,sleep-prepare is + * written, so that list may stop the streams. + * + * Return: 0, or -ENODEV if the DART driver isn't bound. + */ +int apple_dart_sleep_prepare(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + if (!dart) + return -ENODEV; + if (!dart->sleep_restore.count) + return 0; + + apple_dart_sleep_cfg_read(dart, dart->sleep_cfg); + spin_lock_irqsave(&dart->lock, flags); + apple_dart_save_state(dart); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + dart->sleep_enable[i] = apple_dart_readl(dart, dart->hw->enable_streams + 4 * i); + apple_dart_sleep_write(dart, &dart->sleep_prepare); + apple_dart_sleep_wait_tlb(dart); + /* Flushed with no stream left to fill it: nothing may reach a DART whose power goes. */ + if (apple_dart_sleep_list_stops_streams(dart, &dart->sleep_prepare) && + !dart->commands_gated) { + WRITE_ONCE(dart->commands_gated, true); + dart->sleep_gated = true; + } + dart->sleep_saved = true; + spin_unlock_irqrestore(&dart->lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_prepare); + +/** + * apple_dart_sleep_restore_lost() - Put back a DART saved by apple_dart_sleep_prepare() + * @dev: the DART + * @lost: its power went off since, so its registers are at their reset values + * + * A DART that kept its registers only gets back what its prepare list stopped: the late + * list and the streams. + */ +void apple_dart_sleep_restore_lost(struct device *dev, bool lost) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx, i; + unsigned long flags; + + if (!dart || !dart->sleep_saved) + return; + + spin_lock_irqsave(&dart->lock, flags); + dart->sleep_saved = false; + if (lost) { + apple_dart_sleep_write_kept(dart, &dart->sleep_restore); + for (sid = 0; sid < dart->num_streams; sid++) { + if (!dart->save_tcr[sid] && !dart->save_ttbr[sid][0]) + continue; + apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + } + } + if (lost || dart->sleep_gated) { + apple_dart_sleep_write_kept(dart, &dart->sleep_restore_late); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, dart->sleep_enable[i], + dart->hw->enable_streams + 4 * i); + apple_dart_sleep_wait_tlb(dart); + } + if (dart->sleep_gated) { + dart->sleep_gated = false; + WRITE_ONCE(dart->commands_gated, false); + } + spin_unlock_irqrestore(&dart->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_restore_lost); + +/** + * apple_dart_sleep_restore() - The same, for a consumer whose DART loses its registers + * only in a real sleep + * @dev: the DART + * + * A suspend that was refused lost nothing, so nothing is written then. + */ +void apple_dart_sleep_restore(struct device *dev) +{ + apple_dart_sleep_restore_lost(dev, pm_resume_via_firmware()); +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_restore); + static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); dart->tunnel_state_restored = false; + /* Its consumer saved this already and restores it: see apple_dart_sleep_prepare(). */ + if (dart->sleep_saved) + return 0; if (READ_ONCE(dart->commands_gated)) return 0; /* @@ -5759,7 +6165,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) int ret; /* No reset or restore: see apple_dart_suspend(). */ - if (dart->cold_ane) + if (dart->cold_ane || dart->sleep_saved) + return 0; + /* Its block can't answer yet: apple_dart_host_resume() restores it. */ + if (dart->host_resumes && READ_ONCE(dart->commands_gated)) return 0; /* Powered down by its DCP, which restores it at power-up. */ if (READ_ONCE(dart->host_off)) @@ -5799,7 +6208,13 @@ static __maybe_unused int apple_dart_resume(struct device *dev) if (dart->locked) return 0; - if (apple_dart_state_retained(dart)) { + /* + * Not for a DART restored at every power-up, as macOS's SPTM does (its + * t8110dart powerup): T8140's ISP DART keeps Linux's values through a wake + * that kept DRAM up, but its address filter takes effect only once written + * again, and the firmware's first access through it faults. + */ + if (!dart->restore_on_power_up && apple_dart_state_retained(dart)) { /* Nothing was lost, only drop what the TLBs may still hold. */ if (!dart->reset_incomplete) return apple_dart_invalidate_each_stream(dart); @@ -5836,12 +6251,146 @@ static __maybe_unused int apple_dart_resume(struct device *dev) ret = apple_dart_hw_finish_reset(dart); } + /* The address filter went with the same power-off, and so may these. */ + apple_dart_dapf_replay(dart, dart->restore_on_power_up); + if (dart->resume_tunables) { + unsigned int changed = apple_dart_apply_tunables(dart, dart->resume_tunables); + + if (changed) + dev_dbg(dev, "%u of %zu tunables written back\n", changed, + dart->resume_tunables->sz); + } + /* Should they fail again, the next resume repeats them. */ dart->reset_incomplete = ret; return ret; } -static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); +static __maybe_unused int apple_dart_resume_noirq(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + /* + * After a real sleep a DART clocked by its host's port reads nothing until the host + * sets the port up again. Gate it, so that nothing touches it, until then. + */ + if (dart->host_resumes && pm_resume_via_firmware()) { + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, true); + return pm_runtime_force_resume(dev); + } + + ret = pm_runtime_force_resume(dev); + + /* + * A real sleep took the registers even of a DART runtime PM had suspended, which + * force_resume leaves alone. Its consumer's firmware may DMA through it before + * anything resumes it at runtime, so write the saved state back now. Not one with its + * own sleep list, as the SEP's: its consumer restores that with macOS's writes before + * starting its firmware, and this would land on top. + */ + if (!ret && pm_resume_via_firmware() && pm_runtime_status_suspended(dev) && + !dart->sleep_restore.count) { + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after sleep failed: %d\n", ret); + else + dev_dbg(dev, "restored after sleep\n"); + } + if (pm_resume_via_firmware()) + apple_dart_dapf_replay(dart, false); + + return ret; +} + +/* The host will set this DART's port clock up again after a real sleep, and resume it then. */ +int apple_dart_set_host_resumes(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart) + return -EPROBE_DEFER; + if (dart->tunneled || dart->power_retained || dart->sleep_saved || dart->cold_ane) + return -EINVAL; + dart->host_resumes = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_set_host_resumes); + +/* The host's port is up again: ungate the DART and restore what it saved. */ +int apple_dart_host_resume(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + if (!dart || !dart->host_resumes) + return -EINVAL; + if (!READ_ONCE(dart->commands_gated)) + return 0; + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, false); + ret = apple_dart_resume(dev); + if (ret) + dev_err(dev, "restore after its host came back failed: %d\n", ret); + else + dev_info(dev, "restored after its host came back\n"); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_host_resume); + +/* + * Its consumer's own power cycle, outside a system sleep: the registers saved and put back. + * Only for a device apple-dart is bound to. + */ +int apple_dart_state_save(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_suspend(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_save); + +int apple_dart_state_restore(struct device *dev) +{ + if (dev->driver != &apple_dart_driver.driver || !dev_get_drvdata(dev)) + return -ENODEV; + return apple_dart_resume(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_state_restore); + +/* + * A firmware-driven sleep takes the DARTs in the noirq phase, which every device finishes + * before any early resume: coprocessors such as the PMP and SIO restart their firmware in + * their early resume, and its first DMA has to find the DART already restored. Any other + * sleep keeps the normal phase, which the PCIe-C tunnel DARTs need: their host stops and + * restarts the port clock in its own noirq callbacks. + */ +static __maybe_unused int apple_dart_sys_suspend(struct device *dev) +{ + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_suspend(dev); +} + +static __maybe_unused int apple_dart_sys_resume(struct device *dev) +{ + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_resume(dev); +} + +static __maybe_unused int apple_dart_sys_suspend_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? pm_runtime_force_suspend(dev) : 0; +} + +static __maybe_unused int apple_dart_sys_resume_noirq(struct device *dev) +{ + return pm_suspend_via_firmware() ? apple_dart_resume_noirq(dev) : 0; +} + +static const struct dev_pm_ops apple_dart_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend, apple_dart_sys_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend_noirq, apple_dart_sys_resume_noirq) + RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) +}; static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -6213,6 +6762,157 @@ int apple_dart_dcp_dispext_state(struct device_node *consumer) EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_state); #endif +/* + * Bit 4 of this DART's GAPF slice makes the fabric reject writes to its coprocessor's + * CPU power state. It's clear from the flush until the streams come back on. + */ +static int apple_dart_gapf_release(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf) + return 0; + + val = readl(dart->gapf); + writel(val & ~APPLE_DART_GAPF_PROTECT, dart->gapf); + if (readl(dart->gapf) & APPLE_DART_GAPF_PROTECT) { + dev_err(dart->dev, "GAPF slice stayed protected: %#x\n", val); + return -EIO; + } + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); + + return 0; +} + +/* Back to the probe value: after a wake the slice reads 0, and protect alone would give 0x10. */ +static void apple_dart_gapf_protect(struct apple_dart *dart) +{ + u32 val; + + if (!dart->gapf || !dart->sleep_flushed) + return; + + val = readl(dart->gapf); + writel(val | dart->gapf_boot | APPLE_DART_GAPF_PROTECT, dart->gapf); + dev_dbg(dart->dev, "GAPF slice %#x -> %#x\n", val, readl(dart->gapf)); +} + +/* Before its coprocessor's CPU domain is written: a protected slice would fault the write. */ +int apple_dart_gapf_ready(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (!dart->gapf || !(readl(dart->gapf) & APPLE_DART_GAPF_PROTECT)) + return 0; + + return apple_dart_gapf_release(dart); +} +EXPORT_SYMBOL_GPL(apple_dart_gapf_ready); + +/* + * What macOS writes to T8140's display DARTs around a display power cycle: + * measured under aura, with the offsets the driver has no name for. It never + * writes their TCR or TTBR, and reads the same values back after each power-on, + * deep sleep included, so neither does this. + * + * Stopping again after a sleep that never reached the commit does nothing: the + * domains may be off by then. + */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long bits = streams; + unsigned int sid; + u32 status; + int ret; + + if (on) { + apple_dart_gapf_protect(dart); + dart->sleep_flushed = false; + /* Maps or unmaps while it was dark left their invalidations here. */ + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); + writel(0, dart->regs + 0x210); + writel(0, dart->regs + 0x4060); + writel(streams, dart->regs + DART_T8110_ENABLE_STREAMS); + return 0; + } + + if (flush && dart->sleep_flushed) + return 0; + + if (!READ_ONCE(dart->display_dark)) { + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + } + + writel(U32_MAX, dart->regs + 0x4060); + writel(0, dart->regs + 0x4000); + writel(U32_MAX, dart->regs + 0x704); + for_each_set_bit(sid, &bits, BITS_PER_TYPE(streams)) + writel(BIT(sid), dart->regs + DART_T8110_DISABLE_STREAMS); + if (!flush) + return 0; + + writel(0xa000, dart->regs + DART_T8110_TLB_CMD); + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT); + if (ret) { + dev_err(dev, "flush for a power-down didn't finish: %#x\n", status); + return ret; + } + ret = apple_dart_gapf_release(dart); + if (ret) + return ret; + dart->sleep_flushed = true; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_display_streams); + +/* + * A DART whose domain goes off with its consumer's, at runtime, while other users still map + * into it: dark, maps reach memory only and their invalidations wait; lit, those run. Its + * streams are left as they are. Call it dark while it's still powered. + */ +void apple_dart_power_dark(struct device *dev, bool dark) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid; + + if (dark) { + if (READ_ONCE(dart->display_dark)) + return; + for (sid = 0; sid < dart->num_streams; sid++) + __assign_bit(sid, dart->display_tcr_four, + readl(dart->regs + DART_TCR(dart, sid)) & + dart->hw->tcr_4level); + WRITE_ONCE(dart->display_dark, true); + return; + } + if (READ_ONCE(dart->display_tlb_stale)) { + struct apple_dart_stream_map all = { .dart = dart }; + + bitmap_fill(all.sidmap, dart->num_streams); + if (dart->hw->invalidate_tlb(&all)) + dev_err(dev, "TLB invalidation at power-on failed\n"); + WRITE_ONCE(dart->display_tlb_stale, false); + } + WRITE_ONCE(dart->display_dark, false); +} +EXPORT_SYMBOL_GPL(apple_dart_power_dark); + static struct platform_driver apple_dart_driver = { .driver = { .name = "apple-dart", diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 1125008130ef4..393596ff6231b 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -281,6 +284,8 @@ struct aic_info { * comes online. */ bool siq_cfg_routing; + /* A firmware-driven suspend powers the controller down: aic_syscore_ops keep it. */ + bool fw_suspend; }; static const struct aic_info aic1_info __initconst = { @@ -333,6 +338,7 @@ static const struct aic_info aic3_t8140_info __initconst = { .fast_ipi = true, .local_fast_ipi = true, .siq_cfg_routing = true, + .fw_suspend = true, }; static const struct aic_info aic3_t8142_info __initconst = { @@ -388,6 +394,8 @@ struct aic_irq_chip { int nr_die; int max_die; + u32 *sleep_masks; + struct aic_info info; }; @@ -410,6 +418,47 @@ static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) writel_relaxed(val, ic->base + reg); } +/* + * A sleep that a firmware engine finishes waits for the CPUs to stop, and a raised + * line ends WFI even with interrupts off. Every line is masked for that last step. + */ +void aic_sleep_mask_all(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, reg, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++) { + reg = ic->info.mask_set + off + i * 4; + *saved++ = aic_ic_read(ic, reg); + aic_ic_write(ic, reg, U32_MAX); + } + } +} + +void aic_sleep_restore(void) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 words = ic->max_irq >> 5, off = 0, i; + u32 *saved = ic->sleep_masks; + int die; + + for (die = 0; die < ic->nr_die; die++, off += ic->info.die_stride) { + for (i = 0; i < words; i++, saved++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, *saved); + aic_ic_write(ic, ic->info.mask_clr + off + i * 4, ~*saved); + } + } +} + +/* Takes the next event, which also masks its line. */ +u32 aic_sleep_ack(void) +{ + return readl(aic_irqc->event + aic_irqc->info.event); +} + /* * IRQ irqchip */ @@ -1083,6 +1132,133 @@ static void build_fiq_affinity(struct aic_irq_chip *ic, struct device_node *aff) } } +#ifdef CONFIG_PM_SLEEP +/* + * A firmware-driven suspend powers the controller down and back up. How + * much it keeps isn't measured, and macOS saves and restores it in software. So + * mask everything before the commit, so that no line can end the final WFI, and + * put it all back after. Linux never writes IRQ_CFG or these global registers + * outside of config's enable bit, so they hold what the bootloader set. + */ +static struct { + u32 config; + u32 timeout; + u32 cluster_prio; + u32 *irq_cfg; + bool saved; +} aic_sleep; + +static int aic_syscore_suspend(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!pm_suspend_via_firmware()) + return 0; + + aic_sleep.config = aic_ic_read(ic, AIC2_CONFIG); + aic_sleep.timeout = aic_ic_read(ic, AIC2_TIMEOUT); + aic_sleep.cluster_prio = aic_ic_read(ic, AIC2_CLUSTER_PRIO); + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < ic->nr_irq; i++) + aic_sleep.irq_cfg[die * ic->nr_irq + i] = + aic_ic_read(ic, ic->info.irq_cfg + off + i * 4); + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + off += ic->info.die_stride; + } + aic_sleep.saved = true; + + return 0; +} + +/* The boot CPU doesn't pass through hotplug, so aic_init_cpu() doesn't rerun. */ +static void aic_resume_boot_cpu(void) +{ + u32 fiq = __this_cpu_read(aic_fiq_unmasked); + + aic_init_cpu(smp_processor_id()); + /* + * aic_init_cpu() is for a CPU with no FIQ users yet. This one's kept theirs: + * a timer left marked masked here would never be unmasked at its next EOI. + */ + __this_cpu_write(aic_fiq_unmasked, fiq); + if (is_kernel_in_hyp_mode()) { + if (fiq & BIT(AIC_TMR_EL02_PHYS)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_P, false); + if (fiq & BIT(AIC_TMR_EL02_VIRT)) + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V, false); + isb(); + } +} + +static void aic_syscore_resume(void *data) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 die, i, off = 0; + + if (!aic_sleep.saved) + return; + aic_sleep.saved = false; + + /* Otherwise the suspend was refused, and the controller stayed powered. */ + if (pm_resume_via_firmware()) { + aic_ic_write(ic, AIC2_TIMEOUT, aic_sleep.timeout); + aic_ic_write(ic, AIC2_CLUSTER_PRIO, aic_sleep.cluster_prio); + aic_ic_write(ic, AIC2_CONFIG, aic_sleep.config); + aic_resume_boot_cpu(); + } + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < BITS_TO_U32(ic->nr_irq); i++) { + aic_ic_write(ic, ic->info.mask_set + off + i * 4, U32_MAX); + aic_ic_write(ic, ic->info.sw_clr + off + i * 4, U32_MAX); + } + for (i = 0; i < ic->nr_irq; i++) { + u32 saved = aic_sleep.irq_cfg[die * ic->nr_irq + i]; + u32 reg = ic->info.irq_cfg + off + i * 4; + unsigned int virq; + + if (aic_ic_read(ic, reg) != saved) + aic_ic_write(ic, reg, saved); + + /* Restore from the IRQ core's books, not from the hardware's. */ + virq = irq_find_mapping(ic->hw_domain, AIC_IRQ_HWIRQ(die, i)); + if (virq && !irqd_irq_masked(irq_get_irq_data(virq))) + aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(i), + MASK_BIT(i)); + } + off += ic->info.die_stride; + } +} + +static const struct syscore_ops aic_syscore_ops = { + .suspend = aic_syscore_suspend, + .resume = aic_syscore_resume, +}; + +static struct syscore aic_syscore = { + .ops = &aic_syscore_ops, +}; + +static void __init aic_sleep_init(struct aic_irq_chip *irqc) +{ + if (!irqc->info.fw_suspend) + return; + + aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); + if (WARN_ON(!aic_sleep.irq_cfg)) + return; + + register_syscore(&aic_syscore); +} +#else +static void __init aic_sleep_init(struct aic_irq_chip *irqc) +{ +} +#endif + static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { int i, die; @@ -1168,6 +1344,11 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->info.die_stride = off - start_off; + irqc->sleep_masks = kcalloc(irqc->nr_die * (irqc->max_irq >> 5), + sizeof(*irqc->sleep_masks), GFP_KERNEL); + if (WARN_ON(!irqc->sleep_masks)) + goto err_unmap; + irqc->hw_domain = irq_domain_create_tree(of_fwnode_handle(node), &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) @@ -1219,6 +1400,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); + aic_sleep_init(irqc); + if (is_kernel_in_hyp_mode()) { struct irq_fwspec mi = { .fwnode = of_fwnode_handle(node), diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 72bcefdae4373..4cb1ae2800182 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -542,6 +542,61 @@ static void isp_test_command_ack_failures(struct kunit *test) KUNIT_EXPECT_EQ(test, state.calls, 0); } +/* The firmware's answer to the stop key request, as macOS reads it */ +static int isp_test_stop_key_reply(struct apple_isp *isp, + struct isp_channel *chan, + unsigned long timeout) +{ + struct kunit *test = kunit_get_current_test(); + unsigned int *calls = test->priv; + u8 *cmd = isp->cmd_virt; + + (*calls)++; + KUNIT_EXPECT_PTR_EQ(test, chan, isp->chan_dg); + KUNIT_EXPECT_EQ(test, chan->req.arg1, 0x20); + KUNIT_EXPECT_EQ(test, chan->req.arg2, 0x20); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(cmd + 4), 0x1d); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(cmd + 8), 1); + KUNIT_EXPECT_TRUE(test, timeout); + put_unaligned_le64(0x10001e00010ULL, cmd + 0x10); + put_unaligned_le64(0x8042006, cmd + 0x18); + chan->rsp = (struct isp_message){ .arg0 = chan->req.arg0 | ISP_IPC_FLAG_ACK, + .arg1 = chan->req.arg2 }; + return 0; +} + +static void isp_test_stop_key_on_debug(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct cmd_stop_key key; + unsigned int calls = 0; + + isp->cmd_virt = kunit_kzalloc(test, 0x400, GFP_KERNEL); + isp->chan_io = kunit_kzalloc(test, sizeof(*isp->chan_io), GFP_KERNEL); + isp->chan_dg = kunit_kzalloc(test, sizeof(*isp->chan_dg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_io); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_dg); + isp->chan_io->name = "IO"; + isp->chan_dg->name = "DEBUG"; + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_COMMAND; + test->priv = &calls; + kunit_deactivate_static_stub(test, isp_cmd_send); + kunit_activate_static_stub(test, ipc_chan_send, isp_test_stop_key_reply); + + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), 0); + KUNIT_EXPECT_EQ(test, calls, 1); + KUNIT_EXPECT_EQ(test, key.addr, 0x10001e00010ULL); + KUNIT_EXPECT_EQ(test, key.key, 0x8042006ULL); + + /* No DEBUG command ring: nothing is sent, on IO either. */ + isp->chan_dg->type = ISP_IPC_CHAN_TYPE_REPLY; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + isp->chan_dg = NULL; + KUNIT_EXPECT_EQ(test, isp_cmd_stop_key_get(isp, &key), -ENODEV); + KUNIT_EXPECT_EQ(test, calls, 1); +} + static void isp_test_call_crop(struct kunit *test) { struct isp_preset source = { @@ -658,6 +713,7 @@ static struct kunit_case isp_cam_test_cases[] = { KUNIT_CASE(isp_test_setfile_failure_stops_setup), KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failures), + KUNIT_CASE(isp_test_stop_key_on_debug), KUNIT_CASE(isp_test_sensor_geometry), KUNIT_CASE(isp_test_sensor_config_list), KUNIT_CASE(isp_test_sensor_config_failure), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 0c618146c8716..075217c95a9b0 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -565,11 +565,14 @@ int apple_isp_detect_camera(struct apple_isp *isp) err = isp_detect_camera(isp); - /* Resident firmware stays up; the channel is set up per stream. */ + /* + * Resident firmware stays up, the channel set up per stream, unless it starts + * again: then it's off until the first stream, as on macOS. + */ if (isp->hw->resident_fw) { - if (err) + if (err || isp->hw->fw_restart) apple_isp_firmware_halt(isp); - else + if (!err) isp->camera_detected = true; return err; } @@ -1080,7 +1083,7 @@ int apple_isp_start_capture(struct apple_isp *isp) void apple_isp_stop_capture(struct apple_isp *isp) { - if (isp->ipc_failed || isp->fw_state == ISP_FW_DEAD) + if (isp->ipc_failed || isp->fw_state != ISP_FW_RUNNING) return; isp_cmd_ch_stop(isp, 0); // TODO channel mask isp_cmd_ch_buffer_return(isp, isp->current_ch); diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 75b8704ccedcc..a1a5557111ec0 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -16,26 +16,26 @@ #define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) #define CISP_TIMEOUT msecs_to_jiffies(3000) +/* what macOS allows a DEBUG command */ +#define CISP_DEBUG_TIMEOUT msecs_to_jiffies(1000) #define CISP_SEND_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) #define CISP_SEND_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) #define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) #define CISP_POST_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, 0)) #define CISP_POST_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), 0)) -int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +/* Send through the command area on a command channel. */ +static int isp_cmd_send_chan(struct apple_isp *isp, struct isp_channel *chan, + void *args, u32 insize, u32 outsize, + unsigned long timeout) { - struct isp_channel *chan; struct isp_message *req; int err; -#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) - KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); -#endif /* Before the command area is overwritten */ err = isp_ipc_resync(isp); if (err) return err; - chan = isp->chan_io; req = &chan->req; req->arg0 = isp->cmd_iova; @@ -69,6 +69,15 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int return err; } +int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); +#endif + return isp_cmd_send_chan(isp, isp->chan_io, args, insize, outsize, + timeout); +} + static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, u32 outsize) { @@ -99,6 +108,16 @@ int isp_cmd_stop(struct apple_isp *isp, u32 mode) return CISP_SEND_IN(isp, args); } +/* T8140's cold stop, which macOS sends as a command with its reply. */ +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + int isp_cmd_power_down(struct apple_isp *isp) { struct cmd_power_down args = { @@ -133,6 +152,30 @@ int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) return CISP_SEND_INOUT(isp, args); } +/* + * macOS asks for the key on the DEBUG channel. On IO, 0x1d is another + * command, and the firmware answers neither it nor the STOP after it. + */ +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args) +{ + struct isp_channel *chan = isp->chan_dg; + int err; + + if (!chan || chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + return -ENODEV; + + memset(args, 0, sizeof(*args)); + args->opcode = CISP_OPCODE(CISP_DEBUG_CMD_STOP_KEY_GET); + args->enable = 1; + err = isp_cmd_send_chan(isp, chan, args, sizeof(*args), sizeof(*args), + CISP_DEBUG_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, sizeof(*args)); + return 0; +} + int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) { args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 9e2678bd1610c..ab98f3447d6f7 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -101,6 +101,9 @@ #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD 0xc11d #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH 0xc120 +/* Opcodes of the DEBUG channel, which has its own numbering */ +#define CISP_DEBUG_CMD_STOP_KEY_GET 0x001d + #define CISP_POOL_TYPE_META 0x0 #define CISP_POOL_TYPE_RENDERED 0x1 #define CISP_POOL_TYPE_FD 0x2 @@ -159,6 +162,19 @@ struct cmd_print_enable { } __packed; static_assert(sizeof(struct cmd_print_enable) == 0xc); +/* + * CISP_DEBUG_CMD_STOP_KEY_GET: the firmware answers with a word of its + * memory and the value it writes there once it has stopped. + */ +struct cmd_stop_key { + u64 opcode; + u32 enable; + u32 pad_c; + u64 addr; + u64 key; +} __packed; +static_assert(sizeof(struct cmd_stop_key) == 0x20); + struct cmd_trace_enable { u64 opcode; u32 enable; @@ -263,10 +279,12 @@ int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout); int isp_cmd_start(struct apple_isp *isp, u32 mode); int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_stop_cold(struct apple_isp *isp, u32 mode); int isp_cmd_power_down(struct apple_isp *isp); int isp_cmd_suspend(struct apple_isp *isp); int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_stop_key_get(struct apple_isp *isp, struct cmd_stop_key *args); int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 21dd9cdc1be2a..245af9fa32f35 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "isp-cam.h" @@ -525,6 +526,17 @@ isp_check_firmware_version(struct apple_isp *isp) return compat; } +/* After a system sleep, see apple_isp_resume(). */ +static void apple_isp_resume_work(struct work_struct *work) +{ + struct apple_isp *isp = container_of(work, struct apple_isp, resume_work); + + mutex_lock(&isp->video_lock); + isp->fw_sleep_stopped = false; + mutex_unlock(&isp->video_lock); + apple_isp_video_resume(isp); +} + static int apple_isp_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -647,6 +659,7 @@ static int apple_isp_probe(struct platform_device *pdev) } mutex_init(&isp->video_lock); + INIT_WORK(&isp->resume_work, apple_isp_resume_work); spin_lock_init(&isp->buf_lock); init_waitqueue_head(&isp->wait); INIT_LIST_HEAD(&isp->gc); @@ -723,6 +736,7 @@ static void apple_isp_remove(struct platform_device *pdev) { struct apple_isp *isp = platform_get_drvdata(pdev); + cancel_work_sync(&isp->resume_work); apple_isp_remove_video(isp); /* Stop resident firmware before freeing what it may still use. */ apple_isp_firmware_halt(isp); @@ -954,6 +968,9 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .fw_restart = true, + /* As on macOS, the firmware first starts when a client opens the camera. */ + .deferred_start = true, .meta_exposure = true, }; @@ -1244,9 +1261,43 @@ static __maybe_unused int apple_isp_runtime_resume(struct device *dev) return 0; } +/* + * Stop resident firmware. For a system sleep it starts again on resume where + * it can; otherwise, or for hibernation, the camera is gone until the next + * boot. + */ +static void apple_isp_stop_resident(struct apple_isp *isp, bool sleep) +{ + bool restart = sleep && isp->hw->fw_restart; + + if (!isp->hw->resident_fw) + return; + + mutex_lock(&isp->video_lock); + if (isp->fw_state == ISP_FW_RUNNING) { + if (restart) + dev_dbg(isp->dev, "stopping the firmware for system sleep\n"); + else + dev_warn(isp->dev, "stopping the firmware for %s, the camera is unavailable until the next boot\n", + sleep ? "system sleep" : "hibernation"); + isp->fw_sleep_stopped = restart; + } + apple_isp_firmware_halt(isp); + /* Hibernation resumes in another boot, where that memory isn't ours. */ + if (!restart) + isp->fw_state = ISP_FW_DEAD; + mutex_unlock(&isp->video_lock); +} + static __maybe_unused int apple_isp_suspend(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + bool power_lost = isp->hw->resident_fw && pm_suspend_via_firmware(); + + /* A stream that couldn't restart on the other side keeps the system up. */ + if (power_lost && !isp->hw->fw_restart && + test_bit(ISP_STATE_STREAMING, &isp->state)) + return -EBUSY; /* We must restore V4L2 context on system resume. If we were streaming * before, we (essentially) stop streaming and start streaming again. @@ -1254,9 +1305,13 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) apple_isp_video_suspend(isp); /* - * Resident firmware stays up, idle, across suspend-to-idle and - * suspend-to-RAM: its domains stay on, see apple_isp_attach_genpd(). + * Resident firmware stays up, idle, across suspend-to-idle: its domains + * stay on, see apple_isp_attach_genpd(). A sleep that powers the SoC down + * takes it anyway, so stop it first. */ + if (power_lost) + apple_isp_stop_resident(isp, true); + return 0; } @@ -1272,13 +1327,7 @@ static __maybe_unused int apple_isp_freeze(struct device *dev) * that started it. Stop it cleanly instead; if it ran, the camera is * then unavailable until the next system boot. */ - if (isp->hw->resident_fw) { - mutex_lock(&isp->video_lock); - if (isp->fw_state == ISP_FW_RUNNING) - dev_warn(dev, "stopping the firmware for hibernation, the camera is unavailable until the next boot\n"); - apple_isp_firmware_halt(isp); - mutex_unlock(&isp->video_lock); - } + apple_isp_stop_resident(isp, false); return 0; } @@ -1287,6 +1336,16 @@ static __maybe_unused int apple_isp_resume(struct device *dev) { struct apple_isp *isp = dev_get_drvdata(dev); + /* + * Firmware stopped for the sleep starts again in process context: its + * handshake here held up the whole resume. A stream that ran across the + * sleep restarts from a freezable work item, once tasks thaw; without one + * the firmware starts at the next stream. + */ + if (isp->fw_sleep_stopped) { + queue_work(system_freezable_wq, &isp->resume_work); + return 0; + } apple_isp_video_resume(isp); return 0; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 66fa3e1264576..94081701d9a5a 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -241,12 +242,15 @@ struct apple_isp_hw { bool fw_restore; /* - * The firmware can be started only once per system boot, so once - * started it is kept running while the driver is bound. It starts at - * probe, or at the first stream with deferred_start. + * The firmware isn't started for each stream: once started it is kept + * running while the driver is bound. It starts at probe, or at the + * first stream with deferred_start. */ bool resident_fw; + /* Resident firmware that starts again after a stop, from its own memory */ + bool fw_restart; + /* * Start the firmware, and detect the camera, at the first stream * instead of at probe, so the ISP stays off until then. The firmware @@ -393,6 +397,10 @@ struct apple_isp { u64 heap_base; u64 heap_top; u64 data_top; /* 0 unless the bootloader loads the data */ + /* the coprocessor's reset vector at the first start */ + u64 entry; + /* the coprocessor has run since probe */ + bool started; } fw; /* changes under video_lock once the video device is registered */ enum isp_fw_state fw_state; @@ -401,6 +409,11 @@ struct apple_isp { /* configurations the firmware has, once a deferred start detected them */ u32 num_fw_presets; + /* stopped for system sleep, to start again on resume */ + bool fw_sleep_stopped; + /* restarts a stream that ran across that sleep, see apple_isp_resume() */ + struct work_struct resume_work; + struct isp_surf *ipc_surf; struct isp_surf *extra_surf; struct isp_surf *data_surf; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 302b18cd189c6..98125b804a7ef 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -7,9 +7,13 @@ #include #include #include +#include +#include #include +#include #include #include +#include #include #include @@ -29,6 +33,12 @@ #define ISP_COPROC_IN_WFI 0x3 +/* the waits for firmware that has been told to power down */ +#define ISP_STOP_KEY_TRIES 20 +#define ISP_STOP_KEY_MS 10 +#define ISP_STOP_WFI_POLL_US (10 * USEC_PER_MSEC) +#define ISP_STOP_WFI_US (190 * USEC_PER_MSEC) + static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) { return readl(isp->coproc + reg); @@ -254,9 +264,12 @@ static void isp_disable_irq(struct apple_isp *isp) if (isp->hw->mbox_irq_route || isp->hw->mbox_extra_enables) { isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, 0x0); isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, 0x0); + if (isp->hw->mbox_irq_route) + isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, 0x0); } free_irq(isp->irq, isp); isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ + fsleep(10); } static int isp_enable_irq(struct apple_isp *isp) @@ -415,6 +428,39 @@ static void isp_restore_fw_memory(struct apple_isp *isp) clear_highpage(tail->pages[i]); } +/* + * The coprocessor starts at its reset vector, which the bootloader sets and + * which bit 0 latches. Note it at the first start; where a power-off cleared + * it, set it again. + */ +static int isp_coproc_entry(struct apple_isp *isp) +{ + struct isp_firmware *fw = &isp->fw; + u64 rvbar = readq(isp->coproc + ISP_COPROC_RVBAR); + + if (!fw->started) { + fw->entry = rvbar & ~1ULL; + dev_dbg(isp->dev, "firmware reset vector %#llx\n", rvbar); + return 0; + } + + if (!fw->entry || rvbar == (fw->entry | 1)) + return 0; + + if (rvbar & 1) { + isp_err(isp, "reset vector %#llx latched, not the firmware's %#llx\n", + rvbar, fw->entry); + return -EIO; + } + + writeq(fw->entry | 1, isp->coproc + ISP_COPROC_RVBAR); + dev_dbg(isp->dev, "reset vector was %#llx, set to %#llx\n", rvbar, + readq(isp->coproc + ISP_COPROC_RVBAR)); + + return (readq(isp->coproc + ISP_COPROC_RVBAR) & ~1ULL) == fw->entry ? + 0 : -EIO; +} + static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { /* Only a coprocessor that was started has to settle and quiesce */ @@ -429,7 +475,13 @@ static void isp_firmware_shutdown_stage1(struct apple_isp *isp) dev_warn(isp->dev, "coproc not in WFI before power-down (status 0x%x)\n", val); - isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); + /* + * macOS powers T8140's ISP off from WFI with the run bit still set. The sleep + * engine's own restore of the ISP in a wake that kept DRAM up depends on it: + * with the bit cleared that wake stalls until the SoC's watchdog resets it. + */ + if (isp->hw->gen != ISP_GEN_T8140) + isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); if (started) { /* The request bit alone asks for it, and reads back once done */ @@ -477,6 +529,12 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) if (err < 0) goto shutdown; + if (isp->hw->fw_restart) { + err = isp_coproc_entry(isp); + if (err < 0) + goto shutdown; + } + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); isp_gpio_write32(isp, ISP_GPIO_1, 0x0); isp_gpio_write32(isp, ISP_GPIO_2, 0x0); @@ -490,6 +548,7 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); isp_coproc_write32(isp, isp_coproc_control(isp), 0x10); + isp->fw.started = true; /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { @@ -887,10 +946,92 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) return 0; } +/* Wait for the word of its own memory the firmware sets once it has stopped. */ +static void isp_wait_stop_key(struct apple_isp *isp, u64 addr, u32 key) +{ + u64 iova = isp_fw_iova(isp, addr); + phys_addr_t phys = 0; + u32 *word = NULL; + + if (iova < isp_fw_iova(isp, isp->fw.heap_top)) + phys = iommu_iova_to_phys(isp->domain, iova); + if (phys) + word = memremap(phys, sizeof(*word), MEMREMAP_WC); + if (!word) { + dev_warn(isp->dev, "stop key at 0x%llx isn't in firmware memory\n", + addr); + return; + } + + for (int i = 0; i < ISP_STOP_KEY_TRIES && READ_ONCE(*word) != key; i++) + msleep(ISP_STOP_KEY_MS); + + if (READ_ONCE(*word) != key) + dev_warn(isp->dev, "stop key at 0x%llx reads 0x%x, not 0x%x\n", + addr, READ_ONCE(*word), key); + + memunmap(word); +} + +/* + * Stop firmware that starts cold next time. CISP_CMD_SUSPEND is for firmware + * that resumes: it isn't followed by a cold start. + */ +static int isp_stop_firmware_cold(struct apple_isp *isp) +{ + struct cmd_stop_key stop_key = {}; + ktime_t start = ktime_get(); + u32 status; + int err; + + /* Without the key the rest of the stop still works. */ + if (isp_cmd_stop_key_get(isp, &stop_key)) + stop_key.addr = 0; + + err = isp_cmd_stop_cold(isp, 0); + if (err) { + isp_err(isp, "isp_cmd_stop_cold() failed\n"); + return err; + } + + /* The firmware answers the power-down in this word, not on the ring. */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + err = isp_cmd_power_down(isp); + if (err) { + isp_err(isp, "isp_cmd_power_down() failed\n"); + return err; + } + + if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + if (stop_key.addr) + isp_wait_stop_key(isp, stop_key.addr, stop_key.key); + + /* Its domains go off next, so its CPU should be idle by now. */ + if (readl_poll_timeout(isp->coproc + ISP_COPROC_STATUS_T8140, status, + status & ISP_COPROC_IN_WFI, ISP_STOP_WFI_POLL_US, + ISP_STOP_WFI_US)) + dev_warn(isp->dev, "coprocessor not in WFI (status: 0x%x)\n", + status); + else + dev_dbg(isp->dev, "firmware stopped in %lld ms (key at 0x%llx, status 0x%x)\n", + ktime_ms_delta(ktime_get(), start), stop_key.addr, + status); + + return 0; +} + static int isp_stop_command_processor(struct apple_isp *isp) { int res, ret; + if (isp->hw->fw_restart) + return isp_stop_firmware_cold(isp); + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); /* @@ -1085,26 +1226,48 @@ static void isp_firmware_shutdown(struct apple_isp *isp) /* * Stop the coprocessor before releasing any memory the firmware - * uses, also when it did not acknowledge the suspend request. + * uses, also when it did not acknowledge the stop request. */ if (isp_stop_command_processor(isp)) - dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); + dev_warn(isp->dev, "firmware did not stop, stopping the coprocessor\n"); isp_disable_irq(isp); isp_firmware_stop(isp); } +/* + * After a restart, the firmware must find the camera found at probe. The + * query also shows that it takes commands. + */ +static int isp_check_restarted(struct apple_isp *isp) +{ + struct cmd_config_get args = {}; + int err; + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + if (min_t(u32, args.num_channels, ISP_MAX_CHANNELS) != isp->num_channels) { + isp_err(isp, "firmware found %u channels after a restart, %d before\n", + args.num_channels, isp->num_channels); + return -ENODEV; + } + + return 0; +} + /* * Resident firmware (hw->resident_fw) is loaded by the bootloader and - * does not start again once it has been stopped: after CISP_CMD_SUSPEND, - * a coprocessor reset and a power cycle of its domains it never completes - * the first handshake. It is booted once, at probe or with - * deferred_start at the first stream, and runs until the - * driver is unbound, probe fails or the system hibernates, also across - * suspend; after it has stopped the device stays unusable until the next - * system boot. + * changes its own memory as it runs. It is booted at probe, or with + * deferred_start at the first stream, and runs until the driver is unbound, + * probe fails, the system hibernates or a sleep powers the SoC down; + * suspend-to-idle keeps it running. With hw->fw_restart it starts again, and + * rebuilds that memory itself; other stopped firmware stays down until the + * next system boot. */ int apple_isp_firmware_boot(struct apple_isp *isp) { + bool restart = isp->hw->resident_fw && isp->fw.started; int err; switch (isp->fw_state) { @@ -1131,10 +1294,16 @@ int apple_isp_firmware_boot(struct apple_isp *isp) } err = isp_firmware_boot(isp); + if (!err && restart) { + err = isp_check_restarted(isp); + if (err) + isp_firmware_shutdown(isp); + } if (err) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); pm_runtime_put_sync(isp->dev); - if (isp->hw->resident_fw) + /* Firmware that starts again can be tried again at the next open. */ + if (isp->hw->resident_fw && !isp->hw->fw_restart) isp->fw_state = ISP_FW_DEAD; return err; } @@ -1161,9 +1330,17 @@ void apple_isp_firmware_shutdown(struct apple_isp *isp) if (isp->fw_state == ISP_FW_RUNNING) synchronize_irq(isp->irq); isp_free_released_surfaces(isp); + + /* + * Firmware that starts again goes down with the stream, as macOS's camera close + * takes it, and boots again for the next one. A system sleep then finds it off: + * stopped on the way into a sleep that keeps DRAM up, the SoC doesn't wake. + */ + if (isp->hw->fw_restart) + apple_isp_firmware_halt(isp); } -/* Stop the firmware; resident firmware cannot be started again. */ +/* Stop the firmware; resident firmware without hw->fw_restart stays down. */ void apple_isp_firmware_halt(struct apple_isp *isp) { /* Only a capture services the watchdog, but be sure. */ @@ -1175,5 +1352,6 @@ void apple_isp_firmware_halt(struct apple_isp *isp) isp_firmware_shutdown(isp); pm_runtime_put_sync(isp->dev); isp->ipc_stale = NULL; - isp->fw_state = isp->hw->resident_fw ? ISP_FW_DEAD : ISP_FW_OFF; + isp->fw_state = isp->hw->resident_fw && !isp->hw->fw_restart ? + ISP_FW_DEAD : ISP_FW_OFF; } diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 81964ab1916c9..f6df02e6fe8df 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -611,6 +611,8 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) { struct apple_isp *isp = vb2_get_drv_priv(q); + /* A restart still due from a sleep has nothing left to restart. */ + clear_bit(ISP_STATE_SLEEPING, &isp->state); apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); v4l2_ctrl_grab(isp->exposure_priority, false); diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index c4f9329dbc01c..d6ccf8a3970a2 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -9,8 +9,16 @@ struct macsmc_test_context { int result; unsigned int calls; bool require_sram_lock; + /* What the sleep tests saw, in order: keys read, values written, notices. */ + u32 seen[8]; + unsigned int seen_count; + int ap_power_result; + u64 record; }; +#define MACSMC_TEST_AP_IDLE 0 +#define MACSMC_TEST_AP_ON 1 + static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -18,6 +26,9 @@ static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); ctx->calls++; + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = cmd == SMC_MSG_WRITE_KEY ? + get_unaligned((u32 *)ctx->shmem) : arg; if (ret_data && ctx->result >= 0) *ret_data = ctx->data; return ctx->result; @@ -199,6 +210,165 @@ static void macsmc_key_info_lock_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); } +static int macsmc_test_ap_power(struct apple_smc *smc, bool on) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + /* The answer needs a worker that may be waiting for this mutex. */ + KUNIT_EXPECT_FALSE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = on ? MACSMC_TEST_AP_ON : MACSMC_TEST_AP_IDLE; + return ctx->ap_power_result; +} + +static void macsmc_sleep_order_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + apple_smc_sleep_warn(&ctx->smc); + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + + /* As in the trace, values included. */ + KUNIT_ASSERT_EQ(test, ctx->seen_count, 7U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[5], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[6], SMC_KEY(waka)); +} + +static void macsmc_test_outbox_poll(struct apple_smc *smc, const char *after, + unsigned int min_ms, unsigned int quiet_ms) +{ +} + +static void macsmc_sleep_warn_late_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); + ctx->result = 8; + + /* A sleep that cuts the debug link: slpw comes at the noirq suspend, before slpa. */ + ctx->smc.sleep_warn_late = true; + KUNIT_EXPECT_EQ(test, apple_smc_sleep_enter(&ctx->smc), 0); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); +} + +static void macsmc_sleep_refused_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->ap_power_result = -ETIME; + ctx->result = 4; + + /* No answer: refuse, undo the notice and leave commands working. */ + KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), -ETIME); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_ON); +} + +static void macsmc_notice_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + ctx->result = 8; + + /* The test trigger writes no MBSE value: those announce a real sleep. */ + KUNIT_EXPECT_EQ(test, apple_smc_notice_idle(&ctx->smc), 0); + apple_smc_notice_on(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(MBSW)); +} + +/* MBSW reads ctx->record, MSLD reads closed. */ +static int macsmc_test_lid_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = key; + memset(buf, 0, size); + if (key == SMC_KEY(MBSW) && size == sizeof(ctx->record)) + put_unaligned_le64(ctx->record, buf); + else if (key == SMC_KEY(MSLD)) + *(u8 *)buf = 1; + return size; +} + +static void macsmc_lid_wake_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_read, macsmc_test_lid_read); + ctx->result = 4; + + /* The lid alone: MSLD is read last, after macOS's own order. */ + ctx->record = BIT_ULL(10); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 5U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MSLD)); + + /* The lid with anything else is a normal wake: the lid isn't read. */ + ctx->seen_count = 0; + ctx->record = BIT_ULL(10) | BIT_ULL(0); + apple_smc_sleep_wake(&ctx->smc); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 4U); +} + +static void macsmc_wake_reason_test(struct kunit *test) +{ + char names[128]; + + KUNIT_EXPECT_EQ(test, apple_smc_wake_reason_names(0, names, sizeof(names)), 0); + KUNIT_EXPECT_STREQ(test, names, ""); + + /* A bit with no name is left out. */ + apple_smc_wake_reason_names(BIT_ULL(23) | BIT_ULL(42) | BIT_ULL(1), names, sizeof(names)); + KUNIT_EXPECT_STREQ(test, names, "wifibt USB-C_plug"); +} + +static void macsmc_port_reasons_test(struct kunit *test) +{ + /* The debug link's cut: USB-C_plug, acdetach, or both. */ + KUNIT_EXPECT_TRUE(test, apple_smc_port_reasons_only(BIT_ULL(42))); + KUNIT_EXPECT_TRUE(test, apple_smc_port_reasons_only(BIT_ULL(41) | BIT_ULL(42))); + /* Anything else, with it or alone, is a real wake. */ + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(0)); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(42) | BIT_ULL(0))); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(42) | BIT_ULL(10))); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(23))); + KUNIT_EXPECT_FALSE(test, apple_smc_port_reasons_only(BIT_ULL(43))); +} + static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_key_info_lock_test), KUNIT_CASE(macsmc_key_index_test), @@ -209,6 +379,13 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_request_bounds_test), KUNIT_CASE(macsmc_sram_value_test), KUNIT_CASE(macsmc_key_info_test), + KUNIT_CASE(macsmc_sleep_order_test), + KUNIT_CASE(macsmc_sleep_warn_late_test), + KUNIT_CASE(macsmc_sleep_refused_test), + KUNIT_CASE(macsmc_notice_test), + KUNIT_CASE(macsmc_lid_wake_test), + KUNIT_CASE(macsmc_wake_reason_test), + KUNIT_CASE(macsmc_port_reasons_test), {} }; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 927ddaa00e0df..1d0e4d58b68a6 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -22,7 +22,10 @@ #include #include #include +#include #include +#include +#include #include #define SMC_ENDPOINT 0x20 @@ -120,6 +123,14 @@ static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) memcpy_fromio(buf, smc->shmem.iomem, size); } +/* In a noirq phase the mailbox's interrupt is off, so its reply has to be fetched. */ +static bool apple_smc_reply_arrived(struct apple_smc *smc) +{ + apple_rtkit_poll(smc->rtk); + + return completion_done(&smc->cmd_done); +} + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -136,6 +147,9 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return -EIO; if (smc->atomic_mode) return -EIO; + /* Between the AP power notices the firmware expects no commands. */ + if (smc->ap_notified) + return -EAGAIN; reinit_completion(&smc->cmd_done); @@ -152,7 +166,17 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, return ret; } - if (wait_for_completion_timeout(&smc->cmd_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) <= 0) { + if (smc->polled) { + bool done; + + ret = read_poll_timeout(apple_smc_reply_arrived, done, done, 100, + SMC_TIMEOUT_MS * USEC_PER_MSEC, false, smc); + } else { + ret = wait_for_completion_timeout(&smc->cmd_done, + msecs_to_jiffies(SMC_TIMEOUT_MS)) > 0 ? + 0 : -ETIMEDOUT; + } + if (ret) { dev_err(smc->dev, "Command timed out (%llx)", msg); return -ETIMEDOUT; } @@ -598,6 +622,10 @@ static int apple_smc_probe(struct platform_device *pdev) ARRAY_SIZE(apple_smc_telemetry_devs), NULL, 0, NULL); + ret = devm_device_init_wakeup(dev); + if (ret) + return ret; + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -634,10 +662,332 @@ static const struct of_device_id apple_smc_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_smc_of_match); +/* + * The SMC keeps running through a firmware-driven suspend. It is told first and + * stops last, and on a wake it hears first. In order, measured on T8140 and + * unverified on other SoCs: MBSE = slpw before any other coprocessor is told; + * MBSE = slpa and the AP power notice after all the others; on a wake the notice, + * MBSE and MBSW read, MBSE = waka. The noirq phases give that order whatever the + * device order is, so these commands poll. What the values mean is inferred + * from their letters. + * + * SMC.OutboxNotEmpty wakes the SoC while anything waits in the outbox. An event that arrives + * during the sleep's entry (the lid, a key) is left there, so the sleep returns at once. + * + * The debug link's cut, on a debug cable only, makes the SMC see a cable pulled and plugged + * back. With the cut after slpw, the SMC woke the AP right after the commit, for USB-C_plug + * or acdetach, in every such sleep (J700, 223 of 223), even with its outbox quiet for 500 ms + * before the notice. A sleep that cuts the link warns the SMC only after the cut. + */ + +/* The gaps macOS leaves before the AP idle notice and after it. */ +#define SMC_SLEEP_ARM_MS 6 +#define SMC_SLEEP_IDLE_MS 30 +/* The shortest gap macOS left between slpw and slpa, in two traced sleeps. */ +#define SMC_SLEEP_WARN_MS 770 + +/* After the debug link's cut. Its port and charger events keep coming for over a second. */ +#define SMC_OUTBOX_QUIET_MS 500 +#define SMC_OUTBOX_SETTLE_MS 5000 + +static int apple_smc_ap_power(struct apple_smc *smc, bool on) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_ap_power, smc, on); + + return smc->polled ? apple_rtkit_ap_power_noirq(smc->rtk, on) : + apple_rtkit_ap_power(smc->rtk, on); +} + +static void apple_smc_set_ap_notified(struct apple_smc *smc, bool notified) +{ + guard(mutex)(&smc->mutex); + + smc->ap_notified = notified; +} + +static void apple_smc_set_polled(struct apple_smc *smc, bool polled) +{ + guard(mutex)(&smc->mutex); + + smc->polled = polled; +} + +static void apple_smc_write_mbse(struct apple_smc *smc, smc_key value) +{ + int ret; + + ret = apple_smc_write_u32(smc, SMC_KEY(MBSE), value); + if (ret) + dev_err(smc->dev, "Failed to issue MBSE = %p4ch: %d\n", &value, ret); +} + +static void apple_smc_sleep_warn(struct apple_smc *smc) +{ + apple_smc_write_mbse(smc, SMC_KEY(slpw)); + smc->sleep_warned = true; +} + +static int apple_smc_notice_idle(struct apple_smc *smc) +{ + int ret; + + /* + * Commands fail from here on. The mutex isn't held across the notice: the + * answer comes through the same worker as the notifications, and those can + * be waiting for this mutex. + */ + apple_smc_set_ap_notified(smc, true); + + ret = apple_smc_ap_power(smc, false); + if (ret) { + /* Taken and not answered, perhaps, so undo it. */ + apple_smc_ap_power(smc, true); + apple_smc_set_ap_notified(smc, false); + dev_err(smc->dev, "SMC did not take the sleep notice: %d\n", ret); + } else { + dev_dbg(smc->dev, "AP idle notice answered\n"); + } + + return ret; +} + +static int apple_smc_sleep_commit(struct apple_smc *smc) +{ + int ret; + + apple_smc_write_mbse(smc, SMC_KEY(slpa)); + /* Device interrupts are off here. */ + mdelay(SMC_SLEEP_ARM_MS); + + ret = apple_smc_notice_idle(smc); + if (!ret) + mdelay(SMC_SLEEP_IDLE_MS); + + return ret; +} + +/* MBSW's bits, as smc-pmu's event_name properties number them on T8140. Other SoCs differ. */ +static const struct { + u8 bit; + const char *name; +} apple_smc_wake_reasons[] = { + { 0, "pwrbtn" }, { 9, "ipd_touch" }, { 10, "lid" }, { 16, "ipd_kbd" }, + { 18, "charger" }, { 23, "wifibt" }, { 40, "acattach" }, { 41, "acdetach" }, + { 42, "USB-C_plug" }, { 43, "USB2_plug" }, { 44, "USB2_wake" }, { 45, "aupo" }, + { 48, "rtc" }, +}; + +static int apple_smc_wake_reason_names(u64 record, char *buf, size_t size) +{ + int i, len = 0; + + buf[0] = 0; + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (record & BIT_ULL(apple_smc_wake_reasons[i].bit)) + len += scnprintf(buf + len, size - len, "%s%s", len ? " " : "", + apple_smc_wake_reasons[i].name); + + return len; +} + +static void apple_smc_notice_on(struct apple_smc *smc) +{ + u8 record[8] = {}; + char names[128]; + u32 event = 0; + int ret; + + ret = apple_smc_ap_power(smc, true); + if (ret) + dev_err(smc->dev, "SMC did not take the wake notice: %d\n", ret); + apple_smc_set_ap_notified(smc, false); + + ret = apple_smc_read_u32(smc, SMC_KEY(MBSE), &event); + if (ret) + dev_warn(smc->dev, "Failed to read MBSE after a wake: %d\n", ret); + ret = apple_smc_read(smc, SMC_KEY(MBSW), record, sizeof(record)); + if (ret < 0) + dev_warn(smc->dev, "Failed to read MBSW after a wake: %d\n", ret); + + smc->wake_record = get_unaligned_le64(record); + apple_smc_wake_reason_names(smc->wake_record, names, sizeof(names)); + dev_info(smc->dev, "wake record: MBSE %#x, MBSW %8ph%s%s\n", event, record, + names[0] ? ": " : "", names); +} + +/* MBSW's bit for a wake reason, by its name, or -1. */ +static int apple_smc_wake_reason_bit(const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_smc_wake_reasons); i++) + if (!strcmp(apple_smc_wake_reasons[i].name, name)) + return apple_smc_wake_reasons[i].bit; + + return -1; +} + +/* + * Woken for the lid alone, which the SMC reports for the lid opening and for it closing: + * tell the sleep driver if it's closed now. + */ +static void apple_smc_lid_wake(struct apple_smc *smc) +{ + int bit = apple_smc_wake_reason_bit("lid"); + u8 lid = 0; + int ret; + + if (bit < 0 || smc->wake_record != BIT_ULL(bit)) + return; + + ret = apple_smc_read_u8(smc, SMC_KEY(MSLD), &lid); + if (ret < 0) { + dev_warn(smc->dev, "Failed to read the lid after a wake: %d\n", ret); + return; + } + dev_info(smc->dev, "woken for the lid, which is %s\n", lid ? "closed" : "open"); + if (lid) + apple_sleep_lid_closed_wake(); +} + +/* Only what a cable pulled and plugged back reports: what the debug link's cut makes. */ +static bool apple_smc_port_reasons_only(u64 record) +{ + static const char * const names[] = { "acattach", "acdetach", "USB-C_plug" }; + u64 port = 0; + int i, bit; + + for (i = 0; i < ARRAY_SIZE(names); i++) { + bit = apple_smc_wake_reason_bit(names[i]); + if (bit >= 0) + port |= BIT_ULL(bit); + } + + return record && !(record & ~port); +} + +static void apple_smc_sleep_wake(struct apple_smc *smc) +{ + apple_smc_notice_on(smc); + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + apple_smc_lid_wake(smc); + if (apple_smc_port_reasons_only(smc->wake_record)) + apple_sleep_debug_link_cut_wake(); + /* A dark wake isn't one: with wakeup_count in use, it would abort the sleep after it. */ + if (smc->wake_record && !pm_dark_wake()) + pm_wakeup_event(smc->dev, 0); +} + +static int apple_smc_prepare(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (!pm_suspend_via_firmware()) + return 0; + if (apple_sleep_debug_link_up()) + smc->sleep_warn_late = true; + else + apple_smc_sleep_warn(smc); + + return 0; +} + +/* Warned, and a refusal came before the notice: tell it the sleep is off. */ +static void apple_smc_complete(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + smc->sleep_warn_late = false; + if (smc->sleep_warned) { + apple_smc_write_mbse(smc, SMC_KEY(waka)); + smc->sleep_warned = false; + } +} + +/* + * For at least @min_ms, then until the outbox is quiet for @quiet_ms. The events go to their + * handlers as usual, through the RTKit worker. + */ +static void apple_smc_outbox_poll(struct apple_smc *smc, const char *after, + unsigned int min_ms, unsigned int quiet_ms) +{ + unsigned int ms, quiet = 0, count = 0, last = 0; + int ret; + + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_outbox_poll, smc, after, min_ms, quiet_ms); + + for (ms = 0; ms < SMC_OUTBOX_SETTLE_MS && (ms < min_ms || quiet < quiet_ms); ms++) { + ret = apple_rtkit_poll(smc->rtk); + if (ret > 0) { + count += ret; + last = ms; + quiet = 0; + } else { + quiet++; + } + usleep_range(1000, 1200); + } + /* A debug cable only. */ + dev_info(smc->dev, "%s: %u messages in %u ms, the last at %u ms%s\n", after, count, ms, + last, ms >= SMC_OUTBOX_SETTLE_MS ? ", still busy" : ""); +} + +/* The noirq suspend's part, polled: the cut, a late warning, the commit's notices. */ +static int apple_smc_sleep_enter(struct apple_smc *smc) +{ + if (apple_sleep_debug_link_cut()) + apple_smc_outbox_poll(smc, "debug link cut", 0, SMC_OUTBOX_QUIET_MS); + if (smc->sleep_warn_late) { + smc->sleep_warn_late = false; + apple_smc_sleep_warn(smc); + apple_smc_outbox_poll(smc, "late sleep warning", SMC_SLEEP_WARN_MS, 0); + } + + return apple_smc_sleep_commit(smc); +} + +static int apple_smc_suspend_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + int ret; + + if (!smc->sleep_warned && !smc->sleep_warn_late) + return 0; + + apple_smc_set_polled(smc, true); + ret = apple_smc_sleep_enter(smc); + apple_smc_set_polled(smc, false); + + return ret; +} + +static int apple_smc_resume_noirq(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + if (smc->ap_notified) { + apple_smc_set_polled(smc, true); + apple_smc_sleep_wake(smc); + apple_smc_set_polled(smc, false); + } + + return 0; +} + +static const struct dev_pm_ops apple_smc_pm_ops = { + .prepare = pm_sleep_ptr(apple_smc_prepare), + .complete = pm_sleep_ptr(apple_smc_complete), + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_smc_suspend_noirq, apple_smc_resume_noirq) +}; + static struct platform_driver apple_smc_driver = { .driver = { .name = "macsmc", .of_match_table = apple_smc_of_match, + .pm = pm_sleep_ptr(&apple_smc_pm_ops), }, .probe = apple_smc_probe, }; diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index cadf0152d4e9b..87afa3627a9ce 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o wow.o mt7932-fullmac-$(CONFIG_MT7932_AWDL) += awdl.o mt7932-fullmac-$(CONFIG_MT7932_AWDL_KUNIT_TEST) += awdl-test.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o @@ -10,5 +10,6 @@ mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += move-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += wow-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 2c8313d2f523a..8601085b299d7 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -564,7 +564,8 @@ static void mt_cal_work(struct work_struct *work) int ret; mutex_lock(&m->command_mutex); - if (!READ_ONCE(m->rf_ready)) + /* Stopped for a sleep, the requests wait for the restart. */ + if (!READ_ONCE(m->rf_ready) || READ_ONCE(m->hif_suspended)) goto unlock; ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) @@ -633,6 +634,7 @@ int mt_register_interface(struct mt7932 *m) INIT_WORK(&m->connect_work, mt_connect_work); INIT_WORK(&m->disconnect_work, mt_disconnect_work); INIT_WORK(&m->power_work, mt_power_work); + INIT_WORK(&m->wow_work, mt_wow_work); init_completion(&m->assoc_start); init_completion(&m->assoc_done); init_completion(&m->disconnect_done); diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index ed08deb9a6181..f16fb04259c7a 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -55,6 +55,7 @@ void mt_stop_host(struct mt7932 *m) flush_work(&m->disconnect_work); cancel_work_sync(&m->cal_work); cancel_work_sync(&m->power_work); + cancel_work_sync(&m->wow_work); cancel_delayed_work_sync(&m->scan_timeout_work); cancel_work_sync(&m->scan_finish_work); /* With all producers joined, this is the sole owner of any remainder. */ diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 64ec84219d4a2..5d74b7ac234bd 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -7,6 +7,7 @@ #include #include "mt7932.h" +#include "wow.h" #include #include @@ -44,7 +45,8 @@ static void mt_shutdown(struct pci_dev *pdev); bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && - !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed); + !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed) && + !READ_ONCE(m->hif_suspended); } int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name) @@ -130,6 +132,30 @@ static void mt_rx_post(struct mt7932_ring *q, unsigned int index) d->control = cpu_to_le32(MT7932_RX_CAPACITY << 16); } +/* + * The host interface control command and event, the same ID both ways. The command is + * type, direction, stop, suspend in 36 bytes; the event's body is type, TX traffic, RX + * traffic, suspend, with at least as many bytes. + */ +#define MT7932_CID_HIF_CTRL 0xf6 +#define MT7932_HIF_PCIE 2 +#define MT7932_HIF_IDLE 2 +#define MT7932_HIF_LENGTH 36 + +/* response_lock held. */ +static void mt_hif_event(struct mt7932 *m, const struct mt7932_event *event) +{ + const u8 *body = event->packet + 36; + + if (!m->hif_waiting || event->length < 36 + MT7932_HIF_LENGTH || + body[0] != MT7932_HIF_PCIE || !!body[3] != m->hif_suspending) + return; + m->hif_waiting = false; + m->hif_result = body[3] && (body[1] != MT7932_HIF_IDLE || body[2] != MT7932_HIF_IDLE) ? + -EBUSY : 0; + complete(&m->hif_done); +} + static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) { struct mt7932_event event; @@ -153,10 +179,18 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) return; } spin_lock_irqsave(&m->response_lock, flags); + if (event.eid == MT7932_CID_HIF_CTRL) { + mt_hif_event(m, &event); + goto unlock; + } if (event.eid == 0xee) { mt_link_event(m, &event); goto unlock; } + if (event.eid == MT7932_EID_WOW_REASON) { + mt_wow_reason_event(m, &event); + goto unlock; + } if (event.eid == 0x0d) { mt_scan_event(m, &event); goto unlock; @@ -374,6 +408,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) return -ESHUTDOWN; + /* With the host interface stopped only its own control may reach the BAR. */ + if (READ_ONCE(m->hif_gate) && !READ_ONCE(m->hif_waiting)) + return -ESHUTDOWN; if (q->used >= q->slots) { /* Command buffers can wrap only after every published descriptor has * retired. Never reclaim by elapsed time or by a reply alone. @@ -1102,6 +1139,8 @@ static ssize_t removal_retry_store(struct device *dev, } static DEVICE_ATTR_WO(removal_retry); +static struct pci_driver mt_driver; + static int mt_retire_dma(struct mt7932 *m) { int ret; @@ -1124,11 +1163,35 @@ static int mt_retire_dma(struct mt7932 *m) return ret; } +/* + * Removal waits forever for a reset that fails, which a recovery in a worker can't do. So stop + * here first, where a failure keeps the function bound. A radio stopped this way stays down + * until it's removed, and remove then has nothing left to wait for. + */ +static int mt_radio_stop(struct pci_dev *pdev) +{ + struct mt7932 *m; + int ret = 0; + + device_lock(&pdev->dev); + m = pdev->dev.driver == &mt_driver.driver ? pci_get_drvdata(pdev) : NULL; + if (m) { + mt_stop_host(m); + ret = mt_retire_dma(m); + if (ret) + dev_err(&pdev->dev, "restart refused: DMA still owned: %d\n", ret); + } + device_unlock(&pdev->dev); + + return ret; +} + static void mt_remove(struct pci_dev *pdev) { struct mt7932 *m = pci_get_drvdata(pdev); int ret; + apple_piodma_radio_set_stop(NULL); mt_stop_host(m); while ((ret = mt_retire_dma(m))) { WRITE_ONCE(m->removal_error, ret); @@ -1212,6 +1275,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) init_completion(&m->response); init_completion(&m->cal_response); init_completion(&m->reset_retry); + init_completion(&m->hif_done); mutex_init(&m->command_mutex); spin_lock_init(&m->response_lock); ret = mt_ownership(m); @@ -1247,6 +1311,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = device_create_file(&pdev->dev, &dev_attr_removal_retry); if (ret) dev_warn(&pdev->dev, "removal retry attribute unavailable: %d\n", ret); + apple_piodma_radio_set_stop(mt_radio_stop); return 0; } @@ -1260,21 +1325,192 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* command_mutex held; the firmware answers once its host interface has changed state. */ +static int mt_hif_ctrl(struct mt7932 *m, bool suspend) +{ + u8 body[MT7932_HIF_LENGTH] = { MT7932_HIF_PCIE, 1, suspend, suspend }; + unsigned long flags; + int ret; + + reinit_completion(&m->hif_done); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_suspending = suspend; + m->hif_result = -ETIMEDOUT; + m->hif_waiting = true; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt_request(m, MT7932_CID_HIF_CTRL, true, true, false, body, sizeof(body)); + if (!ret) + wait_for_completion_timeout(&m->hif_done, msecs_to_jiffies(1000)); + spin_lock_irqsave(&m->response_lock, flags); + m->hif_waiting = false; + if (!ret) + ret = m->hif_result; + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* Every published command and frame retired, ahead of the firmware's own stop. */ +static int mt_hif_tx_idle(struct mt7932 *m) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(500); + + do { + mt_data_clean(m); + mt_tx_clean(&m->tx[17]); + if (!READ_ONCE(m->tx[0].queued) && !m->tx[17].queued) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + return -EBUSY; +} + +/* + * Driver own again, DMA back on, then the firmware's host interface, and WoW + * off if the sleep had it. Rings stay as they were. + */ +static int mt_hif_restart(struct mt7932 *m) +{ + bool wow; + int ret; + + ret = mt_ownership(m); + if (ret) + return ret; + mt_write(m, 0x10188, 0xff); + if (mt_read(m, m->rx[0].reg) != lower_32_bits(m->rx[0].dma) || + mt_read(m, m->tx[17].reg) != lower_32_bits(m->tx[17].dma)) { + dev_err(&m->pdev->dev, "DMA rings lost across the sleep: RX0 %#x, TX17 %#x\n", + mt_read(m, m->rx[0].reg), mt_read(m, m->tx[17].reg)); + return -EIO; + } + mt_write(m, W + 0x208, m->hif_glo); + WRITE_ONCE(m->running, true); + mt_write(m, W + 0x204, m->irq_mask); + mutex_lock(&m->command_mutex); + wow = m->wow_armed; + ret = mt_hif_ctrl(m, false); + if (!ret) { + WRITE_ONCE(m->hif_gate, false); + ret = mt_wow_disarm(m); + if (ret) + WRITE_ONCE(m->hif_gate, true); + } + mutex_unlock(&m->command_mutex); + if (ret) { + /* Not left running into the next sleep. */ + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + mt_dma_stop(m); + return ret; + } + WRITE_ONCE(m->hif_suspended, false); + /* What arrived or was asked for while stopped, now that it can run. */ + if (READ_ONCE(m->cal_request_count)) + schedule_work(&m->cal_work); + if (wow) + schedule_work(&m->power_work); + if (READ_ONCE(m->interface_registered) && READ_ONCE(m->reg_pending)) + schedule_work(&m->startup_work); + if (m->netdev) + netif_device_attach(m->netdev); + return 0; +} + /* - * Stopping the firmware for sleep would take the whole remove and probe - * sequence, so leave it running across suspend-to-idle. A saved state keeps - * the PCI core from changing the function's power state or command register, - * and the awake path keeps the PCIe controller powered until resume. + * What macOS does at sleep. Without a WoWLAN configuration cfg80211 has already left + * the network; with one the firmware is given WoW first and stays associated. The firmware + * stops its host interface and is given ownership, and keeps running. PERST# then resets + * the link, not the firmware. + * + * Trouble here costs Wi-Fi at the wake, never the machine its sleep. + */ +static int mt_hif_suspend(struct mt7932 *m) +{ + int ret; + + if (!READ_ONCE(m->ready) || READ_ONCE(m->stopping) || READ_ONCE(m->hif_suspended)) + return 0; + /* New work waits from here, and what already runs finishes before the gate. */ + WRITE_ONCE(m->hif_suspended, true); + if (m->interface_registered) { + if (m->netdev) + netif_device_detach(m->netdev); + flush_work(&m->startup_work); + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + flush_work(&m->cal_work); + flush_work(&m->power_work); + mt_abort_scan(m->wiphy, &m->wdev); + ret = mt_scan_quiesce(m); + if (ret) + dev_warn(&m->pdev->dev, "scan still retiring at the sleep: %d\n", ret); + } + mutex_lock(&m->command_mutex); + mt_wow_arm(m); + WRITE_ONCE(m->hif_gate, true); + ret = mt_hif_tx_idle(m); + if (!ret) + ret = mt_hif_ctrl(m, true); + mutex_unlock(&m->command_mutex); + if (ret) + dev_err(&m->pdev->dev, "firmware did not stop its host interface: %d\n", ret); + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + synchronize_irq(pci_irq_vector(m->pdev, 0)); + mt_write(m, W + 0x204, 0); + m->hif_glo = mt_read(m, W + 0x208) & ~(BIT(1) | BIT(3)); + ret = mt_dma_stop(m); + if (ret) + dev_err(&m->pdev->dev, "DMA did not stop for the sleep: %d\n", ret); + mt_write(m, 0xe0010, 1); + ret = mt_poll(m, 0xe0010, BIT(2), BIT(2), 50000); + if (ret) + dev_err(&m->pdev->dev, "firmware did not take ownership for the sleep: %d\n", ret); + else + dev_dbg(&m->pdev->dev, "host interface stopped for the sleep, firmware kept\n"); + return 0; +} + +/* + * Stopping the firmware for sleep takes the whole remove and probe sequence, so + * leave it running across suspend-to-idle. A saved state keeps the PCI core from + * changing the function's power state or command register, and the awake path + * keeps the PCIe controller powered until resume. */ static int mt_suspend(struct device *dev) { if (!pm_suspend_no_platform()) - return -EOPNOTSUPP; + return mt_hif_suspend(pci_get_drvdata(to_pci_dev(dev))); pci_save_state(to_pci_dev(dev)); device_set_awake_path(dev); return 0; } +static int mt_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mt7932 *m = pci_get_drvdata(pdev); + unsigned long flags; + int ret; + + if (!READ_ONCE(m->hif_suspended)) + return 0; + /* As macOS does: a chip that no longer answers isn't touched further. */ + ret = pci_device_is_present(pdev) ? mt_hif_restart(m) : -ENODEV; + if (!ret) { + dev_dbg(dev, "back after the sleep, firmware kept\n"); + return 0; + } + dev_err(dev, "Wi-Fi didn't keep its firmware across the sleep: %d; restarting it\n", ret); + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + apple_piodma_radio_recover(); + return ret; +} + /* * Hibernation is refused. Its freeze phase does not call .suspend, so without * this the firmware would keep writing to memory while the image is saved, @@ -1291,6 +1527,7 @@ static int mt_freeze(struct device *dev) static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .resume = mt_resume, .freeze = mt_freeze, .poweroff = mt_freeze, }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 040206b8815c2..9cbb592320a6a 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -93,6 +93,12 @@ struct mt7932 { bool dma_owned, module_pinned; struct completion reset_retry; int removal_error; + /* The firmware keeps running across a deep sleep with its host interface stopped. */ + struct completion hif_done; + int hif_result; + u32 hif_glo; + /* hif_suspended stops new work; hif_gate then closes the BAR to all but the control. */ + bool hif_waiting, hif_suspending, hif_suspended, hif_gate; bool ram_config; bool runtime_epoch; bool runtime_cal_valid; @@ -158,6 +164,14 @@ struct mt7932 { /* Desired/submitted PS configuration uses command_mutex; TIM uses response_lock. */ u8 power_mode, power_applied; bool power_applied_valid, power_tim; + /* WoWLAN detectors cfg80211 asked for at this sleep; 0 leaves the network. */ + u8 wow_detect; + /* The firmware was given WoW at the sleep; command_mutex. */ + bool wow_armed; + /* A wake reason is awaited, and the one received; response_lock. */ + bool wow_report; + u8 wow_reason; + struct work_struct wow_work; struct completion assoc_start, assoc_done, disconnect_done; struct completion discovery_done; struct completion scan_done; @@ -347,5 +361,15 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); +int mt_power_submit(struct mt7932 *m, u8 mode); +#ifdef CONFIG_PM +extern const struct wiphy_wowlan_support mt_wowlan_support; +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow); +int mt_wow_resume(struct wiphy *wiphy); +#endif +void mt_wow_arm(struct mt7932 *m); +int mt_wow_disarm(struct mt7932 *m); +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event); +void mt_wow_work(struct work_struct *work); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index f3c1c42863c9c..03653ba97eede 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -5,7 +5,7 @@ * submission is accepted configuration, not proof that the radio is asleep. * cfg80211 implements GET_POWER_SAVE from the accepted requested setting. */ -static int mt_power_submit(struct mt7932 *m, u8 mode) +int mt_power_submit(struct mt7932 *m, u8 mode) { u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 04524891d36d0..f60651470b216 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -561,7 +561,8 @@ static void mt_startup_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); generation = m->reg_generation; *reg = m->reg_desired; - if (m->stopping || generation == m->reg_attempted) { + /* Stopped for a sleep, the restart runs this generation again. */ + if (m->stopping || READ_ONCE(m->hif_suspended) || generation == m->reg_attempted) { spin_unlock_irqrestore(&m->response_lock, flags); kfree(reg); return; @@ -809,6 +810,10 @@ static const struct cfg80211_ops mt_cfg_ops = { .del_station = mt_awdl_del_station, .change_station = mt_awdl_change_station, #endif +#ifdef CONFIG_PM + .suspend = mt_wow_suspend, + .resume = mt_wow_resume, +#endif }; int mt_register_regulatory_gate(struct mt7932 *m) @@ -846,6 +851,9 @@ int mt_register_regulatory_gate(struct mt7932 *m) wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_MFP_OPTIONAL); m->wiphy->max_scan_ssids = MT7932_SCAN_SSIDS; m->wiphy->max_scan_ie_len = 600; +#ifdef CONFIG_PM + m->wiphy->wowlan = &mt_wowlan_support; +#endif m->wiphy->reg_notifier = mt_regulatory_notify; /* World roaming may relax NO_IR after observing an AP beacon without * changing alpha2. Request the callback so firmware's CID0f domain is diff --git a/drivers/net/wireless/mediatek/mt7932/wow-test.c b/drivers/net/wireless/mediatek/mt7932/wow-test.c new file mode 100644 index 0000000000000..1301fa93b7bbb --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow-test.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "mt7932.h" +#include "wow.h" + +static bool mt_wow_test_zero(const u8 *p, size_t from, size_t to) +{ + size_t i; + + for (i = from; i < to; i++) + if (p[i]) + return false; + return true; +} + +static void mt_wow_detect_test(struct kunit *test) +{ + struct cfg80211_wowlan wow = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(NULL), 0); + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), MT7932_WOW_DETECT_MAGIC); + wow.magic_pkt = false; + wow.disconnect = true; + /* Losing the AP's beacons ends the association as surely as a deauth. */ + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x14); + wow.magic_pkt = true; + KUNIT_EXPECT_EQ(test, mt7932_wow_detect(&wow), 0x15); +} + +static void mt_wow_ctrl_test(struct kunit *test) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)228); + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, true, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_START); + KUNIT_EXPECT_EQ(test, body[1], 0x15); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), 0x63); + KUNIT_EXPECT_EQ(test, body[4], 0); /* scenario */ + KUNIT_EXPECT_EQ(test, body[5], 1); /* pins */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 6, 8)); + KUNIT_EXPECT_EQ(test, body[8], 3); /* GPIO */ + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + /* Interval, second wake signal, port counts and lists. */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); + + memset(body, 0xa5, sizeof(body)); + mt7932_wow_ctrl(body, false, 0x15); + KUNIT_EXPECT_EQ(test, body[0], MT7932_WOW_STOP); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 1, 5)); + KUNIT_EXPECT_EQ(test, body[5], 1); + KUNIT_EXPECT_EQ(test, body[8], 3); + KUNIT_EXPECT_EQ(test, body[9], 13); + KUNIT_EXPECT_EQ(test, body[10], 5); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 11, sizeof(body))); +} + +static void mt_wow_suspend_mode_test(struct kunit *test) +{ + u8 body[MT7932_SUSPEND_MODE_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(body), (size_t)68); + memset(body, 0xa5, sizeof(body)); + mt7932_suspend_mode(body, true); + KUNIT_EXPECT_EQ(test, body[0], 0); /* BSS */ + KUNIT_EXPECT_EQ(test, body[1], 1); + KUNIT_EXPECT_EQ(test, body[2], 1); /* DTIM multiple */ + KUNIT_EXPECT_EQ(test, body[3], 1); /* WoW suspend */ + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(body, 4, sizeof(body))); + mt7932_suspend_mode(body, false); + KUNIT_EXPECT_EQ(test, body[1], 0); + KUNIT_EXPECT_EQ(test, body[2], 1); + KUNIT_EXPECT_EQ(test, body[3], 1); +} + +static void mt_wow_config_test(struct kunit *test) +{ + static const u8 addr[ETH_ALEN] = { 0x02, 0x11, 0x22, 0x33, 0x44, 0x55 }; + u8 magic[MT7932_WOW_MAGIC_LENGTH], clear[MT7932_WOW_CLEAR_LENGTH]; + + KUNIT_EXPECT_EQ(test, sizeof(magic), (size_t)52); + KUNIT_EXPECT_EQ(test, sizeof(clear), (size_t)44); + memset(magic, 0xa5, sizeof(magic)); + mt7932_wow_magic(magic, addr); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 32), 0x06); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 34), 20); + KUNIT_EXPECT_EQ(test, magic[36], 0); /* entry */ + KUNIT_EXPECT_EQ(test, magic[37], 1); /* enabled */ + KUNIT_EXPECT_EQ(test, magic[38], 1); /* forward */ + KUNIT_EXPECT_EQ(test, magic[39], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(magic + 40), 0); + KUNIT_EXPECT_MEMEQ(test, magic + 42, addr, ETH_ALEN); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(magic, 48, sizeof(magic))); + + memset(clear, 0xa5, sizeof(clear)); + mt7932_wow_clear(clear); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 0, 32)); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 32), 0x18); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(clear + 34), 12); + KUNIT_EXPECT_TRUE(test, mt_wow_test_zero(clear, 36, sizeof(clear))); +} + +static void mt_wow_reason_test(struct kunit *test) +{ + struct cfg80211_wowlan_wakeup wakeup; + u8 body[MT7932_WOW_REASON_LENGTH] = {}; + + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body) - 1), -EINVAL); + body[0] = 9; + KUNIT_EXPECT_EQ(test, mt7932_wow_reason(body, sizeof(body)), 9); + + KUNIT_EXPECT_FALSE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_NONE, &wakeup)); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_MAGIC, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(MT7932_WOW_REASON_DISCONNECT, &wakeup)); + KUNIT_EXPECT_TRUE(test, wakeup.disconnect); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + /* A firmware reason with no cfg80211 trigger is still a wireless wake. */ + KUNIT_EXPECT_TRUE(test, mt7932_wow_wakeup(9, &wakeup)); + KUNIT_EXPECT_FALSE(test, wakeup.magic_pkt); + KUNIT_EXPECT_FALSE(test, wakeup.disconnect); + KUNIT_EXPECT_EQ(test, wakeup.pattern_idx, -1); +} + +#ifdef CONFIG_PM +static void mt_wow_test_work(struct work_struct *work) +{ +} + +static void mt_wow_test_device_release(struct device *dev) +{ +} + +static void mt_wow_test_put(void *data) +{ + put_device(data); +} + +static void mt_wow_reason_event_test(struct kunit *test) +{ + struct mt7932 *m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + u8 *packet = kunit_kzalloc(test, 36 + MT7932_WOW_REASON_LENGTH, GFP_KERNEL); + struct mt7932_event event = { + .eid = MT7932_EID_WOW_REASON, + .packet = packet, + .length = 36 + MT7932_WOW_REASON_LENGTH, + }; + unsigned long flags; + + KUNIT_ASSERT_NOT_NULL(test, m); + KUNIT_ASSERT_NOT_NULL(test, packet); + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, m->pdev); + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_wow_test_device_release; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, mt_wow_test_put, &m->pdev->dev), 0); + KUNIT_ASSERT_EQ(test, dev_set_name(&m->pdev->dev, "mt7932-wow-test"), 0); + spin_lock_init(&m->response_lock); + INIT_WORK(&m->wow_work, mt_wow_test_work); + m->wow_reason = MT7932_WOW_REASON_NONE; + packet[36] = MT7932_WOW_REASON_MAGIC; + + /* Not a sleep WoW was armed for: ignored. */ + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* A short event leaves the wait open. */ + m->wow_report = true; + event.length--; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_TRUE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_NONE); + + /* One report per sleep. */ + event.length++; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + flush_work(&m->wow_work); + KUNIT_EXPECT_FALSE(test, m->wow_report); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); + packet[36] = MT7932_WOW_REASON_DISCONNECT; + spin_lock_irqsave(&m->response_lock, flags); + mt_wow_reason_event(m, &event); + spin_unlock_irqrestore(&m->response_lock, flags); + KUNIT_EXPECT_EQ(test, m->wow_reason, MT7932_WOW_REASON_MAGIC); +} +#endif + +static struct kunit_case mt_wow_test_cases[] = { + KUNIT_CASE(mt_wow_detect_test), + KUNIT_CASE(mt_wow_ctrl_test), + KUNIT_CASE(mt_wow_suspend_mode_test), + KUNIT_CASE(mt_wow_config_test), + KUNIT_CASE(mt_wow_reason_test), +#ifdef CONFIG_PM + KUNIT_CASE(mt_wow_reason_event_test), +#endif + {} +}; + +static struct kunit_suite mt_wow_test_suite = { + .name = "mt7932-wow", + .test_cases = mt_wow_test_cases, +}; + +kunit_test_suite(mt_wow_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/wow.c b/drivers/net/wireless/mediatek/mt7932/wow.c new file mode 100644 index 0000000000000..279a0c01a7b1d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Wake-on-WLAN for a deep sleep. The firmware keeps the association while the + * host sleeps and wakes it through the radio's GPIO, which the SMC turns into + * an AP wake. Nothing in-band can wake the SoC from S2R. + */ +#include + +#include "mt7932.h" +#include "wow.h" + +#ifdef CONFIG_PM +const struct wiphy_wowlan_support mt_wowlan_support = { + .flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT, +}; + +/* response_lock held. */ +static bool mt_wow_associated_locked(struct mt7932 *m) +{ + return m->connected && m->peer_valid && !m->connecting && !m->disconnecting && + !m->link_failed && !m->stopping && m->rf_ready; +} + +/* + * cfg80211 calls this before the device's own suspend. Returning 1 makes it + * leave the network and call again without a configuration. + */ +int mt_wow_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + u8 detect = mt7932_wow_detect(wow); + unsigned long flags; + bool associated; + + WRITE_ONCE(m->wow_detect, 0); + if (!wow) + return 0; + /* Only S2R hands the radio to the firmware; s2idle and hibernation leave. */ + if (!detect || pm_suspend_target_state != PM_SUSPEND_MEM || pm_suspend_no_platform()) + return 1; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) + return 1; + WRITE_ONCE(m->wow_detect, detect); + return 0; +} + +int mt_wow_resume(struct wiphy *wiphy) +{ + WRITE_ONCE(mt_from_wiphy(wiphy)->wow_detect, 0); + return 0; +} + +/* + * command_mutex held, before the host interface stops. The firmware takes the + * commands from the ring before it answers the stop. + */ +void mt_wow_arm(struct mt7932 *m) +{ + u8 detect = READ_ONCE(m->wow_detect); + u8 body[MT7932_WOW_CTRL_LENGTH]; + unsigned long flags; + bool associated; + int ret = 0; + + lockdep_assert_held(&m->command_mutex); + WRITE_ONCE(m->wow_detect, 0); + if (!detect) + return; + spin_lock_irqsave(&m->response_lock, flags); + associated = mt_wow_associated_locked(m); + m->wow_report = false; + m->wow_reason = MT7932_WOW_REASON_NONE; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!associated) { + dev_dbg(&m->pdev->dev, "not associated at the sleep: Wi-Fi wake not armed\n"); + return; + } + /* Whatever reached the firmware is undone at the wake. */ + m->wow_armed = true; + if (detect & MT7932_WOW_DETECT_MAGIC) { + mt7932_wow_magic(body, m->mac); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_MAGIC_LENGTH); + } + if (!ret) + ret = mt_power_submit(m, MT7932_WOW_POWER_MODE); + if (!ret) { + mt7932_suspend_mode(body, true); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + } + if (!ret) { + mt7932_wow_ctrl(body, true, detect); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (ret) { + dev_err(&m->pdev->dev, "Wi-Fi wake not armed: %d\n", ret); + return; + } + spin_lock_irqsave(&m->response_lock, flags); + m->wow_report = true; + spin_unlock_irqrestore(&m->response_lock, flags); + dev_dbg(&m->pdev->dev, "Wi-Fi wake armed:%s%s, staying associated\n", + detect & MT7932_WOW_DETECT_MAGIC ? " magic packet" : "", + detect & MT7932_WOW_DETECT_DISCONNECT ? " disconnect" : ""); +} + +/* + * command_mutex held, once the host interface runs again. A command that + * cannot be queued leaves the firmware filtering, so the caller resets it. + */ +int mt_wow_disarm(struct mt7932 *m) +{ + u8 body[MT7932_WOW_CTRL_LENGTH]; + int ret; + + lockdep_assert_held(&m->command_mutex); + if (!m->wow_armed) + return 0; + m->wow_armed = false; + mt7932_suspend_mode(body, false); + ret = mt_request(m, MT7932_CID_SUSPEND_MODE, true, true, false, body, + MT7932_SUSPEND_MODE_LENGTH); + if (!ret) { + mt7932_wow_ctrl(body, false, 0); + ret = mt_request(m, MT7932_CID_WOW_CTRL, true, true, false, body, + MT7932_WOW_CTRL_LENGTH); + } + if (!ret) { + mt7932_wow_clear(body); + ret = mt_request(m, MT7932_CID_WOW_CONFIG, true, true, false, body, + MT7932_WOW_CLEAR_LENGTH); + } + /* The sleep's profile stays until power_work submits the requested one. */ + m->power_applied_valid = false; + if (ret) + dev_err(&m->pdev->dev, "Wi-Fi wake not stopped: %d\n", ret); + return ret; +} + +/* response_lock held. The firmware reports why it woke the host. */ +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ + int reason = event->length < 36 ? -EINVAL : + mt7932_wow_reason(event->packet + 36, event->length - 36); + + if (reason < 0) { + dev_dbg(&m->pdev->dev, "short wake reason event: %zu bytes\n", event->length); + return; + } + if (!m->wow_report) + return; + m->wow_report = false; + m->wow_reason = reason; + schedule_work(&m->wow_work); +} + +void mt_wow_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, wow_work); + struct cfg80211_wowlan_wakeup wakeup; + unsigned long flags; + u8 reason; + + spin_lock_irqsave(&m->response_lock, flags); + reason = m->wow_reason; + spin_unlock_irqrestore(&m->response_lock, flags); + if (!mt7932_wow_wakeup(reason, &wakeup)) { + dev_dbg(&m->pdev->dev, "Wi-Fi did not wake the host\n"); + cfg80211_report_wowlan_wakeup(&m->wdev, NULL, GFP_KERNEL); + return; + } + dev_info(&m->pdev->dev, "woken by Wi-Fi: firmware reason %u%s\n", reason, + wakeup.magic_pkt ? " (magic packet)" : + wakeup.disconnect ? " (disconnect)" : ""); + cfg80211_report_wowlan_wakeup(&m->wdev, &wakeup, GFP_KERNEL); +} +#else +void mt_wow_arm(struct mt7932 *m) +{ +} + +int mt_wow_disarm(struct mt7932 *m) +{ + return 0; +} + +void mt_wow_reason_event(struct mt7932 *m, const struct mt7932_event *event) +{ +} + +void mt_wow_work(struct work_struct *work) +{ +} +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/wow.h b/drivers/net/wireless/mediatek/mt7932/wow.h new file mode 100644 index 0000000000000..2bbe7d4529a1b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/wow.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_WOW_H +#define MT7932_WOW_H + +#include +#include +#include +#include +#include +#include + +/* + * Wake-on-WLAN commands, in the order the stock driver sends them for an + * associated sleep: the magic packet entry on the WoW configuration command, + * the power-save profile, suspend mode, then WoW start. Waking reverses it. + */ +#define MT7932_CID_WOW_CTRL 0x4a +#define MT7932_CID_WOW_CONFIG 0x4f +#define MT7932_CID_SUSPEND_MODE 0x58 +#define MT7932_EID_WOW_REASON 0x62 + +#define MT7932_WOW_CTRL_LENGTH 228 +#define MT7932_SUSPEND_MODE_LENGTH 68 +/* The configuration command is a zeroed 32-byte header and one TLV. */ +#define MT7932_WOW_CONFIG_HEADER 32 +#define MT7932_WOW_MAGIC_LENGTH (MT7932_WOW_CONFIG_HEADER + 20) +#define MT7932_WOW_CLEAR_LENGTH (MT7932_WOW_CONFIG_HEADER + 12) +#define MT7932_WOW_TLV_MAGIC 0x06 +#define MT7932_WOW_TLV_CLEAR 0x18 + +#define MT7932_WOW_START 1 +#define MT7932_WOW_STOP 2 + +#define MT7932_WOW_DETECT_MAGIC BIT(0) +#define MT7932_WOW_DETECT_DISCONNECT BIT(2) +#define MT7932_WOW_DETECT_BEACON_LOST BIT(4) + +/* + * Drop everything but magic packets, EAPOL and ARP requests to this station, + * as the stock driver filters for a sleep. + */ +#define MT7932_WOW_FILTER 0x63 + +/* The stock configuration wakes the host on the radio's GPIO 13, level 5. */ +#define MT7932_WOW_HIF_GPIO 3 +#define MT7932_WOW_GPIO_PIN 13 +#define MT7932_WOW_GPIO_LEVEL 5 +#define MT7932_WOW_PIN_COUNT 1 + +/* Power-save profile for the sleep, and the suspend mode's DTIM multiple. */ +#define MT7932_WOW_POWER_MODE 2 +#define MT7932_WOW_MDTIM 1 + +#define MT7932_WOW_REASON_MAGIC 0 +#define MT7932_WOW_REASON_DISCONNECT 8 +#define MT7932_WOW_REASON_NONE 0xff +/* The stock driver ignores a shorter reason event. */ +#define MT7932_WOW_REASON_LENGTH 132 + +/* The firmware detectors for the triggers cfg80211 asked for; 0 if none applies. */ +static inline u8 mt7932_wow_detect(const struct cfg80211_wowlan *wow) +{ + u8 detect = 0; + + if (!wow) + return 0; + if (wow->magic_pkt) + detect |= MT7932_WOW_DETECT_MAGIC; + if (wow->disconnect) + detect |= MT7932_WOW_DETECT_DISCONNECT | MT7932_WOW_DETECT_BEACON_LOST; + return detect; +} + +/* Start, with these detectors, or stop. */ +static inline void mt7932_wow_ctrl(u8 out[MT7932_WOW_CTRL_LENGTH], bool start, u8 detect) +{ + memset(out, 0, MT7932_WOW_CTRL_LENGTH); + out[0] = start ? MT7932_WOW_START : MT7932_WOW_STOP; + if (start) { + out[1] = detect; + put_unaligned_le16(MT7932_WOW_FILTER, out + 2); + } + /* Scenario 0, then the wake signal: one pin, GPIO, which pin, level. */ + out[5] = MT7932_WOW_PIN_COUNT; + out[8] = MT7932_WOW_HIF_GPIO; + out[9] = MT7932_WOW_GPIO_PIN; + out[10] = MT7932_WOW_GPIO_LEVEL; + /* No UDP or TCP wake ports. */ +} + +/* BSS 0 into or out of suspend mode; the DTIM multiple and WoW flag stay set. */ +static inline void mt7932_suspend_mode(u8 out[MT7932_SUSPEND_MODE_LENGTH], bool enable) +{ + memset(out, 0, MT7932_SUSPEND_MODE_LENGTH); + out[1] = enable; + out[2] = MT7932_WOW_MDTIM; + out[3] = 1; +} + +/* Magic packet entry 0 for this address, matched from the start of the frame. */ +static inline void mt7932_wow_magic(u8 out[MT7932_WOW_MAGIC_LENGTH], const u8 *addr) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_MAGIC_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_MAGIC, tlv); + put_unaligned_le16(MT7932_WOW_MAGIC_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); + tlv[4] = 0; /* entry */ + tlv[5] = 1; /* enabled */ + tlv[6] = 1; /* offset counts forward */ + put_unaligned_le16(0, tlv + 8); + ether_addr_copy(tlv + 10, addr); +} + +/* Remove every WoW configuration entry. */ +static inline void mt7932_wow_clear(u8 out[MT7932_WOW_CLEAR_LENGTH]) +{ + u8 *tlv = out + MT7932_WOW_CONFIG_HEADER; + + memset(out, 0, MT7932_WOW_CLEAR_LENGTH); + put_unaligned_le16(MT7932_WOW_TLV_CLEAR, tlv); + put_unaligned_le16(MT7932_WOW_CLEAR_LENGTH - MT7932_WOW_CONFIG_HEADER, tlv + 2); +} + +/* + * The reason from a wake reason event body, or -EINVAL if it is short. + * MT7932_WOW_REASON_NONE means the radio did not wake the host. + */ +static inline int mt7932_wow_reason(const u8 *body, size_t length) +{ + if (length < MT7932_WOW_REASON_LENGTH) + return -EINVAL; + return body[0]; +} + +/* + * What to tell cfg80211 for a reason. False if the radio did not wake the + * host. A reason without a cfg80211 trigger reports a wake with none set. + */ +static inline bool mt7932_wow_wakeup(u8 reason, struct cfg80211_wowlan_wakeup *wakeup) +{ + memset(wakeup, 0, sizeof(*wakeup)); + wakeup->pattern_idx = -1; + switch (reason) { + case MT7932_WOW_REASON_NONE: + return false; + case MT7932_WOW_REASON_MAGIC: + wakeup->magic_pkt = true; + break; + case MT7932_WOW_REASON_DISCONNECT: + wakeup->disconnect = true; + break; + } + return true; +} + +#endif diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f6b2b8cf939ac..ded0a31e0e3b0 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -40,17 +40,21 @@ #include #include #include +#include +#include #include #include #include "nvme.h" #define APPLE_ANS_BOOT_TIMEOUT USEC_PER_SEC +#define APPLE_ANS_SLEEP_SHN_TIMEOUT USEC_PER_SEC /* RTKit handshake timeout for the T6050 ANS, see apple_nvme_alloc() */ #define APPLE_NVME_T6050_HANDSHAKE_TIMEOUT_MS 10000 #define APPLE_ANS_COPROC_CPU_CONTROL 0x44 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_ANS_COPROC_CPU_STATUS 0x48 #define APPLE_ANS_ACQ_DB 0x1004 #define APPLE_ANS_IOCQ_DB 0x100c @@ -211,6 +215,14 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; + /* Firmware-driven suspend has been traced on this controller. */ + bool sleep_traced; + /* The last endpoint map reply from that trace */ + u32 sleep_epmap_last_mask; + /* Higher namespaces are the firmware's, and a Flush to one faults the controller. */ + u32 sleep_flush_max_nsid; + /* CC.SHN for a sleep, as read back from the OS the sleep was traced on. */ + u32 sleep_shn; /* The ANS is never reset through PMGR here, so no reset is named. */ bool reset_optional; u32 max_queue_depth; @@ -237,6 +249,17 @@ struct apple_nvme { bool recover_to_live; bool suspended; bool final_shutdown; + /* This suspend is firmware-driven: ANS will lose power at its end. */ + bool sleep_stop; + bool sleep_idled; + u32 sleep_csts; + /* ANS came back across a sleep, or failed to. reset_work picks these up. */ + bool sleep_restarted; + bool sleep_failed; + /* Run bit set after a sleep, and not yet reported on. */ + bool sleep_restarting; + ktime_t sleep_run_time; + bool sleep_registered; unsigned long recovery_flags; void __iomem *mmio_coproc; @@ -1397,6 +1420,31 @@ static int apple_nvme_disable(struct apple_nvme *anv, bool shutdown) return apple_nvme_disable_locked(anv, shutdown); } +/* + * All of a sleep's stop that can be taken back. Nothing is in flight or can start, and + * the controller and its firmware keep running for a refused sleep to come back to. + */ +static int apple_nvme_sleep_quiesce(struct apple_nvme *anv) +{ + guard(mutex)(&anv->disable_lock); + if (READ_ONCE(anv->quarantined)) + return -EIO; + + if (!test_bit(NVME_CTRL_FROZEN, &anv->ctrl.flags)) + nvme_start_freeze(&anv->ctrl); + if (!nvme_wait_freeze_timeout(&anv->ctrl)) { + /* A reset cancels what's still in flight. Don't sleep on that. */ + dev_warn(anv->dev, "requests still in flight after the timeout\n"); + return -EBUSY; + } + nvme_quiesce_io_queues(&anv->ctrl); + nvme_quiesce_admin_queue(&anv->ctrl); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); + + return 0; +} + static void apple_nvme_recovery_work(struct work_struct *work) { struct apple_nvme *anv = container_of(work, struct apple_nvme, recovery_work); @@ -1647,6 +1695,8 @@ static void apple_nvme_stop_crashed_session(struct apple_nvme *anv) anv->ctrl.ctrl_config = 0; } +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv); + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1668,6 +1718,17 @@ static void apple_nvme_reset_work(struct work_struct *work) return; } WRITE_ONCE(anv->abort_reset, false); + if (READ_ONCE(anv->sleep_restarting) && !apple_nvme_sleep_wait_on(anv)) + WRITE_ONCE(anv->sleep_failed, true); + if (READ_ONCE(anv->sleep_failed)) { + /* + * Starting it some other way can only reset it, which faults, and failing the + * controller fails every held write. Leave the queues quiesced and frozen. + */ + if (phase_locked) + mutex_unlock(&anv->disable_lock); + return; + } if (state != NVME_CTRL_RESETTING) { dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state); ret = -ENODEV; @@ -1735,6 +1796,12 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = -EIO; goto out; } + /* A real sleep took the controller's state with its power. */ + if (READ_ONCE(anv->sleep_restarted)) { + WRITE_ONCE(anv->sleep_restarted, false); + if (!(readl(anv->mmio_nvme + NVME_REG_CC) & NVME_CC_ENABLE)) + goto rtkit_ready; + } ret = apple_nvme_disable_locked(anv, false); if (ret) goto out; @@ -2595,6 +2662,57 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) return ERR_PTR(ret); } +/* + * A sleep's shutdown notification, with interrupts off, before anything else stops. macOS + * finds the controller enabled and shut down when it tells the firmware to idle. A failure + * refuses: the firmware hasn't been told, and resume's controller reset takes it back. + */ +static int apple_nvme_sleep_shutdown(void *data) +{ + struct apple_nvme *anv = data; + u32 csts; + int ret; + + anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; + anv->ctrl.ctrl_config |= anv->hw->sleep_shn; + writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); + ret = readl_poll_timeout_atomic(anv->mmio_nvme + NVME_REG_CSTS, csts, + (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, + 100, APPLE_ANS_SLEEP_SHN_TIMEOUT); + if (ret) + dev_err(anv->dev, "shutdown notification failed: CSTS %#x\n", csts); + anv->sleep_csts = csts; + + return ret; +} + +/* + * The rest of a sleep's stop, last of all. From the first idle message only a real sleep + * brings it back, so a failure goes on to the commit: what a flush acknowledged is on + * flash, and the watchdog is the net. + */ +static int apple_nvme_sleep_last_stop(void *data) +{ + struct apple_nvme *anv = data; + unsigned int us; + u32 cc, csts = anv->sleep_csts; + int ret; + + cc = readl(anv->mmio_nvme + NVME_REG_CC); + + anv->sleep_idled = true; + ret = apple_rtkit_sleep_idle_polled(anv->rtk, + anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL, &us); + if (ret) + dev_err(anv->dev, "ANS did not idle: %d\n", ret); + else + dev_dbg(anv->dev, "ANS idle, acknowledged in about %u us, CC %#x CSTS %#x\n", + us, cc, csts); + apple_rtkit_mailbox_recv_off(anv->rtk); + + return ret ? 1 : 0; +} + static int apple_nvme_probe(struct platform_device *pdev) { struct apple_nvme *anv; @@ -2623,6 +2741,19 @@ static int apple_nvme_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + if (anv->hw->sleep_traced) { + anv->sleep_registered = + !apple_sleep_coproc_register(anv->dev, anv->mmio_coproc, true, + apple_nvme_sleep_last_stop, anv); + /* Never idled without its shutdown notification. */ + if (anv->sleep_registered && + apple_sleep_coproc_set_last_check(anv->dev, apple_nvme_sleep_shutdown)) { + apple_sleep_coproc_unregister(anv->dev); + anv->sleep_registered = false; + } + apple_rtkit_set_warm_epmap_mask(anv->rtk, anv->hw->sleep_epmap_last_mask); + } + apple_nvme_set_reset_ready(anv, true); nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -2641,6 +2772,8 @@ static void apple_nvme_remove(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + if (anv->sleep_registered) + apple_sleep_coproc_unregister(anv->dev); WRITE_ONCE(anv->removing, true); apple_nvme_set_reset_ready(anv, false); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); @@ -2706,6 +2839,82 @@ static void apple_nvme_shutdown(struct platform_device *pdev) } } +/* macOS sees the firmware report on 68 ms after HELLO. Each try waits RTKit's timeout again. */ +#define APPLE_NVME_SLEEP_ON_TRIES 15 + +/* + * After a real sleep ANS has lost power, and its resident firmware restarts once the run bit + * is set. Measured on macOS under aura, the same in five runs: HELLO first and unprompted, + * then the endpoint map, the last part answered with endpoint 0x20 only. No endpoint is + * started and no buffer asked for. About 68 ms later the firmware reports on, and the AP state + * follows, with the controller's registers reading 0. A refused suspend never told the + * firmware anything, and takes a reset. + * + * The run bit goes in the early phase, so HELLO comes within milliseconds of the wake as on + * macOS. The wait for on doesn't: macOS rings the SEP a few ms after ANS's HELLO, before ANS + * reports on, so the reset work waits, after the SEP's own resume has had its turn. + */ +static int apple_nvme_resume_early(struct device *dev) +{ + struct apple_nvme *anv = dev_get_drvdata(dev); + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret; + + if (!anv->hw->needs_ioq_registers || !anv->suspended || !anv->sleep_idled) + return 0; + anv->sleep_idled = false; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + /* Its link doesn't come up with the pins as the sleep left them. */ + apple_sleep_restore_pins(); + /* macOS writes the SART's entries back again right before the run bit. */ + if (anv->sart) + apple_sart_replay(anv->sart); + + dev_dbg(dev, "ANS: restarting after the sleep, CPU control %#x\n", readl(control)); + anv->sleep_run_time = ktime_get(); + ret = apple_rtkit_sleep_return_start(anv->rtk, control); + if (ret) { + dev_crit(dev, "ANS restart after sleep failed: %d\n", ret); + anv->sleep_failed = true; + return 0; + } + WRITE_ONCE(anv->sleep_restarting, true); + + return 0; +} + +/* From the reset work: wait for ANS to report on. False means hold every request. */ +static bool apple_nvme_sleep_wait_on(struct apple_nvme *anv) +{ + void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; + int ret, tries = 0; + + do { + ret = apple_rtkit_boot(anv->rtk); + if (ret != -ETIME) + break; + dev_warn(anv->dev, "ANS: not on %lld ms after the run bit, waiting again\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + } while (++tries < APPLE_NVME_SLEEP_ON_TRIES); + + WRITE_ONCE(anv->sleep_restarting, false); + if (ret) { + dev_crit(anv->dev, "ANS did not come back after sleep: %d after %lld ms, CPU control %#x status %#x, crashed %d\n", + ret, ktime_ms_delta(ktime_get(), anv->sleep_run_time), readl(control), + readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_STATUS), + apple_rtkit_is_crashed(anv->rtk)); + dev_crit(anv->dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return false; + } + WRITE_ONCE(anv->sleep_restarted, true); + /* The one line per wake that says ANS is back. */ + dev_info(anv->dev, "ANS: on %lld ms after the run bit, resetting the controller before any I/O\n", + ktime_ms_delta(ktime_get(), anv->sleep_run_time)); + + return true; +} + static int apple_nvme_resume(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2715,6 +2924,20 @@ static int apple_nvme_resume(struct device *dev) return -EIO; if (!anv->suspended) return 0; + if (anv->sleep_stop) + apple_sleep_coproc_set_ready(dev, false); + anv->sleep_idled = false; + anv->sleep_stop = false; + /* + * Hold every request rather than fail it: the queues stay frozen and quiesced + * from the suspend, so nothing reaches a controller that isn't there and no + * filesystem sees a write error. Only a reboot brings ANS back from this. + */ + if (READ_ONCE(anv->sleep_failed)) { + dev_crit(dev, "holding all I/O: ANS is down after the sleep, reboot to recover\n"); + return 0; + } + /* The reset work waits for ANS first, if it's still coming up. */ WRITE_ONCE(anv->suspended, false); return nvme_try_sched_reset(&anv->ctrl); } @@ -2743,19 +2966,76 @@ static void apple_nvme_keep_link_powered(struct apple_nvme *anv) device_set_awake_path(anv->pd_dev[i]); } +/* + * What is written has to be on flash before power goes: an fsynced overwrite is + * known to revert after an abrupt reset on these controllers unless flushed. + */ +static int apple_nvme_sleep_flush(struct apple_nvme *anv) +{ + struct nvme_ctrl *ctrl = &anv->ctrl; + struct nvme_ns *ns; + int srcu_idx, ret = 0; + + srcu_idx = srcu_read_lock(&ctrl->srcu); + list_for_each_entry_srcu(ns, &ctrl->namespaces, list, + srcu_read_lock_held(&ctrl->srcu)) { + struct nvme_command c = { }; + + if (ns->head->ns_id > anv->hw->sleep_flush_max_nsid) + continue; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(ns->head->ns_id); + ret = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (ret) + break; + } + srcu_read_unlock(&ctrl->srcu, srcu_idx); + + return ret; +} + +static bool apple_nvme_sleep_wanted(struct apple_nvme *anv) +{ + return anv->sleep_registered && + pm_suspend_target_state == PM_SUSPEND_MEM && pm_suspend_via_firmware() && + nvme_ctrl_state(&anv->ctrl) == NVME_CTRL_LIVE && anv->owns_rtkit && + apple_rtkit_is_running(anv->rtk); +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); int ret; if (anv->hw->needs_ioq_registers) { - if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) + /* A sleep right after a wake, as from a dark wake: let that reset finish first. */ + flush_work(&anv->ctrl.reset_work); + anv->sleep_stop = apple_nvme_sleep_wanted(anv); + anv->sleep_idled = false; + anv->sleep_failed = false; + anv->sleep_restarted = false; + if (anv->sleep_stop) { + ret = apple_nvme_sleep_flush(anv); + if (ret) { + dev_err(dev, "refusing to sleep: flush failed: %d\n", ret); + anv->sleep_stop = false; + return -EIO; + } + } + if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) { + anv->sleep_stop = false; return -EBUSY; + } WRITE_ONCE(anv->suspended, true); } - ret = apple_nvme_disable(anv, true); - if (ret) - return ret; + if (anv->sleep_stop) { + anv->sleep_failed = apple_nvme_sleep_quiesce(anv); + } else { + ret = apple_nvme_disable(anv, true); + if (ret) + return ret; + } /* * Only the NVMe controller was disabled. Its owned RTKit session stays * live, so resume rebuilds queues without clearing CPU_CONTROL.RUN. @@ -2763,6 +3043,19 @@ static int apple_nvme_suspend(struct device *dev) if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); apple_nvme_keep_link_powered(anv); + if (anv->sleep_failed) { + /* Requests are stuck, so bring it back and refuse. */ + dev_err(dev, "refusing to sleep: NVMe didn't quiesce\n"); + anv->sleep_failed = false; + anv->sleep_stop = false; + WRITE_ONCE(anv->suspended, false); + nvme_try_sched_reset(&anv->ctrl); + return -EIO; + } + if (anv->sleep_stop) { + apple_sleep_coproc_set_ready(dev, true); + dev_dbg(dev, "NVMe suspended for a sleep\n"); + } return 0; } @@ -2775,8 +3068,10 @@ static int apple_nvme_suspend(struct device *dev) return ret; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, - apple_nvme_resume); +static const struct dev_pm_ops apple_nvme_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_nvme_suspend, apple_nvme_resume) + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_nvme_resume_early) +}; static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, @@ -2809,6 +3104,10 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .has_separate_nvmmu = true, .needs_ioq_registers = true, .max_queue_depth = 64, + .sleep_traced = true, + .sleep_epmap_last_mask = BIT(0), + .sleep_flush_max_nsid = 1, + .sleep_shn = NVME_CC_SHN_MASK, /* * An fsynced overwrite completed and later reverted after an abrupt * reset; explicit flushes persisted the preceding data. Let the block diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h index 21627bfa2ad4a..46dddb9f42610 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h @@ -15,7 +15,10 @@ #define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) #define PIODMA_IRQ_LIMIT 16 -/* Only RELEASE before publication can end guard ownership. No reset event. */ +/* + * Only RELEASE before publication can end guard ownership, or POWER_LOST: the + * engine's whole register file reading as reset after the SoC powered it off. + */ enum apple_piodma_state { PIODMA_IDLE, PIODMA_GUARDED, @@ -37,6 +40,7 @@ enum apple_piodma_event { PIODMA_COMPLETE, PIODMA_ADMIT, PIODMA_FAIL, + PIODMA_POWER_LOST, PIODMA_EVENT_COUNT, }; @@ -105,6 +109,10 @@ static inline int apple_piodma_next(enum apple_piodma_state state, if (state == PIODMA_RETAINED) return PIODMA_FAILED_RETAINED; break; + case PIODMA_POWER_LOST: + if (state == PIODMA_ADMITTED) + return PIODMA_GUARDED; + break; default: break; } diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c index 941a1786c0483..9cccb3ca4bcd0 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c @@ -61,12 +61,12 @@ static void apple_piodma_transition_guards_test(struct kunit *test) { int state, event, next; - /* Exhaust all transitions: no retained state can release or retry. */ + /* Exhaust all transitions: only a power loss can end a retained state. */ for (state = PIODMA_IDLE; state < PIODMA_STATE_COUNT; state++) { for (event = PIODMA_GUARD; event < PIODMA_EVENT_COUNT; event++) { next = apple_piodma_next(state, event); if (apple_piodma_retained(state)) { - if (next >= 0) + if (next >= 0 && event != PIODMA_POWER_LOST) KUNIT_EXPECT_TRUE(test, apple_piodma_retained(next)); if (event == PIODMA_RELEASE || event == PIODMA_PRIME || event == PIODMA_GUARD) @@ -128,8 +128,24 @@ static void apple_piodma_irq_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 1, 2)); } +/* Losing power reopens the guard from an admitted radio, and from nothing else. */ +static void apple_piodma_power_lost_test(struct kunit *test) +{ + static const enum apple_piodma_state refused[] = { + PIODMA_IDLE, PIODMA_GUARDED, PIODMA_ATTEMPTED, PIODMA_FAILED, + PIODMA_REJECTED, PIODMA_RETAINED, PIODMA_READY, PIODMA_FAILED_RETAINED, + }; + int i; + + KUNIT_EXPECT_EQ(test, apple_piodma_next(PIODMA_ADMITTED, PIODMA_POWER_LOST), + PIODMA_GUARDED); + for (i = 0; i < ARRAY_SIZE(refused); i++) + KUNIT_EXPECT_LT(test, apple_piodma_next(refused[i], PIODMA_POWER_LOST), 0); +} + static struct kunit_case apple_piodma_cases[] = { KUNIT_CASE(apple_piodma_lifecycle_test), + KUNIT_CASE(apple_piodma_power_lost_test), KUNIT_CASE(apple_piodma_failure_test), KUNIT_CASE(apple_piodma_transition_guards_test), KUNIT_CASE(apple_piodma_arena_test), diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c index a95dd5112f201..9261d23abee16 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c @@ -92,6 +92,16 @@ struct apple_t8140_bootstrap { struct kexec_blocker kexec_blocker; struct delayed_work admission_retry; struct device *retry_host; + /* The host stops the radios for a sleep that takes this engine's power. */ + bool sleep_armed; + bool power_lost; + /* After a real sleep: read the engine once the host's port is up again. */ + bool power_check; + bool irq_held; + int (*radio_stop)(struct pci_dev *pdev); + struct module *radio_stop_owner; + void (*radio_recover)(void *data); + void *radio_recover_data; }; /* A successful supplier probe owns its unpublished arena until full reset. */ @@ -193,6 +203,8 @@ static int apple_piodma_pm_prepare(struct device *dev) return 0; if (apple_piodma_sleep_allowed(apple_piodma_state(bootstrap), pm_suspend_no_platform())) return 0; + if (READ_ONCE(bootstrap->sleep_armed)) + return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this * supplier probed. Its later image/restore/suspend ioctls go straight @@ -372,6 +384,10 @@ static irqreturn_t apple_t8140_bootstrap_irq(int irq, void *data) u32 fabric, status, tag, active; int count; + /* After a real sleep its block can't answer until the host's port is back. */ + if (READ_ONCE(bootstrap->irq_held)) + return IRQ_NONE; + /* Acknowledge order follows the attributed prior controller contract. */ fabric = readl(bootstrap->fabric); writel(fabric, bootstrap->fabric); @@ -744,6 +760,60 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + WRITE_ONCE(bootstrap->sleep_armed, armed); + /* A check still pending means the block never came back, and stays unread. */ + if (armed) + bootstrap->power_lost = false; +} + +/* The next bootstrap starts as the first did, on the same retained memory. */ +int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + void *arena = bootstrap->arena; + int i; + + /* + * Everything reading as it did before the first command is what shows the sleep took + * the engine's pointers with it. Read only now: the block answers once the host's + * port is set up, after resume_noirq. + */ + if (bootstrap->power_check) { + bootstrap->power_check = false; + bootstrap->power_lost = !apple_t8140_bootstrap_idle(bootstrap); + if (!bootstrap->power_lost) + dev_err(supplier, "engine kept state across sleep; radios stay down\n"); + WRITE_ONCE(bootstrap->irq_held, false); + } + if (!bootstrap->power_lost || READ_ONCE(bootstrap->irq_fault)) + return -EIO; + bootstrap->power_lost = false; + + memset(arena, 0, PIODMA_ARENA_SIZE); + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { + void *slot = arena + i * PIODMA_SLOT_SIZE; + + memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); + *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); + memset(bootstrap->slots[i].packet, 0, sizeof(bootstrap->slots[i].packet)); + memset(bootstrap->slots[i].completion, 0, sizeof(bootstrap->slots[i].completion)); + bootstrap->slots[i].result = 0; + WRITE_ONCE(bootstrap->slots[i].matched_status, 0); + bootstrap->slots[i].submitted = false; + bootstrap->slots[i].observed = false; + reinit_completion(&bootstrap->slots[i].event); + } + dma_wmb(); + WRITE_ONCE(bootstrap->active_tag, 0); + atomic_set(&bootstrap->irq_count, 0); + + return apple_piodma_transition(bootstrap, PIODMA_POWER_LOST); +} + bool apple_piodma_bootstrap_retained(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -753,6 +823,43 @@ bool apple_piodma_bootstrap_retained(struct device *supplier) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); +/* The stop and the module it's in, set and read together. */ +static DEFINE_SPINLOCK(radio_stop_lock); + +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + int (*stop)(struct pci_dev *pdev); + struct module *owner; + bool held; + int ret; + + /* Its module stays while it runs. One that is unloading: its remove stops the radio. */ + spin_lock(&radio_stop_lock); + stop = bootstrap->radio_stop; + owner = bootstrap->radio_stop_owner; + held = !stop || try_module_get(owner); + spin_unlock(&radio_stop_lock); + if (!held) + return -EBUSY; + if (!stop) + return 0; + + ret = stop(pdev); + module_put(owner); + + return ret; +} + +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data) +{ + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); + + bootstrap->radio_recover_data = data; + WRITE_ONCE(bootstrap->radio_recover, recover); +} + void apple_piodma_bootstrap_admitted(struct device *supplier) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); @@ -790,6 +897,31 @@ int apple_piodma_radio_check(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(apple_piodma_radio_check); +void apple_piodma_radio_recover(void) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + void (*recover)(void *data) = bootstrap ? READ_ONCE(bootstrap->radio_recover) : NULL; + + if (recover) + recover(bootstrap->radio_recover_data); +} +EXPORT_SYMBOL_GPL(apple_piodma_radio_recover); + +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner) +{ + /* Pairs with the release at registration. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); + + if (!bootstrap) + return; + spin_lock(&radio_stop_lock); + bootstrap->radio_stop = stop; + bootstrap->radio_stop_owner = owner; + spin_unlock(&radio_stop_lock); +} +EXPORT_SYMBOL_GPL(__apple_piodma_radio_set_stop); + static int apple_t8140_bootstrap_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -822,7 +954,8 @@ static int apple_t8140_bootstrap_probe(struct platform_device *pdev) ret = apple_t8140_bootstrap_resources(pdev, bootstrap); if (ret) goto err_arena; - ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, 0, + /* The host's resume_noirq waits for one completion, with device interrupts still off. */ + ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, IRQF_EARLY_RESUME, dev_name(dev), bootstrap); if (ret) goto err_arena; @@ -873,12 +1006,20 @@ static int apple_t8140_bootstrap_sleep_check(struct apple_t8140_bootstrap *boots static int apple_t8140_bootstrap_suspend_noirq(struct device *dev) { struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); + int ret; if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !READ_ONCE(bootstrap->sleep_armed)) return -EOPNOTSUPP; - return apple_t8140_bootstrap_sleep_check(bootstrap); + /* Radios that failed to come back don't cost the machine a deep sleep. */ + if (READ_ONCE(bootstrap->sleep_armed) && + (!apple_piodma_ready(apple_piodma_state(bootstrap)) || bootstrap->power_check)) + return 0; + ret = apple_t8140_bootstrap_sleep_check(bootstrap); + if (!ret && READ_ONCE(bootstrap->sleep_armed)) + WRITE_ONCE(bootstrap->irq_held, true); + return ret; } static int apple_t8140_bootstrap_resume_noirq(struct device *dev) @@ -890,6 +1031,15 @@ static int apple_t8140_bootstrap_resume_noirq(struct device *dev) return 0; if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; + /* + * Its block reads nothing until the host sets its port up again, so neither read it + * nor take its interrupt until then: apple_piodma_bootstrap_power_lost() checks it. + */ + if (READ_ONCE(bootstrap->sleep_armed) && pm_resume_via_firmware()) { + bootstrap->power_check = true; + return 0; + } + WRITE_ONCE(bootstrap->irq_held, false); /* Genpd may gate this island; restore configuration, never a command. */ ret = apple_t8140_bootstrap_initialize(bootstrap); if (ret) diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h index b7c4eeedb92c9..713ad546cbedc 100644 --- a/drivers/pci/controller/pcie-apple-t8140-bootstrap.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h @@ -17,6 +17,11 @@ void apple_piodma_bootstrap_guard_end(struct device *supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); +void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed); +int apple_piodma_bootstrap_power_lost(struct device *supplier); +int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev); +void apple_piodma_bootstrap_set_recover(struct device *supplier, void (*recover)(void *data), + void *data); #else static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { @@ -46,5 +51,24 @@ static inline bool apple_piodma_bootstrap_retained(struct device *supplier) { return false; } + +static inline void apple_piodma_bootstrap_set_sleep(struct device *supplier, bool armed) +{ +} + +static inline int apple_piodma_bootstrap_power_lost(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_piodma_bootstrap_radio_stop(struct device *supplier, struct pci_dev *pdev) +{ + return -ENODEV; +} + +static inline void apple_piodma_bootstrap_set_recover(struct device *supplier, + void (*recover)(void *data), void *data) +{ +} #endif #endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index b0f9cc67858f5..e2d1fc7b31534 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -516,6 +517,19 @@ struct apple_pcie { bool neo_config_ready; u8 neo_secondary_bus; struct device *piodma_supplier; + /* What it takes to come back from a deep sleep, if the loader gave it. */ + struct apple_pcie_neo_cold *neo_cold; + struct notifier_block neo_pm_nb; + bool neo_sleep_armed; + struct work_struct neo_recover_work; + bool neo_recover; + /* The functions' state from the last sleep that found them answering */ + struct pci_saved_state *neo_fn_state[2]; + /* Their MSI-X, set aside while they're out of reach */ + bool neo_fn_msix[2]; + bool neo_rc_down; + /* Down already when the sleep began: a failed wake left it unprogrammed */ + bool neo_rc_was_down; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -544,6 +558,8 @@ struct apple_pcie_port { unsigned int link_irqs[2]; u32 saved_intmask; int sid_map_sz; + struct gpio_desc *perst_gpio; + struct gpio_desc *pwren_gpio; int idx; bool started; bool needs_stop; @@ -2683,6 +2699,13 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, u32 stat; int ret; + /* Bringing the link up again, after a deep sleep. */ + if (port->perst_gpio) { + reset = port->perst_gpio; + pwren = port->pwren_gpio; + goto have_gpios; + } + /* * Assert PERST# and configure the pin as output. * The Aquantia AQC113 10GB nic used desktop macs is sensitive to @@ -2720,7 +2743,12 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, else return PTR_ERR(pwren); } + if (!num_aux_resets) { + port->perst_gpio = reset; + port->pwren_gpio = pwren; + } +have_gpios: rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ @@ -3704,6 +3732,10 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); + /* Nothing answers on a host that has lost power and isn't set up again. */ + if (READ_ONCE(pcie->neo_rc_down)) + return NULL; + if (bus->number != cfg->busr.start) { /* Pair with ready publication before consuming the bus number. */ if (!smp_load_acquire(&pcie->neo_config_ready) || @@ -4273,6 +4305,233 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) return ret; } +/* + * What the bootloader did to this host at boot, on the same registers, in its + * order, with its tunables. On this SoC a deep sleep powers all of it off. + */ +struct apple_pcie_neo_cold { + void __iomem *axi2af; + void __iomem *cio3pll; + void __iomem *clkgen; + void __iomem *phy_block; + struct apple_tunable *axi2af_tunable; + struct apple_tunable *cio3pll_tunable; + struct apple_tunable *clkgen_tunable; + struct apple_tunable *phy_tunable; + struct apple_tunable *port_tunable; + struct apple_tunable *rc_tunable; + struct apple_tunable *rc_gen3_tunable; + struct apple_tunable *rc_gen4_tunable; + /* The port's DART, clocked by the port: restored after the port is set up. */ + struct device *dart; +}; + +#define NEO_PHY_CTRL 0x0 +#define NEO_PHY_CTRL_CLK0REQ BIT(0) +#define NEO_PHY_CTRL_CLK1REQ BIT(1) +#define NEO_PHY_CTRL_CLK0ACK BIT(2) +#define NEO_PHY_CTRL_CLK1ACK BIT(3) +#define NEO_PHY_CTRL_RESET BIT(4) +#define NEO_PHY_RUN 0x4 +#define NEO_PHY_READY 0x8 +#define NEO_PHY_CTRL_UP BIT(27) +#define NEO_PHY_SETTLE_US 300 +#define NEO_AXI_EN0 0x104 +#define NEO_AXI_EN1 0x108 +#define NEO_CLK_CTRL 0x0 +#define NEO_CLK_BUSY BIT(31) +#define NEO_CLK_OFF BIT(2) +#define NEO_CLK_START 0x4 +#define NEO_CLK_START_GO BIT(31) +#define NEO_PHY_BLOCK_COMMON 0x4000 +#define NEO_PHY_BLOCK_PHY 0x8000 +#define NEO_PHY_POLL_US 50000 +#define NEO_RC_WRITE_ZERO 0x04 +#define NEO_RC_READY_CFG 0x54 +#define NEO_RC_READY_START 0x50 +#define NEO_RC_READY_STAT 0x58 +#define NEO_RC_POLL_US 250000 +#define NEO_DBI_CAP_LNKCAP 0x7c +#define NEO_DBI_CAP_LNKCAP2 0x9c +#define NEO_DBI_LNKCAP2_SLS GENMASK(6, 1) +#define NEO_DBI_CAP_LNKCTL2 0xa0 +#define NEO_DBI_PORT_LINK_CONTROL 0x710 +#define NEO_DBI_PORT_LINK_MODE GENMASK(21, 16) +#define NEO_DBI_WIDTH_SPEED_CONTROL 0x80c +#define NEO_DBI_LINK_WIDTH GENMASK(12, 8) +#define NEO_DBI_SPEED_CHANGE BIT(17) +#define NEO_DBI_RO_WR 0x8bc +#define NEO_DBI_RO_WR_EN BIT(0) + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001000ff }, + { 0x82c, 0x00000000 }, +}; + +static const struct apple_pcie_reset_reg apple_pcie_neo_port_regs2[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static int apple_pcie_neo_phy_clocks(void __iomem *phy) +{ + u32 stat; + int ret; + + rmw_set(NEO_PHY_CTRL_CLK0REQ, phy + NEO_PHY_CTRL); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK0ACK, + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(NEO_PHY_CTRL_CLK1REQ, phy + NEO_PHY_CTRL); + return readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, stat & NEO_PHY_CTRL_CLK1ACK, + 10, NEO_PHY_POLL_US); +} + +/* + * The part of the block that ANS shares, from power-off in macOS's order, measured on + * one sleep. The sleep driver does it at the wake when ANS is bound, and it can't be + * redone under a running PHY. + */ +static int apple_pcie_neo_common_init(struct apple_pcie *pcie) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *phy = cold->phy_block + NEO_PHY_BLOCK_PHY; + u32 stat, val; + int ret; + + if (readl_relaxed(pcie->base + NEO_RC_READY_STAT) & BIT(0)) + return 0; + + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN0); + rmw_set(BIT(0), cold->axi2af + NEO_AXI_EN1); + apple_tunable_apply(cold->axi2af, cold->axi2af_tunable); + apple_tunable_apply(cold->cio3pll, cold->cio3pll_tunable); + apple_tunable_apply(cold->clkgen, cold->clkgen_tunable); + rmw_set(NEO_CLK_START_GO, cold->clkgen + NEO_CLK_START); + rmw_clear(NEO_CLK_OFF, cold->clkgen + NEO_CLK_CTRL); + ret = readl_relaxed_poll_timeout(cold->clkgen + NEO_CLK_CTRL, stat, !(stat & NEO_CLK_BUSY), + 1, NEO_PHY_POLL_US); + if (ret) + return ret; + apple_tunable_apply(cold->phy_block, cold->phy_tunable); + + ret = apple_pcie_neo_phy_clocks(phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, phy + NEO_PHY_CTRL); + udelay(1); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_CTRL, stat, !(stat & NEO_PHY_CTRL_RESET), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + rmw_set(BIT(0), phy + NEO_PHY_RUN); + udelay(NEO_PHY_SETTLE_US); + rmw_set(BIT(4), phy + NEO_PHY_RUN); + ret = readl_relaxed_poll_timeout(phy + NEO_PHY_READY, stat, stat & BIT(0), + 10, NEO_PHY_POLL_US); + if (ret) + return ret; + + val = readl_relaxed(cold->phy_block + NEO_PHY_BLOCK_COMMON); + writel_relaxed((val & ~GENMASK(1, 0)) | 1, cold->phy_block + NEO_PHY_BLOCK_COMMON); + rmw_set(NEO_PHY_CTRL_UP, phy + NEO_PHY_CTRL); + + writel_relaxed(0, pcie->base + NEO_RC_WRITE_ZERO); + writel_relaxed(0x140, pcie->base + NEO_RC_READY_CFG); + writel_relaxed(1, pcie->base + NEO_RC_READY_START); + + return readl_relaxed_poll_timeout(pcie->base + NEO_RC_READY_STAT, stat, stat & BIT(0), + 100, NEO_RC_POLL_US); +} + +static int apple_pcie_neo_cold_init(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct apple_pcie_neo_cold *cold = pcie->neo_cold; + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + int speed = of_pci_get_max_link_speed(port->np); + u32 stat, val; + int i, ret; + + ret = apple_pcie_neo_common_init(pcie); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs[i].value, + apple_pcie_neo_port_regs[i].offset); + for (i = 0; i < 16; i++) + apple_pcie_port_writel(port, 0, 0x3000 + 4 * i); + for (i = 0; i < 512; i++) + apple_pcie_port_writel(port, 0, pcie->hw->port_msimap + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_neo_port_regs2); i++) + apple_pcie_port_writel(port, apple_pcie_neo_port_regs2[i].value, + apple_pcie_neo_port_regs2[i].offset); + apple_tunable_apply(port->base, cold->port_tunable); + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* PERST# off before the PHY's clocks, as macOS has it coming from power-off. */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, NEO_RC_POLL_US); + if (ret) + return ret; + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, + !(stat & PORT_LINKSTS_BUSY), 100, NEO_RC_POLL_US); + if (ret) + return ret; + + rmw_clear(NEO_PHY_CTRL_CLK0REQ | NEO_PHY_CTRL_CLK1REQ, port->phy + NEO_PHY_CTRL); + ret = apple_pcie_neo_phy_clocks(port->phy); + if (ret) + return ret; + rmw_clear(NEO_PHY_CTRL_RESET, port->phy + NEO_PHY_CTRL); + rmw_set(BIT(9), port->phy + NEO_PHY_CTRL); + rmw_set(BIT(10), port->phy + NEO_PHY_CTRL); + + rmw_set(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + apple_tunable_apply(dbi, cold->rc_tunable); + apple_tunable_apply(dbi, cold->rc_gen3_tunable); + apple_tunable_apply(dbi, cold->rc_gen4_tunable); + if (speed > 0) { + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP); + writel_relaxed((val & ~PCI_EXP_LNKCAP_SLS) | speed, dbi + NEO_DBI_CAP_LNKCAP); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP2); + writel_relaxed((val & ~NEO_DBI_LNKCAP2_SLS) | + FIELD_PREP(NEO_DBI_LNKCAP2_SLS, (1 << speed) - 1), + dbi + NEO_DBI_CAP_LNKCAP2); + writew_relaxed((readw_relaxed(dbi + NEO_DBI_CAP_LNKCTL2) & ~PCI_EXP_LNKCTL2_TLS) | + speed, dbi + NEO_DBI_CAP_LNKCTL2); + rmw_set(NEO_DBI_SPEED_CHANGE, dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + } + val = readl_relaxed(dbi + NEO_DBI_PORT_LINK_CONTROL) & ~NEO_DBI_PORT_LINK_MODE; + writel_relaxed(val | FIELD_PREP(NEO_DBI_PORT_LINK_MODE, 1), + dbi + NEO_DBI_PORT_LINK_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_WIDTH_SPEED_CONTROL) & ~NEO_DBI_LINK_WIDTH; + writel_relaxed(val | FIELD_PREP(NEO_DBI_LINK_WIDTH, 1), dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + val = readl_relaxed(dbi + NEO_DBI_CAP_LNKCAP) & ~PCI_EXP_LNKCAP_MLW; + writel_relaxed(val | FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1), dbi + NEO_DBI_CAP_LNKCAP); + rmw_clear(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + + return 0; +} + static int apple_pcie_attach_domains(struct apple_pcie *pcie) { struct device *dev = pcie->dev; @@ -4311,6 +4570,169 @@ static int apple_pcie_attach_domains(struct apple_pcie *pcie) return 0; } +static void apple_pcie_neo_recover_work(struct work_struct *work); + +/* A radio left unusable by the wake, or its link. POST_SUSPEND runs this once all is thawed. */ +static void apple_pcie_neo_recover(void *data) +{ + struct apple_pcie *pcie = data; + + WRITE_ONCE(pcie->neo_recover, true); + /* Pairs with the barrier at PM_POST_SUSPEND: one side sees the other's write. */ + smp_mb(); + if (!READ_ONCE(pcie->neo_sleep_armed)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); +} + +/* Runs before the freezer. The radios stay bound, and their drivers do their own suspend. */ +static int apple_pcie_neo_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct apple_pcie *pcie = container_of(nb, struct apple_pcie, neo_pm_nb); + + switch (action) { + case PM_SUSPEND_PREPARE: + if (pm_suspend_default_s2idle()) + break; + flush_work(&pcie->neo_recover_work); + pcie->neo_sleep_armed = true; + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, true); + break; + case PM_POST_SUSPEND: + if (!pcie->neo_sleep_armed) + break; + WRITE_ONCE(pcie->neo_sleep_armed, false); + /* Pairs with the barrier in the recovery request. */ + smp_mb(); + apple_piodma_bootstrap_set_sleep(pcie->piodma_supplier, false); + if (READ_ONCE(pcie->neo_recover)) + queue_work(system_unbound_wq, &pcie->neo_recover_work); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +static void __iomem *apple_pcie_neo_map(struct platform_device *pdev, const char *name, + struct resource **res) +{ + *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + if (!*res) + return IOMEM_ERR_PTR(-ENOENT); + + /* These overlap each other and the port PHYs, so they can't be claimed. */ + return devm_ioremap(&pdev->dev, (*res)->start, resource_size(*res)) ?: + IOMEM_ERR_PTR(-ENOMEM); +} + +/* A shared block and the tunable the loader publishes for it. */ +static int apple_pcie_neo_get(struct platform_device *pdev, const char *name, const char *prop, + void __iomem **base, struct apple_tunable **tunable) +{ + struct resource *res; + + *base = apple_pcie_neo_map(pdev, name, &res); + if (IS_ERR(*base)) + return PTR_ERR(*base); + *tunable = devm_apple_tunable_parse(&pdev->dev, pdev->dev.of_node, prop, res); + + return PTR_ERR_OR_ZERO(*tunable); +} + +/* The DART in the host's iommu-map, which reads nothing until the port is set up. */ +static struct device *apple_pcie_neo_dart(struct device *dev) +{ + struct device_node *np = of_parse_phandle(dev->of_node, "iommu-map", 1); + struct platform_device *pdev = np ? of_find_device_by_node(np) : NULL; + int ret; + + of_node_put(np); + if (!pdev) + return NULL; + /* The reference is kept: this host can't be unbound. */ + ret = apple_dart_set_host_resumes(&pdev->dev); + /* The DART resumes, and gates itself, before this host restores it. */ + if (!ret && !device_link_add(dev, &pdev->dev, DL_FLAG_STATELESS)) + ret = -EINVAL; + if (ret) { + put_device(&pdev->dev); + dev_warn(dev, "no deep sleep: its DART can't wait for the port: %d\n", ret); + return NULL; + } + return &pdev->dev; +} + +/* Without all of this the host can't come back, and a deep sleep stays refused. */ +static void apple_pcie_neo_sleep_probe(struct platform_device *pdev, struct apple_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pcie_neo_cold *cold; + struct apple_pcie_port *port; + struct resource *res, *cfg; + int ret; + + port = list_first_entry_or_null(&pcie->ports, struct apple_pcie_port, entry); + cfg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!port || !cfg || !port->perst_gpio || !port->phy) { + dev_warn(dev, "no deep sleep: nothing here took the link up, so no pins to restart it\n"); + return; + } + cold = devm_kzalloc(dev, sizeof(*cold), GFP_KERNEL); + if (!cold) + return; + + if (apple_pcie_neo_get(pdev, "axi2af", "apple,tunable-axi2af", &cold->axi2af, + &cold->axi2af_tunable) || + apple_pcie_neo_get(pdev, "cio3pll", "apple,tunable-cio3pll", &cold->cio3pll, + &cold->cio3pll_tunable) || + apple_pcie_neo_get(pdev, "clkgen", "apple,tunable-clkgen", &cold->clkgen, + &cold->clkgen_tunable)) + goto absent; + cold->phy_block = apple_pcie_neo_map(pdev, "phy-block", &res); + if (IS_ERR(cold->phy_block)) + goto absent; + cold->phy_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-phy", res); + if (IS_ERR(cold->phy_tunable)) + goto absent; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "port0"); + if (!res) + goto absent; + cold->port_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-port", res); + if (IS_ERR(cold->port_tunable)) + goto absent; + cold->rc_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc", cfg); + if (IS_ERR(cold->rc_tunable)) + goto absent; + cold->rc_gen3_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen3", + cfg); + if (IS_ERR(cold->rc_gen3_tunable)) + goto absent; + cold->rc_gen4_tunable = devm_apple_tunable_parse(dev, port->np, "apple,tunable-rc-gen4", + cfg); + if (IS_ERR(cold->rc_gen4_tunable)) + goto absent; + cold->dart = apple_pcie_neo_dart(dev); + if (!cold->dart) + goto absent; + + pcie->neo_cold = cold; + INIT_WORK(&pcie->neo_recover_work, apple_pcie_neo_recover_work); + apple_piodma_bootstrap_set_recover(pcie->piodma_supplier, apple_pcie_neo_recover, pcie); + pcie->neo_pm_nb.notifier_call = apple_pcie_neo_pm_notify; + ret = register_pm_notifier(&pcie->neo_pm_nb); + if (ret) + pcie->neo_cold = NULL; + else + dev_dbg(dev, "deep sleep: radios stay powered, link stopped and retrained\n"); + return; + +absent: + dev_warn(dev, "no deep sleep: the loader didn't publish what this host needs\n"); +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -4525,6 +4947,8 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) dev_err(dev, "radio admission failed=%d; host/supplier retained, no retry\n", ret); + else + apple_pcie_neo_sleep_probe(pdev, pcie); return 0; } @@ -5234,6 +5658,8 @@ static void apple_pcie_stop_for_sleep(struct device *dev) apple_pcie_tunnel_hierarchy_lost(dev); } +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie); + /* * The J716c radio drivers save PCI state without entering D3. The PCI core * keeps the root port in D0 as well. Its host and DART have no switchable @@ -5291,6 +5717,34 @@ static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); + if (pcie->neo_sleep_armed) { + struct apple_pcie_port *port; + + pcie->neo_rc_was_down = READ_ONCE(pcie->neo_rc_down); + /* A port that didn't come back from the last sleep keeps what it saved then. */ + list_for_each_entry(port, &pcie->ports, entry) { + int i; + + apple_pcie_port_disable_irq(port); + if (pcie->neo_rc_was_down) + continue; + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + } + /* The functions too: the core read them through closed config space. */ + if (!pcie->neo_rc_was_down) + apple_pcie_neo_save_functions(pcie); + WRITE_ONCE(pcie->neo_rc_down, true); + /* + * Genpd would gate the host here, and an abandoned sleep then finds + * it unprogrammed with nothing to bring it up. The commit lowers it. + */ + device_set_awake_path(dev); + return 0; + } + /* * The managed radio has no suspend boundary of its own; suspend-to-idle * leaves it in place like any other internal port. @@ -5356,6 +5810,324 @@ static int apple_pcie_reset_for_resume(struct apple_pcie *pcie) return 0; } +/* + * The radio kept its power and firmware, in reset since the commit. Its rail isn't touched: + * cycling it, or the port's own reset, is what a cold start needs. PERST# released, the link + * trains. Device interrupts are off in this phase, so this polls for it. + */ +static int apple_pcie_neo_link_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + usleep_range(1000, 2000); + ret = gpiod_set_value_cansleep(port->perst_gpio, 0); + if (ret) + return ret; + /* 100 ms after PERST# as at boot, and after the link (PCIe r5.0, 6.6.1). */ + msleep(100); + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, + 100, 250000); + if (ret) + return ret; + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, stat & PORT_LINKSTS_UP, + 100, link_up_timeout * 1000); + if (ret) + return ret; + apple_pcie_port_writel(port, BIT(PORT_INT_LINK_UP), PORT_INTSTAT); + msleep(100); + + return 0; +} + +/* + * The PCI core saves the functions again at every sleep, and with the radio down it reads them + * through closed config space. Keep the last state they saved while they answered. + */ +static void apple_pcie_neo_save_functions(struct apple_pcie *pcie) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) + return; + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + kfree(pcie->neo_fn_state[i]); + pcie->neo_fn_state[i] = pci_store_saved_state(fn); + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* With the link back: what the core restores next is that state. */ +static void apple_pcie_neo_load_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int i; + + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (pcie->neo_fn_state[i]) + pci_load_saved_state(fn, pcie->neo_fn_state[i]); + pci_dev_put(fn); + } +} + +/* + * Closed config space keeps the PCI core's restore out of the functions, except for one thing: + * it writes an MSI-X table in their BAR, with no check of its own. With the functions out of + * reach, set that aside so that it has nothing to write; their reach back puts it in place. + */ +static void apple_pcie_neo_msix_gate(struct apple_pcie *pcie, bool reachable) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root, *fn; + int i; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_msix); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + if (reachable && pcie->neo_fn_msix[i]) { + fn->msix_enabled = 1; + pcie->neo_fn_msix[i] = false; + } else if (!reachable && fn->msix_enabled) { + fn->msix_enabled = 0; + pcie->neo_fn_msix[i] = true; + } + pci_dev_put(fn); + } + pci_dev_put(root); +} + +/* + * The link back, and the one engine read that native ECAM needs on it. The root port's + * bus number is restored first, as that read crosses it. Not a firmware load. + */ +static int apple_pcie_neo_radio_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct pci_dev *root; + int ret; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return -ENODEV; + pci_restore_state(root); + ret = apple_pcie_neo_link_up(pcie, port); + if (!ret) + ret = apple_piodma_bootstrap_power_lost(pcie->piodma_supplier); + if (!ret) + ret = apple_piodma_bootstrap_prime(pcie->piodma_supplier, root); + if (!ret) { + apple_piodma_bootstrap_admitted(pcie->piodma_supplier); + apple_pcie_neo_load_functions(pcie, root); + /* The functions' configuration before the flag that says it's there. */ + smp_store_release(&pcie->neo_config_ready, true); + apple_pcie_neo_msix_gate(pcie, true); + } + pci_dev_put(root); + + return ret; +} + +/* + * The rail is the one thing that brings back a chip whose link won't train: macOS's last resort + * for a dead chip too. A cold chip then has its firmware loaded by the probe that follows. + */ +static int apple_pcie_neo_rail_cycle(struct apple_pcie *pcie, struct apple_pcie_port *port, + struct pci_dev *root) +{ + struct pci_dev *fn; + int i, ret; + + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = gpiod_set_value_cansleep(port->pwren_gpio, 0); + if (ret) + return ret; + msleep(pcie->hw->pwren_off_ms); + ret = gpiod_set_value_cansleep(port->pwren_gpio, 1); + if (ret) + return ret; + msleep(pcie->hw->pwren_on_ms); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) + return ret; + /* The PCI core's restore found them gone. Wi-Fi's stop needs its BAR back. */ + for (i = 0; root->subordinate && i < ARRAY_SIZE(pcie->neo_fn_state); i++) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, i)); + if (!fn) + continue; + pci_set_power_state(fn, PCI_D0); + pci_restore_state(fn); + pci_dev_put(fn); + } + return 0; +} + +/* The way a hotplug would. Bluetooth first: Wi-Fi's reset needs it gone. */ +static int apple_pcie_neo_stop_functions(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *fn; + int ret = 0; + + /* Stopping Wi-Fi writes its BAR, which needs the chip to answer. */ + fn = root->subordinate ? pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)) : NULL; + if (fn && !pci_device_is_present(fn)) + ret = -ENODEV; + pci_dev_put(fn); + if (ret) + return ret; + + pci_lock_rescan_remove(); + if (root->subordinate) { + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 1)); + if (fn) { + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + fn = pci_get_slot(root->subordinate, PCI_DEVFN(0, 0)); + if (fn) { + ret = apple_piodma_bootstrap_radio_stop(pcie->piodma_supplier, fn); + if (!ret) + pci_stop_and_remove_bus_device(fn); + pci_dev_put(fn); + } + } + pci_unlock_rescan_remove(); + + return ret; +} + +static void apple_pcie_neo_scan_functions(struct pci_dev *root) +{ + pci_lock_rescan_remove(); + if (root->subordinate) { + pci_scan_child_bus(root->subordinate); + pci_assign_unassigned_bridge_resources(root); + pci_bus_add_devices(root->subordinate); + } + pci_unlock_rescan_remove(); +} + +/* + * Keeping the firmware across a sleep is what's wanted, and this is for when it didn't work: the + * radio lost power, its rings or its IPC. Removing the functions resets the chip, and their + * probes load the firmware again from ROM, as at boot. The link only needs its rail cycled if + * it didn't come back. Otherwise the next sleep tries the whole wake again. + */ +static void apple_pcie_neo_recover_work(struct work_struct *work) +{ + struct apple_pcie *pcie = container_of(work, struct apple_pcie, neo_recover_work); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + struct pci_dev *root; + int ret = 0; + + WRITE_ONCE(pcie->neo_recover, false); + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return; + if (READ_ONCE(pcie->neo_rc_down)) { + ret = -EIO; + goto out; + } + /* Pairs with the release once the functions' configuration is loaded. */ + if (!smp_load_acquire(&pcie->neo_config_ready)) { + dev_warn(pcie->dev, "radio link down after sleep: cycling its rail\n"); + ret = apple_pcie_neo_rail_cycle(pcie, port, root); + if (ret) + goto out; + } + ret = apple_pcie_neo_stop_functions(pcie, root); + if (ret != -ENODEV) + apple_pcie_neo_scan_functions(root); +out: + if (ret) + dev_err(pcie->dev, "radios stay down until the next sleep: %d\n", ret); + else + dev_info(pcie->dev, "radios restarted after sleep, firmware loaded again\n"); + pci_dev_put(root); +} + +/* + * Before the PCI core restores the functions, which need the link. macOS's host bridge + * holds them back the same way until its radio port is enabled. + */ +static int apple_pcie_neo_resume_noirq(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i, ret = 0; + + /* Refused: as before the sleep. A host a failed wake left unprogrammed stays down. */ + if (!pm_resume_via_firmware()) { + if (pcie->neo_rc_was_down) + return 0; + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + WRITE_ONCE(pcie->neo_rc_down, false); + return 0; + } + + list_for_each_entry(port, &pcie->ports, entry) { + /* The pin controller just put it back as it was: released, with no clock yet. */ + ret = gpiod_set_value_cansleep(port->perst_gpio, 1); + if (!ret) + ret = apple_pcie_neo_cold_init(pcie, port); + if (ret) { + dev_err(pcie->dev, "host did not come back after sleep: %d\n", ret); + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + apple_pcie_tunnel_restore_irq_hw(port); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, port->saved_rid2sid[i]); + } + ret = apple_dart_host_resume(pcie->neo_cold->dart); + if (ret) { + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + return ret; + } + WRITE_ONCE(pcie->neo_rc_down, false); + apple_sleep_radio_port_up(); + + /* A failed link stays clocked, and its functions read as gone until the next sleep. */ + port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + ret = apple_pcie_neo_radio_up(pcie, port); + if (ret) { + /* The functions' restore then finds nothing, and their drivers say so. */ + WRITE_ONCE(pcie->neo_config_ready, false); + apple_pcie_neo_msix_gate(pcie, false); + dev_err(pcie->dev, "radios did not come back after sleep: %d\n", ret); + apple_pcie_neo_recover(pcie); + } else { + dev_dbg(pcie->dev, "radio link back after sleep, rail and firmware kept\n"); + } + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); + + return 0; +} + static int apple_pcie_resume_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -5364,6 +6136,8 @@ static int apple_pcie_resume_noirq(struct device *dev) /* The domain's hold for the sleep ends first, whatever follows. */ apple_pcie_release_domains(pcie); + if (pcie->neo_sleep_armed) + return apple_pcie_neo_resume_noirq(pcie); if (pcie->hw->adopt_link) { if (!pcie->link_kept) return 0; diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 36e5afed0227f..09eec7f9d6185 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -564,6 +564,33 @@ static void atcphy_test_update_port_taken(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); } +/* The root hub, once another update that held it has brought the port up again */ +static int atcphy_test_pm_get_after_other(struct apple_atcphy *atcphy, + struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('g'); + guard(mutex)(&atcphy->lock); + atcphy->usb3_port_off = false; + return 0; +} + +/* An update that waited for another one doing the same change leaves the port alone */ +static void atcphy_test_update_after_other(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get_after_other); + steps->mode_port_on = true; + + /* The lanes and the PIPE are in place, the port still owed by the other update */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3_DP); +} + /* * Zeroed memory in place of the core, PIPE handler and USB2 PHY registers: * enough to park the PIPE and power the PHY off, with a PIPE lock request @@ -641,18 +668,25 @@ static struct apple_atcphy *atcphy_test_alloc_failed_handoff(struct kunit *test) return atcphy; } +static int atcphy_test_count_refused_lock(struct apple_atcphy *atcphy); + /* Once dwc3 has stopped, its USB3 PHY power-off parks the PIPE without one */ static void atcphy_test_stopped_power_off_common(struct kunit *test, bool dp) { struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + unsigned int requests = 0; KUNIT_ASSERT_NOT_NULL(test, phy); phy_set_drvdata(phy, atcphy); if (!dp) atcphy->mode = APPLE_ATCPHY_MODE_USB3; + /* The stopped controller can't acknowledge a lock, so none is asked for. */ + test->priv = &requests; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, atcphy_test_count_refused_lock); KUNIT_EXPECT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); KUNIT_EXPECT_EQ(test, atcphy->mode, @@ -1368,6 +1402,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_retry), KUNIT_CASE(atcphy_test_update_unsupported), KUNIT_CASE(atcphy_test_update_port_taken), + KUNIT_CASE(atcphy_test_update_after_other), KUNIT_CASE(atcphy_test_set_port_on), KUNIT_CASE(atcphy_test_host_start), KUNIT_CASE(atcphy_test_stopped_power_off), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 33507b68de52e..fe7eac401a1b2 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -32,13 +32,16 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -558,6 +561,34 @@ enum atcphy_lane_mode { #define USB2PHY_MISCTUNE_APBCLK_GATE_OFF BIT(29) #define USB2PHY_MISCTUNE_REFCLK_GATE_OFF BIT(30) +/* + * The power control of the USB controller the T8140 PHY serves ("pwrctl"). macOS's + * controller driver takes the core into hibernation with it at a system sleep and back + * at the wake, in the sequences below. + */ +#define AUSBC_PWR_CFG 0x00 +#define AUSBC_PWR_CFG_STATE_REQ GENMASK(5, 4) +#define AUSBC_PWR_STATUS 0x04 +#define AUSBC_PWR_STATUS_SLEEP BIT(20) +#define AUSBC_PWR_EVENTS 0x08 +#define AUSBC_PWR_ENABLE 0x10 +#define AUSBC_PWR_CLEAR 0x14 +#define AUSBC_PWR_EVENT_MASK GENMASK(3, 0) +#define AUSBC_PWR_EVENT_D0_ON BIT(0) +#define AUSBC_PWR_EVENT_VDD_OFF BIT(1) +#define AUSBC_PWR_EVENT_WAKE GENMASK(3, 2) +#define AUSBC_PWR_TIMEOUT_US 200000 +/* + * Held around the PHY's USB3 setup at a sleep's entry and wake, with the controller's + * always-on side up: set, acknowledged at once, cleared (measured). + */ +#define AUSBC_PWR_PHY_LOCK 0x18 +#define AUSBC_PWR_PHY_LOCK_ACK 0x1c +#define AUSBC_PWR_PHY_LOCK_US 10000 + +/* The controller's DARTs the PHY saves and restores around its power for a sleep */ +#define ATCPHY_SLEEP_DARTS 4 + enum atcphy_generation { ATCPHY_GENERATION_T8103, ATCPHY_GENERATION_T8122, @@ -757,6 +788,8 @@ enum atcphy_t8132_cio_tunable { * off for a PIPE or lane change and is not powered on again yet * @usb3_port_work: Retries the power-on of @usb3_port_off after a USB failure * @usb3_port_retries: Retries left for @usb3_port_work + * @sleep_finish_work: Ends a sleep for the controller, outside its device lock + * @sleep_finish_kept: Whether the controller kept its state, for @sleep_finish_work * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -858,9 +891,38 @@ struct apple_atcphy { bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; + /* + * The controller sleeps in place for this system sleep, the SuperSpeed side is checked + * against the mux after it, and the controller is in hibernation now. + */ + bool sleep_in_place; + bool sleep_ss_down; + bool hibernated; + int sleep_ret; + /* Going through hibernation failed on this boot: the controller stops from now on. */ + bool sleep_no_keep; + /* + * What macOS's controller driver takes down after the hibernation and brings up at + * the wake: the controller's DARTs, the PHY's power domain and that domain's parent, + * and the fabric bridge bits around the parent. Found at probe; this sleep took them + * down. + */ + bool sleep_power; + bool sleep_powered_down; + /* They didn't come back on: nothing in them may be touched until a reboot. */ + bool sleep_dead; + struct device *sleep_darts[ATCPHY_SLEEP_DARTS]; + unsigned int sleep_ndarts; + struct device_node *pd_usb, *pd_common, *pd_aon; + u32 *bridge_gate; + u32 *bridge_saved; + unsigned int bridge_count; + bool bridge_gated; bool usb3_port_off; struct delayed_work usb3_port_work; unsigned int usb3_port_retries; + struct work_struct sleep_finish_work; + bool sleep_finish_kept; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -898,6 +960,7 @@ struct apple_atcphy { void __iomem *usb2phy_reg; void __iomem *pipehandler; void __iomem *lpdptx; + void __iomem *pwrctl; } regs; struct { @@ -2675,9 +2738,11 @@ static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) /* * The T8140 keeps per-device defaults for its secondary eUSB2 register * bank in the ADT. The power-off path asserts the PHY resets, so apply - * them after every power-on. + * them after every power-on, but not with the controller's core in + * hibernation: see atcphy_resume_noirq(). */ - apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + if (!atcphy->hibernated) + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); } static int atcphy_power_on(struct apple_atcphy *atcphy) @@ -4115,6 +4180,8 @@ static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy); */ static int atcphy_rpm_get(struct apple_atcphy *atcphy) { + if (READ_ONCE(atcphy->sleep_dead)) + return -ENODEV; if (!atcphy->idle_rpm) return 0; return pm_runtime_resume_and_get(atcphy->dev); @@ -4171,6 +4238,26 @@ static void atcphy_rpm_release(void *data) atcphy->rpm_held = false; } +/* What a power-down of the common block takes: the DisplayPort side and a tunnel clock. */ +static void atcphy_forget_common_state(struct apple_atcphy *atcphy) +{ + atcphy->dp_t8122.aux = false; + atcphy->dp_t8122.pll = false; + atcphy->dp_t8122.pairs = 0; + atcphy->dp_t8122.rate = 0; + atcphy->tunnel_clock_on = false; + atcphy->tunnel_rate = 0; + atcphy->tunnel_users = 0; + if (atcphy->hw->has_t8140_dp) { + atcphy->dp_owned_pmas = 0; + atcphy->dp_idle_pmas = 0; + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->aux_power_requested = false; + atcphy->mode_dirty = false; + } +} + /* * A T8122 PHY loses its register state whenever its power domain is switched * off. In system sleep that happens to a domain that no running device holds: @@ -4206,13 +4293,7 @@ static void atcphy_restore_after_pd_off(struct apple_atcphy *atcphy) return; WRITE_ONCE(atcphy->pd_was_off, false); - atcphy->dp_t8122.aux = false; - atcphy->dp_t8122.pll = false; - atcphy->dp_t8122.pairs = 0; - atcphy->dp_t8122.rate = 0; - atcphy->tunnel_clock_on = false; - atcphy->tunnel_rate = 0; - atcphy->tunnel_users = 0; + atcphy_forget_common_state(atcphy); if (!atcphy->dp_only) { _atcphy_dwc3_reset_assert(atcphy); @@ -5690,16 +5771,24 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return ret; } -static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +/* Whether an update to @target_mode has nothing left to do, recording the request if so */ +static bool atcphy_mux_done(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + guard(mutex)(&atcphy->lock); + if (!atcphy_mux_is_noop(atcphy, target_mode)) + return false; + atcphy->typec_mode = target_mode; + return true; +} + +static int atcphy_mux_update_to(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { struct atcphy_usb3_pm pm = {}; - enum atcphy_mode target_mode; bool in_mode, port_on, usb_err = true; int ret; scoped_guard(mutex, &atcphy->lock) { atcphy_restore_after_pd_off(atcphy); - target_mode = atcphy_typec_target_mode(atcphy, state); /* * Only a lane or PIPE change, or a SuperSpeed port still owed * its power, needs the USB3 root hub below. Taking it means @@ -5724,6 +5813,13 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux if (ret) goto out; + /* + * Another update holds the root hub for the whole of its change. One that finished + * while this one waited may have left nothing to do: don't cycle the port again. + */ + if (atcphy_mux_done(atcphy, target_mode)) + goto put; + /* * The order in which dwc3 is taken down and brought up around a PHY * change (see dwc3-apple.c), for the SuperSpeed side only: the @@ -5775,6 +5871,11 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux return ret; } +static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) +{ + return atcphy_mux_update_to(atcphy, atcphy_typec_target_mode(atcphy, state)); +} + static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); @@ -6181,6 +6282,16 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph atcphy->res.usb2phy_reg = res; } + /* Optional: without it the controller stops for a system sleep. */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwrctl"); + if (res) { + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map pwrctl regs"); + atcphy->regs.pwrctl = addr; + } + return 0; } @@ -6203,6 +6314,91 @@ static bool atcphy_usb2_behind_fixed_hub(struct apple_atcphy *atcphy) return hub; } +static void atcphy_put_node(void *np) +{ + of_node_put(np); +} + +static void atcphy_put_device(void *dev) +{ + put_device(dev); +} + +/* + * The pieces of a sleep that keeps the controller's power, from the device tree: the + * bridge bits (apple,axi2af-gate), the PHY's power domain and its parent, and the + * controller's DARTs (its iommus). Without all of them the controller stops for a sleep. + */ +static int atcphy_probe_sleep_power(struct apple_atcphy *atcphy) +{ + struct device *dev = atcphy->dev; + struct device_node *ep __free(device_node) = + of_graph_get_endpoint_by_regs(atcphy->np, 1, -1); + struct device_node *ctrl __free(device_node) = + ep ? of_graph_get_remote_port_parent(ep) : NULL; + int i, n, ret; + + if (!atcphy->regs.pwrctl || !atcphy->res.axi2af || !ctrl) + return 0; + + n = of_property_count_u32_elems(atcphy->np, "apple,axi2af-gate"); + if (n <= 0) + return 0; + if (n % 2) + return dev_err_probe(dev, -EINVAL, "apple,axi2af-gate: not offset and bit pairs\n"); + atcphy->bridge_gate = devm_kcalloc(dev, n, sizeof(u32), GFP_KERNEL); + atcphy->bridge_saved = devm_kcalloc(dev, n / 2, sizeof(u32), GFP_KERNEL); + if (!atcphy->bridge_gate || !atcphy->bridge_saved) + return -ENOMEM; + ret = of_property_read_u32_array(atcphy->np, "apple,axi2af-gate", atcphy->bridge_gate, n); + if (ret) + return ret; + for (i = 0; i < n; i += 2) + if (atcphy->bridge_gate[i] % 4 || + atcphy->bridge_gate[i] + 4 > resource_size(atcphy->res.axi2af)) + return dev_err_probe(dev, -EINVAL, "apple,axi2af-gate: bad offset %#x\n", + atcphy->bridge_gate[i]); + atcphy->bridge_count = n / 2; + + atcphy->pd_usb = of_parse_phandle(atcphy->np, "power-domains", 0); + ret = devm_add_action_or_reset(dev, atcphy_put_node, atcphy->pd_usb); + if (ret) + return ret; + atcphy->pd_common = of_parse_phandle(atcphy->pd_usb, "power-domains", 0); + ret = devm_add_action_or_reset(dev, atcphy_put_node, atcphy->pd_common); + if (ret) + return ret; + if (!atcphy->pd_common) + return 0; + atcphy->pd_aon = of_parse_phandle(atcphy->pd_common, "power-domains", 0); + ret = devm_add_action_or_reset(dev, atcphy_put_node, atcphy->pd_aon); + if (ret) + return ret; + + n = of_count_phandle_with_args(ctrl, "iommus", "#iommu-cells"); + if (n <= 0 || n > ATCPHY_SLEEP_DARTS) + return 0; + for (i = 0; i < n; i++) { + struct of_phandle_args args; + struct platform_device *pdev; + + if (of_parse_phandle_with_args(ctrl, "iommus", "#iommu-cells", i, &args)) + return 0; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!pdev) + return 0; + ret = devm_add_action_or_reset(dev, atcphy_put_device, &pdev->dev); + if (ret) + return ret; + atcphy->sleep_darts[i] = &pdev->dev; + } + atcphy->sleep_ndarts = n; + atcphy->sleep_power = true; + + return 0; +} + /* * A DisplayPort-only instance has no USB controller to reset and no PIPE: put * the PHY in its off state and register the mux, switch and DP PHY. @@ -6368,6 +6564,8 @@ static bool atcphy_has_t8103_usb3_workaround(const struct atcphy_hw *hw, of_device_is_compatible(root, "apple,t8103"); } +static void atcphy_sleep_finish_work(struct work_struct *work); + static int atcphy_probe(struct platform_device *pdev) { struct apple_atcphy *atcphy; @@ -6390,6 +6588,9 @@ static int atcphy_probe(struct platform_device *pdev) mutex_init(&atcphy->lock); platform_set_drvdata(pdev, atcphy); ret = devm_delayed_work_autocancel(dev, &atcphy->usb3_port_work, atcphy_usb3_port_work); + if (ret) + return ret; + ret = devm_work_autocancel(dev, &atcphy->sleep_finish_work, atcphy_sleep_finish_work); if (ret) return ret; @@ -6418,6 +6619,11 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; atcphy->fixed_usb2 = atcphy_usb2_behind_fixed_hub(atcphy); atcphy->typec_mode = APPLE_ATCPHY_MODE_USB2; + if (atcphy->fixed_usb2) { + ret = atcphy_probe_sleep_power(atcphy); + if (ret) + return ret; + } followed = atcphy_probe_pd_notifier(atcphy); /* @@ -6443,6 +6649,604 @@ static int atcphy_probe(struct platform_device *pdev) return 0; } +/* + * A system sleep with the controller kept, as macOS does it on the J700: its host goes to + * Sleep, not Off. xHCI has halted and saved its state by the time this PHY suspends, as + * the controller is its consumer. The PHY then takes the controller's core into + * hibernation through its power control, holds the core's reset without the clamp, and + * puts the eUSB2 PHY in its sleep state: reference clock gated, APB in reset, the link + * left as it is. Then, as macOS's controller driver does next (measured), it saves + * and stops the controller's DARTs, lowers ATC0_USB, gates the fabric bridge and lowers + * ATC0_COMMON. ATC0_USB_AON stays on with the power control, the PIPE handler and the + * USB2 PHY. + * + * At the wake ATC0_COMMON comes back, the bridge, ATC0_USB and the DARTs after it, with + * the common block at its reset values. It's set up in the mode it had again, the + * controller is brought to D0, and xHCI restores its own state. + * + * USB2 and USB3 qualify, without DisplayPort, whose link the common block's power-down + * would end. With USB3 the PIPE stays routed to the USB3 PHY, as macOS leaves it (measured): + * the halted controller doesn't acknowledge a PIPE lock, and the power control acknowledges + * its PHY lock at the wake only with the PIPE on the PHY. The lanes are left as they are: the + * commit takes them down. At the wake the common block is set up again under that lock, with + * the PIPE parked, and the PIPE is routed to the lanes when the controller's + * core is set up, before xHCI starts, with no Type-C mode change and no SuperSpeed port power + * cycle. macOS's raise from Sleep to its saved USB3 mode is also just a handshake and the USB3 + * PHY's power level (static, setUSB3Mode). macOS also stops its PHY's fabric traffic before + * the clock gate; that block isn't known here, and a halted controller has none. + */ +/* + * The DARTs' own callbacks leave them to the PHY once it saved them, and the sleep driver + * knows the domains' words. + */ +static bool atcphy_sleep_power_ready(const struct apple_atcphy *atcphy) +{ + unsigned int i; + + for (i = 0; i < atcphy->sleep_ndarts; i++) + if (!apple_dart_sleep_listed(atcphy->sleep_darts[i])) + return false; + return !apple_sleep_domain_check(atcphy->pd_usb) && + !apple_sleep_domain_check(atcphy->pd_common); +} + +static bool atcphy_can_sleep_in_place(const struct apple_atcphy *atcphy) +{ + enum atcphy_pipehandler_state pipe = atcphy_pipe_state_for(atcphy, atcphy->mode); + + return atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->sleep_power && + atcphy_sleep_power_ready(atcphy) && !atcphy->sleep_no_keep && atcphy->host_active && + (atcphy->mode == APPLE_ATCPHY_MODE_USB2 || atcphy->mode == APPLE_ATCPHY_MODE_USB3) && + !atcphy->mode_dirty && !atcphy->pipe_pending && + (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY || + atcphy->pipe_state == pipe) && !atcphy->usb_off_pending && + !atcphy->dp_owned_pmas && !atcphy->aux_power_requested && !atcphy->tunnel_users; +} + +static void atcphy_usb2_sleep_enter(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); +} + +static void atcphy_usb2_sleep_exit(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); +} + +/* + * macOS's raise from Sleep: D0 asked for, the core's reset released, D0_ON awaited. Returns + * APPLE_ATCPHY_SLEEP_POWER_LOST for a block that lost power, and -ENODEV only if the core + * may be unpowered, when touching its registers would be an SError. + * With @went_off the core's rail is known to have gone off, so only D0_ON will do. Otherwise + * a core no longer reporting SLEEP has left hibernation, and the event alone is missing. + */ +static int atcphy_controller_d0(struct apple_atcphy *atcphy, bool went_off) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 val, status; + int ret; + + /* All zeros: the block lost power in the sleep and was reset, so no D0_ON will come. */ + status = readl(pc + AUSBC_PWR_STATUS); + val = readl(pc + AUSBC_PWR_EVENTS); + if (!went_off && (val & AUSBC_PWR_EVENT_WAKE)) + dev_dbg(atcphy->dev, "USB controller's wake event up at the wake: %#x\n", val); + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, 0); + clear32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + /* As the reset's release does: a block that lost its state may have the clamp on. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + if (!went_off && !status) { + dev_dbg(atcphy->dev, "USB controller lost its power state in the sleep\n"); + return APPLE_ATCPHY_SLEEP_POWER_LOST; + } + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, val, val & AUSBC_PWR_EVENT_D0_ON, 100, + AUSBC_PWR_TIMEOUT_US); + status = readl(pc + AUSBC_PWR_STATUS); + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, + AUSBC_PWR_EVENT_D0_ON | AUSBC_PWR_EVENT_WAKE); + if (!ret) + return 0; + + dev_err(atcphy->dev, "USB controller gave no D0_ON: events %#x, status %#x\n", val, status); + if (!went_off && !(status & AUSBC_PWR_STATUS_SLEEP)) + return 0; + + return -ENODEV; +} + +/* + * A failure refuses this sleep, and the controller stops for the later ones, so that it + * can't hold every other driver's sleep up. sleep_ret says what state the core is in. + */ +static int atcphy_controller_hibernate(struct apple_atcphy *atcphy) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 events, status = 0; + bool went_off; + int ret; + + set32(pc + AUSBC_PWR_CFG, AUSBC_PWR_CFG_STATE_REQ); + ret = readl_poll_timeout(pc + AUSBC_PWR_EVENTS, events, events & AUSBC_PWR_EVENT_VDD_OFF, + 100, AUSBC_PWR_TIMEOUT_US); + went_off = !ret; + if (!ret) { + mask32(pc + AUSBC_PWR_CLEAR, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_VDD_OFF); + /* Armed as macOS arms them. That they drive wake source 70 is inferred. */ + mask32(pc + AUSBC_PWR_ENABLE, AUSBC_PWR_EVENT_MASK, AUSBC_PWR_EVENT_WAKE); + ret = readl_poll_timeout(pc + AUSBC_PWR_STATUS, status, + status & AUSBC_PWR_STATUS_SLEEP, 100, + AUSBC_PWR_TIMEOUT_US); + } + if (ret) { + dev_err(atcphy->dev, "USB controller did not hibernate: events %#x, status %#x\n", + events, status); + atcphy->sleep_no_keep = true; + atcphy->sleep_ret = atcphy_controller_d0(atcphy, went_off); + return ret; + } + + /* The core's reset, without the clamp that only macOS's Off path sets. */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + atcphy_usb2_sleep_enter(atcphy); + dev_dbg(atcphy->dev, "USB controller in hibernation for the sleep\n"); + + return 0; +} + +/* + * The fabric bridge bits, cleared before ATC0_COMMON goes off. The wake sets those that + * were set, as the bridge comes back at its reset values. + */ +static void atcphy_bridge_gate(struct apple_atcphy *atcphy, bool open) +{ + unsigned int i; + + if (open && !atcphy->bridge_gated) + return; + atcphy->bridge_gated = !open; + for (i = 0; i < atcphy->bridge_count; i++) { + void __iomem *reg = atcphy->regs.axi2af + atcphy->bridge_gate[2 * i]; + u32 bits = atcphy->bridge_gate[2 * i + 1]; + + if (open) { + if (atcphy->bridge_saved[i]) + set32(reg, atcphy->bridge_saved[i]); + continue; + } + atcphy->bridge_saved[i] = readl(reg) & bits; + if (atcphy->bridge_saved[i]) + clear32(reg, bits); + } +} + +#define ATCPHY_DOMAIN_TRIES 3 + +static int atcphy_sleep_domain_up(struct device_node *pd) +{ + int i, ret; + + for (i = 0; i < ATCPHY_DOMAIN_TRIES; i++) { + ret = apple_sleep_domain_set(pd, true); + if (!ret) + break; + } + + return ret; +} + +/* + * Back up in macOS's order: ATC0_COMMON, the bridge, ATC0_USB, then the DARTs. -ENODEV if + * a domain didn't come on, when nothing in it may be touched. + */ +static int atcphy_sleep_power_up(struct apple_atcphy *atcphy) +{ + unsigned int i; + int ret; + + atcphy->sleep_powered_down = false; + ret = atcphy_sleep_domain_up(atcphy->pd_common); + if (!ret) { + atcphy_bridge_gate(atcphy, true); + ret = atcphy_sleep_domain_up(atcphy->pd_usb); + } + if (ret) { + dev_err(atcphy->dev, "USB host's power domains not back on: %d, USB stays off\n", + ret); + WRITE_ONCE(atcphy->sleep_dead, true); + atcphy->sleep_no_keep = true; + return -ENODEV; + } + for (i = 0; i < atcphy->sleep_ndarts; i++) + apple_dart_sleep_restore_lost(atcphy->sleep_darts[i], true); + + return 0; +} + +/* + * What macOS's controller driver does right after the hibernation: the DARTs saved and + * stopped, ATC0_USB off, the bridge gated, ATC0_COMMON off, 0.7 to 1.7 s before the CPUs + * stop. A failure leaves everything up again. + */ +static int atcphy_sleep_power_down(struct apple_atcphy *atcphy) +{ + char bridge[48] = ""; + unsigned int i, n; + int ret = 0, len = 0; + + for (n = 0; n < atcphy->sleep_ndarts; n++) { + ret = apple_dart_sleep_prepare(atcphy->sleep_darts[n]); + if (ret) + break; + } + if (ret) { + while (n--) + apple_dart_sleep_restore_lost(atcphy->sleep_darts[n], false); + dev_err(atcphy->dev, "USB host's DARTs not saved for the sleep: %d\n", ret); + return ret; + } + + atcphy->sleep_powered_down = true; + ret = apple_sleep_domain_set(atcphy->pd_usb, false); + if (!ret) { + for (i = 0; i < atcphy->bridge_count; i++) + len += scnprintf(bridge + len, sizeof(bridge) - len, " %#x", + readl(atcphy->regs.axi2af + atcphy->bridge_gate[2 * i])); + atcphy_bridge_gate(atcphy, false); + ret = apple_sleep_domain_set(atcphy->pd_common, false); + } + if (ret) { + dev_err(atcphy->dev, "USB host's power domains not off for the sleep: %d\n", ret); + return ret; + } + dev_info(atcphy->dev, "USB host's power domains off for the sleep, fabric bridge was%s\n", + bridge); + + return 0; +} + +/* + * The common block went off with ATC0_COMMON. Set it up again in the mode it had, USB2 or + * USB3, without the controller's reset or a USB2 PHY power cycle: atcphy_configure() powers + * the USB2 PHY on over the running one, which rewrites only what it holds, so the hub's link + * stays. The host role goes in again as at probe, for a USB2 PHY that lost its registers + * too. The PIPE is parked first, unlocked as the controller's core is in reset, and stays + * parked until the core is set up and routes it. + */ +static int atcphy_common_restore(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode = atcphy->mode; + int ret; + + ret = atcphy_park_pipehandler(atcphy, true); + if (ret) + dev_warn(atcphy->dev, "PIPE not parked after the sleep: %d\n", ret); + atcphy_forget_common_state(atcphy); + ret = atcphy_power_off(atcphy); + if (ret) + dev_warn(atcphy->dev, "PHY did not power down after the sleep: %d\n", ret); + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy_setup_pipehandler(atcphy); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + + ret = atcphy_configure(atcphy, mode); + if (!ret || mode == APPLE_ATCPHY_MODE_USB2) + return ret; + /* USB2 keeps the hub; the mux update after the wake then tries the mode again. */ + dev_warn(atcphy->dev, "mode %d not set up again after the sleep (%d): USB2 for now\n", + mode, ret); + return atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB2); +} + +/* + * The wake's path, and that of an entry that failed after the hibernation: the domains, + * the USB2 PHY, the common block if it went off, then D0. See atcphy_controller_d0(). + */ +static void atcphy_phy_lock(struct apple_atcphy *atcphy, bool lock) +{ + void __iomem *pc = atcphy->regs.pwrctl; + u32 val; + + writel(lock, pc + AUSBC_PWR_PHY_LOCK); + if (readl_poll_timeout(pc + AUSBC_PWR_PHY_LOCK_ACK, val, !!(val & BIT(0)) == lock, 10, + AUSBC_PWR_PHY_LOCK_US)) + dev_warn(atcphy->dev, "PHY lock %s not acknowledged: %#x\n", + lock ? "request" : "release", val); +} + +static int atcphy_sleep_come_back(struct apple_atcphy *atcphy, bool common_lost, bool went_off) +{ + bool kept; + int ret = 0, d0; + + if (atcphy->sleep_powered_down && atcphy_sleep_power_up(atcphy)) + return -ENODEV; + atcphy_usb2_sleep_exit(atcphy); + /* Not all zeros: the always-on side kept power, and the link with it. */ + kept = readl(atcphy->regs.pwrctl + AUSBC_PWR_STATUS); + if (common_lost && kept && atcphy->mode == APPLE_ATCPHY_MODE_USB2) { + /* As macOS: USB2 needs nothing of the core, which a mode change sets up. */ + atcphy_forget_common_state(atcphy); + } else if (common_lost) { + if (kept) + atcphy_phy_lock(atcphy, true); + ret = atcphy_common_restore(atcphy); + if (kept) + atcphy_phy_lock(atcphy, false); + if (ret) + dev_err(atcphy->dev, "PHY not set up again after the sleep: %d\n", ret); + } + + d0 = atcphy_controller_d0(atcphy, went_off); + return d0 < 0 ? d0 : ret ?: d0; +} + +/* + * macOS takes a USB host that fails its raise Off, then starts it cold. The same here, with + * nothing touching the core: clamped in reset, its domains and the always-on one off, then + * on again. The block comes back reset, as after a sleep that took its power, and the host + * starts over. -ENODEV if the core still may be unpowered. + */ +static int atcphy_sleep_recover(struct apple_atcphy *atcphy) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + if (!atcphy->sleep_power || !atcphy->pd_aon) + return -ENODEV; + dev_warn(atcphy->dev, "USB controller didn't come back: power cycling it, as macOS's Off does\n"); + /* Domains that didn't come on get another try first. */ + if (READ_ONCE(atcphy->sleep_dead)) { + WRITE_ONCE(atcphy->sleep_dead, false); + atcphy->sleep_powered_down = true; + if (atcphy_sleep_power_up(atcphy)) + return -ENODEV; + } + _atcphy_dwc3_reset_assert(atcphy); + atcphy_usb2_sleep_enter(atcphy); + ret = atcphy_sleep_power_down(atcphy); + if (!ret) { + ret = apple_sleep_domain_set(atcphy->pd_aon, false); + ret = atcphy_sleep_domain_up(atcphy->pd_aon) ?: ret; + } + if (ret) + dev_warn(atcphy->dev, "USB power cycle incomplete: %d\n", ret); + ret = atcphy_sleep_come_back(atcphy, true, false); + if (ret < 0) + dev_err(atcphy->dev, "USB controller not back after the power cycle: %d\n", ret); + else + dev_info(atcphy->dev, "USB controller back after a power cycle: the host starts over\n"); + + return ret; +} + +static int atcphy_suspend_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + int ret; + + guard(mutex)(&atcphy->lock); + /* Suspend-to-idle keeps every domain on: nothing to do. */ + if (!atcphy->sleep_in_place || !pm_suspend_via_firmware()) + return 0; + /* A mode change since the prepare: the controller would lose its PIPE. */ + if (!atcphy_can_sleep_in_place(atcphy)) { + dev_err(dev, "USB changed for the sleep (mode %d, PIPE %d, host %d): refusing it\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + atcphy->sleep_ret = -EBUSY; + return -EBUSY; + } + ret = atcphy_controller_hibernate(atcphy); + if (ret) { + if (atcphy->sleep_ret == -ENODEV) + atcphy->sleep_ret = atcphy_sleep_recover(atcphy); + return ret; + } + atcphy->hibernated = true; + ret = atcphy_sleep_power_down(atcphy); + if (ret) { + /* No resume callback follows a failed suspend: back now, and stop from now on. */ + atcphy->sleep_no_keep = true; + atcphy->sleep_ret = atcphy_sleep_come_back(atcphy, atcphy->sleep_powered_down, + true); + if (atcphy->sleep_ret == -ENODEV) + atcphy->sleep_ret = atcphy_sleep_recover(atcphy); + atcphy->hibernated = false; + return ret; + } + apple_sleep_usb_set_asleep(true); + + return 0; +} + +/* + * A PHY in a DisplayPort mode doesn't sleep in place, and a sleep that reached its commit + * lowered ATC0_COMMON under it, behind its power domain's back: the lanes and the AUX + * channel went. Have atcphy_resume() set the tracked mode up again, before the display + * controller connects the link anew. + */ +static void atcphy_sleep_lost_dp(struct apple_atcphy *atcphy) +{ + if (!atcphy->hw->has_t8140_dp || !atcphy->hw->restore_after_pd_off || + !pm_resume_via_firmware() || !atcphy_modes[atcphy->mode].enable_dp_aux) + return; + WRITE_ONCE(atcphy->pd_was_off, true); + dev_dbg(atcphy->dev, "DisplayPort mode %d lost in the sleep: setting it up again\n", + atcphy->mode); +} + +static int atcphy_resume_noirq(struct device *dev) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + + guard(mutex)(&atcphy->lock); + if (!atcphy->hibernated) { + atcphy_sleep_lost_dp(atcphy); + return 0; + } + apple_sleep_usb_set_asleep(false); + /* The PHY took ATC0_COMMON down itself, so this goes for a refused sleep too. */ + atcphy->sleep_ret = atcphy_sleep_come_back(atcphy, true, false); + if (atcphy->sleep_ret == -ENODEV) + atcphy->sleep_ret = atcphy_sleep_recover(atcphy); + atcphy->hibernated = false; + if (!atcphy->sleep_ret) + dev_dbg(dev, "USB controller back in D0 after the sleep, always-on side kept, PHY in mode %d\n", + atcphy->mode); + if (atcphy->sleep_ret == APPLE_ATCPHY_SLEEP_POWER_LOST) { + dev_dbg(dev, "USB controller back in D0 after the sleep, always-on side lost, PHY in mode %d\n", + atcphy->mode); + /* + * The secondary bank may have lost its defaults. It's next to the core, maybe + * under the core's rail, and an access with the rail off would be an SError: + * only now. macOS writes nothing there at a wake that kept the always-on side. + */ + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); + } + + return 0; +} + +static struct apple_atcphy *atcphy_from_usb2_phy(struct phy *phy) +{ + if (!phy || phy->ops != &apple_atc_usb2_phy_ops) + return NULL; + return phy_get_drvdata(phy); +} + +/* + * After a sleep in place the PHY goes back to what the mux last asked for. The wake set the + * block up in its mode again and the controller's core routed the PIPE, so a mux update is + * left only if the mux changed during the sleep or that setup failed. + */ +static void atcphy_sleep_ss_up(struct apple_atcphy *atcphy) +{ + enum atcphy_mode mode; + int ret; + + scoped_guard(mutex, &atcphy->lock) { + if (!atcphy->sleep_ss_down || atcphy->sleep_dead) + return; + atcphy->sleep_ss_down = false; + mode = atcphy->typec_mode; + atcphy_restore_after_pd_off(atcphy); + if (atcphy_mux_is_noop(atcphy, mode)) { + dev_dbg(atcphy->dev, "SuperSpeed side already in mode %d after the sleep\n", + mode); + return; + } + } + ret = atcphy_mux_update_to(atcphy, mode); + dev_dbg(atcphy->dev, "SuperSpeed side back after the sleep, mode %d: %d\n", mode, ret); +} + +/* No lane or port change here: anything the sleep needs is done at noirq, in a few ms. */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + + guard(mutex)(&atcphy->lock); + if (atcphy_can_sleep_in_place(atcphy)) { + atcphy->sleep_in_place = true; + atcphy->sleep_ss_down = true; + atcphy->sleep_ret = 0; + return 0; + } + if (atcphy->regs.pwrctl && atcphy->fixed_usb2) + dev_dbg(atcphy->dev, + "USB controller stops for the sleep: mode %d, PIPE %d, host %d\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active); + + return -EBUSY; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_prepare); + +int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + + return atcphy->sleep_in_place ? atcphy->sleep_ret : -ENODATA; +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_result); + +int apple_atcphy_sleep_recover(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + + return atcphy_sleep_recover(atcphy); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_recover); + +/* + * A controller that was restarted after the sleep is set up as at boot and may hibernate + * again. Only the PHY's own failures stop it for the later sleeps; a sleep refused for a + * USB change doesn't. + */ +static void atcphy_sleep_finish(struct apple_atcphy *atcphy, bool kept) +{ + scoped_guard(mutex, &atcphy->lock) { + if (!kept && atcphy->sleep_ret < 0 && atcphy->sleep_ret != -EBUSY) + atcphy->sleep_no_keep = true; + atcphy->sleep_in_place = false; + atcphy->sleep_ret = 0; + } + atcphy_sleep_ss_up(atcphy); +} + +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (atcphy) + atcphy_sleep_finish(atcphy, kept); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish); + +static void atcphy_sleep_finish_work(struct work_struct *work) +{ + struct apple_atcphy *atcphy = container_of(work, struct apple_atcphy, sleep_finish_work); + + atcphy_sleep_finish(atcphy, READ_ONCE(atcphy->sleep_finish_kept)); +} + +/* + * The SuperSpeed side may wait for the controller's device lock, which an unbind of the + * controller holds while it cancels the controller's own work: so this work is the PHY's. + */ +void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (!atcphy) + return; + WRITE_ONCE(atcphy->sleep_finish_kept, kept); + queue_work(system_unbound_wq, &atcphy->sleep_finish_work); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish_async); + +void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy) +{ + struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy); + + if (atcphy) + flush_work(&atcphy->sleep_finish_work); +} +EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish_flush); + static int atcphy_suspend(struct device *dev) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); @@ -6519,7 +7323,10 @@ static void atcphy_shutdown(struct platform_device *pdev) dev_err(atcphy->dev, "USB3 common shutdown failed: %d\n", ret); } -static DEFINE_SIMPLE_DEV_PM_OPS(atcphy_pm_ops, atcphy_suspend, atcphy_resume); +static const struct dev_pm_ops atcphy_pm_ops = { + SYSTEM_SLEEP_PM_OPS(atcphy_suspend, atcphy_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(atcphy_suspend_noirq, atcphy_resume_noirq) +}; static const struct atcphy_hw atcphy_hw_t8103 = { .gen = ATCPHY_GENERATION_T8103, @@ -6566,6 +7373,8 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .has_usb2phy_reg = true, .has_t8140_dp = true, .optional_tunables = true, + .park_pipe_unlocked = true, + .restore_after_pd_off = true, }; static const struct atcphy_hw atcphy_hw_t8142 = { diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 7390d28dfafab..3e8ab5a9b933b 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -26,6 +27,7 @@ #include #include +#include #include "pinctrl-utils.h" #include "core.h" @@ -49,6 +51,11 @@ struct apple_gpio_pinctrl { struct pinctrl_desc pinctrl_desc; struct gpio_chip gpio_chip; u8 irq_group_base; + /* The pins' words across a firmware-driven suspend */ + u32 *saved; + unsigned long *saved_valid; + /* Pins this driver has written. Others belong to someone else. */ + unsigned long *touched; struct apple_gpio_irq_group irqgrps[]; }; @@ -91,6 +98,8 @@ static const struct regmap_config regmap_config = { static void apple_gpio_set_reg(struct apple_gpio_pinctrl *pctl, unsigned int pin, u32 mask, u32 value) { + if (pctl->touched) + set_bit(pin, pctl->touched); regmap_update_bits(pctl->map, REG_GPIO(pin), mask, value); } @@ -594,6 +603,15 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); + /* Only with the regmap reaching the hardware: a cache would hide the loss. */ + if (hw && hw->no_regcache) { + pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); + pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->saved || !pctl->saved_valid || !pctl->touched) + return -ENOMEM; + } + for (i = 0; i < npins; i++) { pins[i].number = i; pins[i].name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "PIN%u", i); @@ -645,10 +663,63 @@ static const struct of_device_id apple_gpio_pinctrl_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_gpio_pinctrl_of_match); +/* + * A firmware-driven suspend powers this block off, and neither the pin core nor + * a consumer sets a pin up again. Slots that fault aren't readable in the regmap, + * so they stay out of this. + */ +static int apple_gpio_pinctrl_suspend_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i; + + if (!pctl->saved || pm_suspend_target_state != PM_SUSPEND_MEM || + !pm_suspend_via_firmware()) + return 0; + + /* + * Only the pins this driver configured. The others belong to the firmware or another + * processor, and writing one back after the wake raises an SError. + */ + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) + if (!regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) + set_bit(i, pctl->saved_valid); + + return 0; +} + +static int apple_gpio_pinctrl_resume_noirq(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i, val, restored = 0; + + if (!pctl->saved || !pm_resume_via_firmware()) + return 0; + + for_each_set_bit(i, pctl->saved_valid, pctl->pinctrl_desc.npins) { + if (!regmap_read(pctl->map, REG_GPIO(i), &val) && val == pctl->saved[i]) + continue; + regmap_write(pctl->map, REG_GPIO(i), pctl->saved[i]); + restored++; + } + dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + + return 0; +} + +static const struct dev_pm_ops apple_gpio_pinctrl_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_gpio_pinctrl_suspend_noirq, + apple_gpio_pinctrl_resume_noirq) +}; + static struct platform_driver apple_gpio_pinctrl_driver = { .driver = { .name = "apple-gpio-pinctrl", .of_match_table = apple_gpio_pinctrl_of_match, + .pm = pm_sleep_ptr(&apple_gpio_pinctrl_pm_ops), .suppress_bind_attrs = true, }, .probe = apple_gpio_pinctrl_probe, diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 39e0972acd0e8..586e3f675db9e 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -1529,6 +1530,40 @@ static const struct apple_pmc_hw apple_pmc_hw_t6050 = { .has_pending = true, }; +/* + * A system sleep that reached the hardware may have taken the enables and floors + * with it. Put back what the driver's books say: these are no-ops if they stayed. + */ +static int apple_pmc_resume_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + unsigned long flags; + unsigned int rail; + int ret = 0, err; + + if (!pm_resume_via_firmware() || !pmc) + return 0; + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails; rail++) { + err = apple_pmc_enable_write(pmc, rail); + ret = ret ?: err; + if (pmc->floor_rails & BIT(rail)) { + err = apple_pmc_floor_apply(pmc, rail); + ret = ret ?: err; + } + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + if (ret) + dev_err(dev, "state not restored after sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_pmc_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmc_resume_early) +}; + static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t6050-pmc", .data = &apple_pmc_hw_t6050 }, { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, @@ -1545,6 +1580,7 @@ static struct platform_driver apple_pmc_driver = { .driver = { .name = "apple-pmc", .of_match_table = apple_pmc_of_match, + .pm = pm_sleep_ptr(&apple_pmc_pm_ops), /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index d6b0cc28b6c2c..403161b5224d4 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -15,11 +15,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #define APPLE_PMGR_RESET BIT(31) @@ -67,6 +69,8 @@ struct apple_pmgr_ps { bool firmware_owned; /* apple,sleep-retained: kept on while the system sleeps */ bool sleep_retained; + /* apple,sleep-driver-owned: the driver behind it brings it back after a sleep */ + bool sleep_driver_owned; /* Never enable automatic power management (apple,no-auto-pm). */ bool no_auto_pm; bool inherited_on; @@ -75,6 +79,10 @@ struct apple_pmgr_ps { bool strict_state; bool wait_parent; u32 reset_time_us; + /* Off in genpd but on in hardware as the system slept: someone else raised it. */ + bool sleep_raised_by_other; + /* That domain's target as the system slept. */ + u32 sleep_target; /* * apple,consumer-switched: firmware-owned, but powered off and on by its * consumer (apple_pmgr_ps_consumer_off()). The state word before @@ -126,6 +134,22 @@ static int apple_pmgr_ps_wait_parent(struct apple_pmgr_ps *ps, u32 *cur) * power-on then fails as it did before, and genpd, which ignores errors in * that phase, keeps the domain requested on. */ +/* Pulse the PS reset of a domain that is off with BUSY latched; it stays off. */ +static void apple_pmgr_ps_unlatch_off(struct apple_pmgr_ps *ps, u32 *cur) +{ + u32 off = *cur & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY | + APPLE_PMGR_PS_RESET | APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_TARGET); + + regmap_write(ps->regmap, ps->offset, off | APPLE_PMGR_PS_RESET); + regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + !(*cur & APPLE_PMGR_BUSY), 1, APPLE_PMGR_PS_SET_TIMEOUT); + regmap_write(ps->regmap, ps->offset, off); + regmap_read(ps->regmap, ps->offset, cur); + if (*cur & APPLE_PMGR_BUSY) + dev_warn_ratelimited(ps->dev, "PS %s: off with BUSY latched: %#x\n", + ps->genpd.name, *cur); +} + static int apple_pmgr_ps_clear_latch(struct apple_pmgr_ps *ps, u32 reg, u32 *cur) { u32 off = (reg & ~APPLE_PMGR_PS_TARGET) | APPLE_PMGR_PS_PWRGATE; @@ -258,6 +282,19 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } } + /* + * A domain can reach off with BUSY latched: on T8140 both ISP CPU cores do, + * every time they go down under Linux. macOS leaves them off with it clear, + * and the sleep engine, which powers them up itself in a wake that keeps DRAM + * up, stalls on a latched one until its watchdog resets the SoC. Only a PS + * reset clears it, and the domain is off, so pulse one as on the power-on + * path (not in a system sleep transition, see apple_pmgr_ps_clear_latch()). + */ + if (!ret && pstate != APPLE_PMGR_PS_ACTIVE && (cur & APPLE_PMGR_BUSY) && + !FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) && !ps->system_sleep && + !ps->wait_parent && !ps->strict_state && !ps->externally_clocked) + apple_pmgr_ps_unlatch_off(ps, &cur); + report: if (ret < 0) dev_err_ratelimited(ps->dev, @@ -355,6 +392,12 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) /* A live coprocessor keeps this domain through system sleep. */ if (ps->sleep_retained && ps->system_sleep) return -EBUSY; + /* + * Its driver took it down for the sleep and raises it in its own order. Gating it + * here would take its parents too, and genpd would raise them all first at the wake. + */ + if (ps->system_sleep && apple_sleep_domain_is_down(ps->dev->of_node)) + return -EBUSY; ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); if (ret) @@ -444,6 +487,86 @@ static int apple_pmgr_ps_resume_early(struct device *dev) return 0; } +/* + * By the noirq phase genpd has powered off what it is going to. A domain it has off + * whose target is still on was raised by a coprocessor's firmware: on T8140 the AOP + * raises audio domains of its own in the always-on PMGR, and the sleep keeps them. + */ +static int apple_pmgr_ps_suspend_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + u32 reg; + + ps->sleep_raised_by_other = false; + if (ps->firmware_owned || ps->inherited_on) + return 0; + + spin_lock_irqsave(&ps->genpd.slock, flags); + if (!regmap_read(ps->regmap, ps->offset, ®)) { + ps->sleep_target = FIELD_GET(APPLE_PMGR_PS_TARGET, reg); + ps->sleep_raised_by_other = ps->genpd.status == GENPD_STATE_OFF && + ps->sleep_target != APPLE_PMGR_PS_PWRGATE; + } + spin_unlock_irqrestore(&ps->genpd.slock, flags); + + return 0; +} + +/* + * A firmware-driven suspend powers the SoC down and its sleep engine leaves the + * domains in states of its own: AUTO_ENABLE cleared, some at clock gating. genpd + * only powers a domain back on if it turned it off itself, so its idea of a + * domain it left on is now wrong. Put each domain back where genpd thinks it is. + * + * On T8140 macOS re-raises 22 domains in the first 5 ms after a wake and lowers + * 30 others (measured under aura). Which of them Linux needs isn't established, + * so this follows genpd's books rather than a list. A domain whose driver took it + * down for the sleep, as the USB host's PHY does, is that driver's to raise, in + * its own order. A domain someone else raised before the sleep, and still at that + * target, stays theirs. + */ +static int apple_pmgr_ps_resume_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + u32 reg; + int ret; + + if (!pm_resume_via_firmware() || ps->firmware_owned || ps->inherited_on || + ps->sleep_driver_owned || apple_sleep_domain_is_down(dev->of_node)) + return 0; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ret = regmap_read(ps->regmap, ps->offset, ®); + if (!ret) { + if (ps->genpd.status == GENPD_STATE_OFF) { + u32 target = FIELD_GET(APPLE_PMGR_PS_TARGET, reg); + + /* + * Lowering an AOP-raised audio domain here leaves the internal + * microphone array's data path dark: the AOP never raises it again. + */ + if (target != APPLE_PMGR_PS_PWRGATE && + !(ps->sleep_raised_by_other && target == ps->sleep_target)) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_PWRGATE, + false); + } else { + bool auto_enable = !ps->no_auto_enable && !ps->no_auto_pm; + + if (FIELD_GET(APPLE_PMGR_PS_TARGET, reg) != APPLE_PMGR_PS_ACTIVE || + (auto_enable && !(reg & APPLE_PMGR_AUTO_ENABLE))) + ret = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + auto_enable); + } + } + spin_unlock_irqrestore(&ps->genpd.slock, flags); + if (ret) + dev_err(dev, "PS %s: resync after sleep failed: %d\n", ps->genpd.name, ret); + + return 0; +} + static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); @@ -694,6 +817,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) "retention workaround needs a consumer switch\n"); } ps->sleep_retained = of_property_read_bool(node, "apple,sleep-retained"); + ps->sleep_driver_owned = of_property_read_bool(node, "apple,sleep-driver-owned"); /* * Only T8132 hands over inherited-on domains with this contract. Older * M3 device trees carry the property with another meaning; keep them @@ -1087,6 +1211,7 @@ MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_noirq, apple_pmgr_ps_resume_noirq) }; static struct platform_driver apple_pmgr_ps_driver = { diff --git a/drivers/pmdomain/apple/pmp-report-test.c b/drivers/pmdomain/apple/pmp-report-test.c index 204e19fbf28ce..179b9e752e3f3 100644 --- a/drivers/pmdomain/apple/pmp-report-test.c +++ b/drivers/pmdomain/apple/pmp-report-test.c @@ -12,6 +12,16 @@ static void pmp_range(u8 *table, u32 id, u32 base, u32 count) put_unaligned_le32(count, table + 8); } +static void pmp_request_word_replay(struct kunit *test) +{ + /* A request an entry withdrew stays off; one nobody here owns stays on. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x31, 0x03, 0x02), 0x32ULL); + /* The found word can be empty: only the requests come back. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0, 0x03, 0x01), 0x01ULL); + /* Nothing requested, nothing known. */ + KUNIT_EXPECT_EQ(test, apple_pmp_request_word(0x10, 0, 0), 0x10ULL); +} + static void pmp_ranges_t6030(struct kunit *test) { u8 table[2 * PMP_PTD_RANGE_SIZE]; @@ -277,6 +287,7 @@ static void pmp_dvfs_words(struct kunit *test) static struct kunit_case apple_pmp_report_cases[] = { KUNIT_CASE(pmp_ranges_t6030), + KUNIT_CASE(pmp_request_word_replay), KUNIT_CASE(pmp_ranges_t8122), KUNIT_CASE(pmp_ranges_t6031), KUNIT_CASE(pmp_ranges_reject_bad_records), diff --git a/drivers/pmdomain/apple/pmp-report-validation.h b/drivers/pmdomain/apple/pmp-report-validation.h index c3e5ad495650b..3165ba33ded3c 100644 --- a/drivers/pmdomain/apple/pmp-report-validation.h +++ b/drivers/pmdomain/apple/pmp-report-validation.h @@ -15,6 +15,15 @@ #define PMP_PTD_RANGE_REQUEST 10 #define PMP_PTD_RANGE_ACK 11 +/* + * The request word for a PMP that is starting: requests made through entries, + * and every other bit as it was found. + */ +static inline u64 apple_pmp_request_word(u64 found, u64 known, u64 shadow) +{ + return (found & ~known) | shadow; +} + /* * A SOC-DEV-DVFS request word: the valid bit, and the level either in the * default five-bit lane at bits 4:0 or, for request classes 0x21 through diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 11178eef8fe7c..fd1c0c0ab0f3c 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -103,8 +103,17 @@ struct apple_pmp_report { /* Requests made through entries: bits requested on, all bits. */ u64 shadow; u64 known; - /* Requests are held in memory until the PMP runs (@defer_requests). */ + /* Bits whose drivers withdraw them at the sleep's commit, with the PMP still running. */ + u64 commit_owned; + /* + * Requests are held in memory until the PMP runs (@defer_requests), and + * while it's stopped for a system sleep. + */ bool defer; + /* The request word when a system sleep stopped the PMP. */ + u64 sleep_request; + /* The last SOC-DEV-DVFS word written to each entry, 0 if none. */ + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; }; @@ -430,7 +439,7 @@ static void apple_pmp_report_ready_work(struct work_struct *work) spin_lock_irqsave(&rep->lock, flags); val = readq(rep->base + offs->tgt_read); - val = (val & ~rep->known) | rep->shadow; + val = apple_pmp_request_word(val, rep->known, rep->shadow); writeq(val, rep->base + offs->tgt_write); want = rep->shadow; rep->defer = false; @@ -444,6 +453,97 @@ static void apple_pmp_report_ready_work(struct work_struct *work) readq(rep->base + offs->status), want); } +/* A report names its PMP where a SoC has more than one kind of link to it. */ +static bool apple_pmp_report_of(const struct apple_pmp_report *rep, const struct device *pmp) +{ + struct device_node *np = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + bool match = !np || np == pmp->of_node; + + of_node_put(np); + return match; +} + +/** + * apple_pmp_report_sleep() - withdraw and hold requests before a system sleep stops the PMP + * @pmp: the PMP device, still running + * @at_commit: the PMP stops at the sleep's commit, so requests whose drivers withdraw them + * there are left set + * + * The devices behind these requests have stopped by now, so the PMP is told before + * it goes. Requests made from here on only update the books, so nothing waits on an + * acknowledgment from a stopped PMP. Bits no entry owns are left alone: nothing here + * knows whether their devices have stopped. + */ +void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 val, withdrawn; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + rep->sleep_request = readq(rep->base + offs->tgt_read); + withdrawn = rep->sleep_request & rep->known & ~(at_commit ? rep->commit_owned : 0); + if (withdrawn) + writeq(rep->sleep_request & ~withdrawn, rep->base + offs->tgt_write); + rep->defer = true; + spin_unlock_irqrestore(&rep->lock, flags); + + if (withdrawn && readq_poll_timeout(rep->base + offs->actual, val, + !(val & withdrawn), 100, 50000)) + dev_warn(rep->dev, "PMP still acknowledges withdrawn requests %#llx (ack %#llx)\n", + withdrawn, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_sleep); + +/** + * apple_pmp_report_wake() - hand the PMP every request after a system sleep + * @pmp: the PMP device, running again + * @lost_power: the sleep reached the hardware, so the report may have lost its + * contents: bits no entry owns come from the word kept at the sleep + * + * The power requests go first and the performance requests after them, the + * order an entry uses when it powers on. + */ +void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ + struct apple_pmp_report *rep; + unsigned long flags; + unsigned int i; + u64 val, want; + + guard(mutex)(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + const struct apple_pmp_report_offsets *offs = rep->offsets; + + if (!apple_pmp_report_of(rep, pmp)) + continue; + spin_lock_irqsave(&rep->lock, flags); + val = lost_power ? rep->sleep_request : readq(rep->base + offs->tgt_read); + writeq(apple_pmp_request_word(val, rep->known, rep->shadow), + rep->base + offs->tgt_write); + for (i = 0; rep->dvfs_shadow && i < offs->dvfs_count; i++) + if (rep->dvfs_shadow[i]) + writeq(rep->dvfs_shadow[i], + rep->base + offs->update + 8 * (offs->dvfs_first + i)); + want = rep->shadow; + rep->defer = false; + spin_unlock_irqrestore(&rep->lock, flags); + + if (want && readq_poll_timeout(rep->base + offs->actual, val, + (val & want) == want, 1000, 1000000)) + dev_warn(rep->dev, "PMP did not acknowledge requests %#llx (ack %#llx)\n", + want, val); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_wake); + /* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) { @@ -1363,6 +1463,12 @@ static int apple_pmp_report_probe(struct platform_device *pdev) } rep->defer = rep->offsets->defer_requests; + if (rep->offsets->dvfs_count) { + rep->dvfs_shadow = devm_kcalloc(dev, rep->offsets->dvfs_count, + sizeof(*rep->dvfs_shadow), GFP_KERNEL); + if (!rep->dvfs_shadow) + return -ENOMEM; + } INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); @@ -1898,7 +2004,13 @@ EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); */ static void apple_pmp_report_dvfs_write(struct apple_pmp_report *rep, u32 entry, u64 val) { - writeq(val, rep->base + rep->offsets->update + 8 * entry); + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + rep->dvfs_shadow[entry - rep->offsets->dvfs_first] = val; + if (!rep->defer) + writeq(val, rep->base + rep->offsets->update + 8 * entry); + spin_unlock_irqrestore(&rep->lock, flags); } /** @@ -1973,6 +2085,136 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo return ret; } +/** + * apple_pmp_report_entry_request() - set or withdraw one entry's request + * @entry: the report entry's node + * @enable: make the request, or withdraw it + * + * For a driver that switches its device's power itself around a system sleep. The + * PMP may use a device whose request is set, so that driver withdraws the request + * before its power goes and makes it again after its power is back. A withdrawn + * request isn't replayed after the PMP restarts. + * + * Return: 0, also with no report bound or the PMP not running, or the error from + * waiting for its acknowledgment. + */ +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable) +{ + struct device_node *report; + struct apple_pmp_report *rep; + u32 id; + int ret = 0; + + if (of_property_read_u32(entry, "reg", &id)) + return -EINVAL; + + report = of_get_parent(entry); + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != report) + continue; + ret = apple_pmp_report_set_state(rep, id, enable, false); + break; + } + rcu_read_unlock(); + of_node_put(report); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_request); + +static struct apple_pmp_report *apple_pmp_report_of_entry(const struct device_node *entry, + u32 *id) +{ + struct device_node *report; + struct apple_pmp_report *rep, *found = NULL; + + if (of_property_read_u32(entry, "reg", id) || *id >= 64) + return NULL; + report = of_get_parent(entry); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == report) { + found = rep; + break; + } + } + of_node_put(report); + + return found; +} + +/** + * apple_pmp_report_entry_commit_owned() - mark an entry withdrawn by its driver at the commit + * @entry: the report entry's node + * @owned: its driver withdraws it at the sleep's commit, or no longer does + * + * For a device whose stop moves to the sleep's commit: a PMP that stops there too keeps the + * request through apple_pmp_report_sleep(), until apple_pmp_report_entry_withdraw_polled(). + * + * Return: 0, or -ENODEV with no report bound for the entry. + */ +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned) +{ + struct apple_pmp_report *rep; + unsigned long flags; + int ret = -ENODEV; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + spin_lock_irqsave(&rep->lock, flags); + if (owned) + rep->commit_owned |= BIT_ULL(id); + else + rep->commit_owned &= ~BIT_ULL(id); + spin_unlock_irqrestore(&rep->lock, flags); + ret = 0; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_commit_owned); + +/** + * apple_pmp_report_entry_withdraw_polled() - withdraw an entry's request at the sleep's commit + * @entry: the report entry's node + * + * Interrupts off, one CPU: the request goes even with requests held for the sleep, and the + * PMP's acknowledgment is polled for while it runs. One already withdrawn returns at once. + * + * Return: 0, also with no report bound, or -ETIMEDOUT with no acknowledgment. + */ +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + struct apple_pmp_report *rep; + unsigned long flags; + u64 bit, val; + int ret = 0; + u32 id; + + rcu_read_lock(); + rep = apple_pmp_report_of_entry(entry, &id); + if (rep) { + bit = BIT_ULL(id); + spin_lock_irqsave(&rep->lock, flags); + rep->known |= bit; + rep->shadow &= ~bit; + val = readq(rep->base + rep->offsets->tgt_read); + if (val & bit) + writeq(val & ~bit, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + if (apple_pmp_report_running(rep)) + ret = readq_poll_timeout_atomic(rep->base + rep->offsets->actual, val, + !(val & bit), 10, 50000); + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_entry_withdraw_polled); + /* * System sleep. An external display processor's request keeps the PMP's * external display state up while nothing is shown, about 0.25 W in diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index a5d949d545029..5bd714eaf1b27 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -1334,6 +1334,24 @@ static void macsmc_power_remove(struct platform_device *pdev) blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); } +/* + * The charge moved while asleep, and an event's uevent that ran before the SMC + * was back found nothing to report. + */ +static void macsmc_power_complete(struct device *dev) +{ + struct macsmc_power *power = dev_get_drvdata(dev); + + if (power->batt) + power_supply_changed(power->batt); + if (power->ac) + power_supply_changed(power->ac); +} + +static const struct dev_pm_ops macsmc_power_pm_ops = { + .complete = pm_sleep_ptr(macsmc_power_complete), +}; + static const struct platform_device_id macsmc_power_id[] = { { .name = "macsmc-power" }, { /* sentinel */ } @@ -1343,6 +1361,7 @@ MODULE_DEVICE_TABLE(platform, macsmc_power_id); static struct platform_driver macsmc_power_driver = { .driver = { .name = "macsmc-power", + .pm = pm_sleep_ptr(&macsmc_power_pm_ops), }, .id_table = macsmc_power_id, .probe = macsmc_power_probe, diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index bae3a9f856c20..a7e6eba6e7910 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2099,10 +2099,14 @@ config RTC_DRV_APPLE_PMU depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP help Read the PMU clock on M1 Pro/Max/Ultra, M2, M4 Pro and A18 Pro - systems and use its alarm to wake the system - from suspend-to-idle. Alarms are limited to one hour; deeper - suspend states are not supported. Continue setting the shared - clock through the SMC RTC. + systems and use its alarm to wake the system from suspend-to-idle, + and on A18 Pro from a firmware-driven suspend too. A18 Pro alarms + reach as far as the PMU's comparator, and the clock can be set here; + elsewhere alarms are limited to one hour and the shared clock is set + through the SMC RTC. + + Say Y where this is the system's first RTC (rtc0), so that it sets + the system time at boot. This driver can be built as the module rtc-apple-pmu. diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 6043d7ee87a84..cd46c9b9a10b1 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #define APPLE_PMU_RTC_ARM BIT(6) #define APPLE_PMU_RTC_ALARM BIT(0) #define APPLE_PMU_RTC_HZ_SHIFT 16 +/* The suspend-to-idle limit, where the alarm is all that wakes the system. */ #define APPLE_PMU_RTC_MAX_DELAY 3600 /* @@ -33,6 +35,8 @@ struct apple_pmu_rtc_hw { u8 control_event_bit; u8 monitor_bit; bool offset_in_seconds; + /* How far ahead an alarm may be, in seconds */ + u32 max_delay; }; static const struct apple_pmu_rtc_hw apple_pmu_rtc_baku = { @@ -43,6 +47,8 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_baku = { .event = 0xf80c, .event_bit = BIT(0), .offset_in_seconds = true, + /* As far as the comparator reaches: macOS's own deep-sleep wakes use it. */ + .max_delay = U32_MAX, }; static const struct apple_pmu_rtc_hw apple_pmu_rtc_abbey = { @@ -53,6 +59,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_abbey = { .event = 0x2000, .event_bit = BIT(1), .monitor_bit = BIT(0), + .max_delay = APPLE_PMU_RTC_MAX_DELAY, }; /* M2 (t8112): Baku's register block with the offset in 1/32768 s. */ @@ -64,6 +71,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = { .event = 0xf80c, .event_bit = BIT(0), .monitor_bit = BIT(0), + .max_delay = APPLE_PMU_RTC_MAX_DELAY, }; /* M1 Pro, Max and Ultra (t600x), offset in 1/32768 s */ @@ -76,6 +84,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_maverick = { .event_bit = BIT(0), .control_event_bit = BIT(1), .monitor_bit = BIT(0), + .max_delay = APPLE_PMU_RTC_MAX_DELAY, }; struct apple_pmu_rtc { @@ -149,6 +158,31 @@ static int apple_pmu_rtc_read_time(struct device *dev, struct rtc_time *time) return 0; } +/* + * Baku's offset is in whole seconds, and this sets it as the SMC RTC does: the + * counter rounded to the nearest second, so the clock is at most half a second off. + */ +static int apple_pmu_rtc_set_time(struct device *dev, struct rtc_time *time) +{ + struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); + __le64 value; + u64 counter; + s64 offset; + int error; + + guard(mutex)(&rtc->transaction); + error = apple_pmu_rtc_counter(rtc, &counter); + if (error) + return error; + offset = rtc_tm_to_time64(time) - + (s64)((counter + BIT(APPLE_PMU_RTC_HZ_SHIFT - 1)) >> APPLE_PMU_RTC_HZ_SHIFT); + if (offset < 0 || offset > RTC_TIMESTAMP_END_9999 - U32_MAX) + return -ERANGE; + value = cpu_to_le64(offset); + error = nvmem_cell_write(rtc->offset, &value, sizeof(value)); + return error < 0 ? error : 0; +} + /* All comparator/control accesses are serialized with the IRQ thread. */ static int apple_pmu_rtc_ack_alarm(struct apple_pmu_rtc *rtc) { @@ -228,7 +262,7 @@ static int apple_pmu_rtc_validate_alarm(struct apple_pmu_rtc *rtc, u64 comparato counter >>= APPLE_PMU_RTC_HZ_SHIFT; if (comparator <= counter) return -ETIME; - if (comparator - counter > APPLE_PMU_RTC_MAX_DELAY) + if (comparator - counter > rtc->hw->max_delay) return -ERANGE; return 0; } @@ -375,6 +409,7 @@ static int apple_pmu_rtc_init_alarm(struct apple_pmu_rtc *rtc) return apple_pmu_rtc_stop_alarm(rtc); } +/* The others' offsets are set through the SMC RTC. */ static const struct rtc_class_ops apple_pmu_rtc_ops = { .read_time = apple_pmu_rtc_read_time, .read_alarm = apple_pmu_rtc_read_alarm, @@ -382,11 +417,26 @@ static const struct rtc_class_ops apple_pmu_rtc_ops = { .alarm_irq_enable = apple_pmu_rtc_alarm_enable, }; +static const struct rtc_class_ops apple_pmu_rtc_seconds_ops = { + .read_time = apple_pmu_rtc_read_time, + .set_time = apple_pmu_rtc_set_time, + .read_alarm = apple_pmu_rtc_read_alarm, + .set_alarm = apple_pmu_rtc_set_alarm, + .alarm_irq_enable = apple_pmu_rtc_alarm_enable, +}; + static int apple_pmu_rtc_prepare(struct device *dev) { struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || rtc->io_failed) + if (rtc->io_failed) + return -EBUSY; + /* + * A firmware-driven suspend programs the wake sources itself, and the + * alarm stays armed in the PMU, which stays powered. Only a sleep state + * without either has no way to bring the alarm back. + */ + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !pm_suspend_via_firmware()) return -EBUSY; return 0; } @@ -401,6 +451,7 @@ static void apple_pmu_rtc_release(void *context) struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); disable_irq(rtc->interrupt); + dev_pm_clear_wake_irq(dev); mutex_lock(&rtc->transaction); if (apple_pmu_rtc_stop_alarm(rtc)) dev_err(dev, "cannot disable RTC alarm on removal\n"); @@ -434,23 +485,27 @@ static int apple_pmu_rtc_probe(struct platform_device *pdev) rtc->clock = devm_rtc_allocate_device(dev); if (IS_ERR(rtc->clock)) return PTR_ERR(rtc->clock); - rtc->clock->ops = &apple_pmu_rtc_ops; + rtc->clock->ops = rtc->hw->offset_in_seconds ? &apple_pmu_rtc_seconds_ops : + &apple_pmu_rtc_ops; rtc->clock->range_min = 0; rtc->clock->range_max = RTC_TIMESTAMP_END_9999; - rtc->clock->alarm_offset_max = APPLE_PMU_RTC_MAX_DELAY; + rtc->clock->alarm_offset_max = rtc->hw->max_delay; clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->clock->features); error = device_init_wakeup(dev, true); if (error) return error; /* - * This interrupt remains live during retained-power s2idle. The - * thread uses the parent PMIC's SPMI regmap to acknowledge the alarm. - * Off-domain wake routing is not established by this driver. + * Let the IRQ core wake the system before the thread acknowledges the + * alarm over the parent PMIC's SPMI regmap. Off-domain wake routing is + * still supplied by the platform sleep implementation. */ error = devm_request_threaded_irq(dev, rtc->interrupt, NULL, apple_pmu_rtc_interrupt, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND | IRQF_ONESHOT, + IRQF_NO_AUTOEN | IRQF_ONESHOT, dev_name(dev), dev); + if (error) + goto disable_wakeup; + error = dev_pm_set_wake_irq(dev, rtc->interrupt); if (error) goto disable_wakeup; error = apple_pmu_rtc_init_alarm(rtc); @@ -466,6 +521,7 @@ static int apple_pmu_rtc_probe(struct platform_device *pdev) return 0; disable_wakeup: + dev_pm_clear_wake_irq(dev); device_init_wakeup(dev, false); return error; } diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 710b19569790b..e4fd7c4c31689 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -96,6 +96,44 @@ config APPLE_PMGR_DCS_KUNIT_TEST Also verify that boot constraints and concurrent endpoints retain their votes, and that suspend restores the last confirmed request. +config APPLE_CLUSTER_PWRGATE + bool "Apple SoC CPU cluster power gates" + depends on OF + default ARCH_APPLE + help + Sets the power-gate enable of each CPU cluster the device tree + lists, early in boot. Without it, a core that the boot loader + started and PMGR never powered on, such as the boot CPU, only + clock-gates in WFI. + +config APPLE_SOC_SLEEP + bool "Apple SoC system sleep at EL2" + default APPLE_T8140_SLEEP + depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) + depends on HOTPLUG_CPU && SUSPEND + select APPLE_TUNABLE + help + Powers the secondary CPUs off and suspends the system to RAM on an + Apple SoC whose kernel runs directly at EL2, with no monitor to ask. + It does nothing unless the boot object asks for it through + /chosen/aurora,s2r-mailbox. + + Only SoCs whose sleep has been measured and tested are supported, + each by its match data: today T8140. On every other SoC the driver + doesn't bind, and s2idle stays the only system sleep. + +config APPLE_SOC_SLEEP_KUNIT_TEST + bool "KUnit tests for the Apple SoC sleep match data" if !KUNIT_ALL_TESTS + depends on APPLE_SOC_SLEEP && KUNIT=y + default KUNIT_ALL_TESTS + help + Checks each matched SoC's sleep data: that only tested SoCs are + matched, that every list names only words of the SoC's full lists, + with no duplicates, and that the lists fit the driver's books. The + tests touch no hardware. + + If unsure, say N. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX @@ -215,8 +253,9 @@ config APPLE_SEP_KUNIT_TEST mailbox message encoding and decoding, the shared-memory registration limits, the rules that keep a keybag's lookup identity apart from the identity the enclave reports, the format and validation of the - Linux host-state store, and the xART guard's decisions and its list - of Linux's records. The tests touch no hardware. + Linux host-state store, the xART guard's decisions and its list + of Linux's records, and how the fingerprint sensor is put to sleep. + The tests touch no hardware. If unsure, say N. @@ -358,3 +397,9 @@ config APPLE_DISPLAY_GATE_KUNIT_TEST endmenu endif + +config APPLE_T8140_SLEEP + bool + transitional + help + Transitional config for APPLE_T8140_SLEEP to APPLE_SOC_SLEEP migration. diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 34ede633d2165..61c96d6048889 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -13,6 +13,11 @@ apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o +obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o + +obj-$(CONFIG_APPLE_SOC_SLEEP) += apple-sleep.o +apple-sleep-y = sleep.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 7f446caf407a8..e1520bcdf66ba 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -63,6 +63,7 @@ use kernel::{ atomic::{ Acquire, Atomic, + Relaxed, Release, // }, Arc, @@ -108,12 +109,16 @@ const AFK_OPC_INIT_UNK: u64 = 0x8c; const AFK_OPC_SEND: u64 = 0xa2; const AFK_OPC_START_ACK: u64 = 0x86; const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_SLEEP_ACK: u64 = 0x87; const AFK_OPC_RECV: u64 = 0x85; const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; const AFK_MSG_START: u64 = 0xa3 << 48; const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +/// Stops an endpoint for a system sleep. It keeps its rings, and `AFK_MSG_START` +/// brings it back. +const AFK_MSG_SLEEP: u64 = 0xa4 << 48; const AFK_RB_BLOCK_STEP: usize = 0x40; const EPIC_TYPE_NOTIFY: u32 = 0; const EPIC_CATEGORY_REPORT: u8 = 0x00; @@ -141,6 +146,8 @@ const EPIC_CALL_TIMEOUT_MS: u32 = 5000; /// Bound on the wait for the shutdown acknowledgments of all endpoints /// together, so that a firmware that stopped answering costs one timeout. const AFK_SHUTDOWN_TIMEOUT_MS: u32 = 5000; +/// Bound on the answer to each stop or start around a system sleep. +const AFK_SLEEP_TIMEOUT_MS: u32 = 1000; // The T8140 AOP boots through a second mailbox, its "setup port", besides the // RTKit one. Its management endpoint runs a HELLO / endpoint map / power @@ -207,6 +214,32 @@ mod aop_ring; /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one /// of the endpoints this driver drives. +/// Iterates over `list`, or over every AFK endpoint if `list` is empty. +struct AfkOrder { + list: &'static [u8], + next: usize, +} + +impl AfkOrder { + fn new(list: &'static [u8]) -> AfkOrder { + AfkOrder { list, next: 0 } + } +} + +impl Iterator for AfkOrder { + type Item = u8; + + fn next(&mut self) -> Option { + let at = self.next; + self.next += 1; + if self.list.is_empty() { + (at < AFK_ENDPOINT_COUNT as usize).then(|| AFK_ENDPOINT_START + at as u8) + } else { + self.list.get(at).copied() + } + } +} + fn afk_endpoint_index(ep: u8) -> Option { ep.checked_sub(AFK_ENDPOINT_START) .filter(|i| *i < AFK_ENDPOINT_COUNT) @@ -470,7 +503,7 @@ impl AFKEndpoint { self.recv_get_buf(client.dev.clone(), rtkit, msg)?; } AFK_OPC_INIT_UNK => {} // no-op - AFK_OPC_START_ACK => {} + AFK_OPC_START_ACK | AFK_OPC_SLEEP_ACK => client.sleep_answered(self.index), AFK_OPC_INIT_RX => { if self.rxbuf.is_some() { dev_err!( @@ -1258,6 +1291,21 @@ struct AopData { call_turn: [Mutex<()>; AFK_ENDPOINT_COUNT as usize], #[pin] ep_shutdown: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// Answers to the stop and start of each endpoint around a system sleep. + #[pin] + ep_sleep: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], + /// The order endpoints stop for a system sleep and start after it. Empty + /// means all started ones in endpoint order. + sleep_endpoints: &'static [u8], + wake_endpoints: &'static [u8], + /// No call starts until the system has resumed. + sleeping: Atomic, + /// RTKit reported the firmware crashed. Nothing restarts it short of a reboot. + crashed: Atomic, + /// The endpoints stopped for this sleep, by index. + sleep_stopped: Atomic, + /// The firmware was told the AP is going to sleep. + ap_idle: Atomic, #[pin] hid_listeners: Mutex>, #[pin] @@ -1392,6 +1440,13 @@ impl AopData { }), call_turn <- pin_init::pin_init_array_from_fn(|_| new_mutex!(())), ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + ep_sleep <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), + sleep_endpoints: cfg.sleep_endpoints, + wake_endpoints: cfg.wake_endpoints, + sleeping: Atomic::new(false), + crashed: Atomic::new(false), + sleep_stopped: Atomic::new(0), + ap_idle: Atomic::new(false), hid_listeners <- new_mutex!(KVec::new()), report_listeners <- new_mutex!(KVec::new()), source_ring <- new_mutex!(None), @@ -1601,6 +1656,127 @@ impl AopData { ret } + fn sleep_answered(&self, endpoint: u8) { + if let Some(index) = afk_endpoint_index(endpoint) { + self.ep_sleep[index].complete(()); + } + } + + /// Sends `msg` to one endpoint and waits for its answer. + fn sleep_switch(&self, endpoint: u8, msg: u64) -> Result<()> { + let index = afk_endpoint_index(endpoint).ok_or(EINVAL)?; + self.ep_sleep[index].reset(); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + rtk.send_message(endpoint, msg)?; + } + self.ep_sleep[index] + .wait_timeout(AFK_SLEEP_TIMEOUT_MS) + .ok_or(ETIMEDOUT) + } + + /// Tells the firmware about the AP's power state. Waits with the RTKit lock + /// dropped, as the answer shares a worker with messages that take it. + fn ap_power(&self, on: bool) -> Result<()> { + let rtk = (*self.rtkit.lock()) + .as_ref() + .map(|rtk| rtk.as_raw()) + .ok_or(ENODEV)?; + // SAFETY: the handle lives until removal, which holds the device lock + // that system sleep also runs under. + to_result(unsafe { bindings::apple_rtkit_ap_power(rtk, on) }) + } + + /// A crashed firmware hears nothing more, so it's left out of every sleep from then on, + /// rather than refusing them all. + fn crashed_skip(&self) -> bool { + self.crashed.load(Acquire) + } + + /// Puts back what `sleep_enter` did, in the order the firmware expects. + fn sleep_undo(&self) { + if self.crashed_skip() { + self.sleep_stopped.store(0, Relaxed); + self.ap_idle.store(false, Relaxed); + self.sleeping.store(false, Release); + return; + } + let stopped = self.sleep_stopped.xchg(0, Relaxed); + let was_idle = self.ap_idle.load(Relaxed); + if self.ap_idle.xchg(false, Relaxed) { + if let Err(e) = self.ap_power(true) { + dev_err!(self.dev, "Wake notice not answered ({:?})", e); + } + } + let order = AfkOrder::new(self.wake_endpoints); + for endpoint in order { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if stopped & (1 << index) == 0 { + continue; + } + if let Err(e) = self.sleep_switch(endpoint, AFK_MSG_START) { + dev_err!(self.dev, "Endpoint {:#04x} did not restart ({:?})", endpoint, e); + } + } + self.sleeping.store(false, Release); + if was_idle { + dev_dbg!(self.dev, "AP on notice sent and endpoints restarted after a sleep\n"); + } + } + + /// The AOP keeps running through a firmware-driven suspend. It stops its + /// endpoints and hears the AP's power state, then the reverse after a wake, + /// with no firmware restart. Measured on T8140, unverified on other SoCs. + fn sleep_enter(&self) -> Result<()> { + if !self.cpu_started.load(Acquire) { + return Ok(()); + } + if self.crashed_skip() { + dev_warn!(self.dev, "firmware crashed: sleeping without it\n"); + self.sleeping.store(true, Release); + return Ok(()); + } + self.sleeping.store(true, Release); + // Any call in progress finishes first; new ones see `sleeping`. + for turn in &self.call_turn { + drop(turn.lock()); + } + + let mut result = Ok(()); + for endpoint in AfkOrder::new(self.sleep_endpoints) { + let Some(index) = afk_endpoint_index(endpoint) else { + continue; + }; + if !self.endpoints[index].lock().started { + continue; + } + result = self.sleep_switch(endpoint, AFK_MSG_SLEEP).inspect_err(|e| { + dev_err!(self.dev, "Endpoint {:#04x} did not stop ({:?})", endpoint, e); + }); + if result.is_err() { + break; + } + self.sleep_stopped + .store(self.sleep_stopped.load(Relaxed) | (1 << index), Relaxed); + } + if result.is_ok() { + // Set first: a notice without an answer may still have been taken. + self.ap_idle.store(true, Relaxed); + result = self.ap_power(false).inspect_err(|e| { + dev_err!(self.dev, "Sleep notice not answered ({:?})", e); + }); + } + if result.is_err() { + self.sleep_undo(); + } else { + dev_dbg!(self.dev, "Endpoints stopped and AP idle notice answered for a sleep\n"); + } + result + } + fn patch_bootargs( &self, aop_mmio: &Mmio<'_, Region>, @@ -1915,8 +2091,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -1935,8 +2118,15 @@ impl AOP for AopData { if self.transport_closing.load(Acquire) { return Err(ENODEV); } + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let _turn = self.take_call_turn(ep_idx); + // The sleep's drain may have passed this endpoint since the check above. + if self.sleeping.load(Acquire) { + return Err(EAGAIN); + } let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -2554,6 +2744,7 @@ impl rtkit::Operations for AopData { } fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + data.crashed.store(true, Release); dev_err!(data.dev, "AOP firmware crashed"); } } @@ -2576,6 +2767,10 @@ struct AopHwConfig { epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. setup_port: bool, + /// The order endpoints stop for a system sleep; empty means endpoint order. + sleep_endpoints: &'static [u8], + /// The order they start after one. + wake_endpoints: &'static [u8], /// Shut the co-processor down on unbind and retain its buffers if the /// shutdown cannot be confirmed. quiesce_on_unbind: bool, @@ -2598,6 +2793,8 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2614,6 +2811,8 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2629,6 +2828,8 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2643,6 +2844,8 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: false, required_endpoints: &[], fw_regions: false, @@ -2662,6 +2865,8 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + sleep_endpoints: &[], + wake_endpoints: &[], quiesce_on_unbind: true, required_endpoints: &[], fw_regions: true, @@ -2685,6 +2890,8 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: true, + sleep_endpoints: &[0x2b, 0x23, 0x22, 0x20, 0x21], + wake_endpoints: &[0x20, 0x21, 0x22, 0x2b, 0x23], quiesce_on_unbind: true, required_endpoints: &[0x20, 0x21, 0x22, 0x23, 0x2b], fw_regions: false, @@ -2692,6 +2899,50 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { halt_on_failed_start: false, }; +/// The data of a bound AOP device. +/// +/// # Safety +/// +/// `dev` is an AOP device between the end of probe and the start of removal. +unsafe fn aop_from_dev<'a>(dev: *mut bindings::device) -> &'a AopData { + // SAFETY: by the caller's contract, drvdata is the `AopDriver` probe set. + let driver = unsafe { &*bindings::dev_get_drvdata(dev).cast::() }; + let data: *const dyn AOP = &*driver.0; + // SAFETY: probe made this `Arc` from an `Arc`. + unsafe { &*data.cast::() } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_suspend_late(dev: *mut bindings::device) -> c_int { + // SAFETY: only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: by this function's contract. + match unsafe { aop_from_dev(dev) }.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_resume(dev: *mut bindings::device) -> c_int { + // SAFETY: by this function's contract. + unsafe { aop_from_dev(dev) }.sleep_undo(); + 0 +} + +static AOP_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend_late: Some(aop_suspend_late), + resume: Some(aop_resume), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + kernel::of_device_table!( OF_TABLE, ::IdInfo, @@ -2712,6 +2963,10 @@ impl platform::Driver for AopDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/apple-cluster-pwrgate.c b/drivers/soc/apple/apple-cluster-pwrgate.c new file mode 100644 index 0000000000000..67d493cff8af6 --- /dev/null +++ b/drivers/soc/apple/apple-cluster-pwrgate.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC CPU cluster power gates + * + * Each CPU cluster has a PMGR word whose top bit lets its cores power down + * in WFI. A core PMGR powers on comes up allowed to, but the boot CPU was + * started by the boot ROM, so it only follows the word, which the boot + * loader may leave clear. Set it for every cluster the device tree lists. + */ + +#include +#include +#include +#include +#include +#include + +#define CLUSTER_PWRGATE_EN BIT(31) + +static const struct of_device_id apple_cluster_pwrgate_of_match[] __initconst = { + { .compatible = "apple,cluster-pwrgate" }, + { } +}; + +/* Before the secondaries start. A core picks the word up at its next WFI. */ +static int __init apple_cluster_pwrgate_init(void) +{ + struct device_node *np; + + for_each_matching_node(np, apple_cluster_pwrgate_of_match) { + void __iomem *gate; + int i; + + if (!of_device_is_available(np)) + continue; + + /* One word per cluster. Shared with the sleep code, so not claimed. */ + for (i = 0; (gate = of_iomap(np, i)); i++) { + u32 was = readl(gate); + + writel(was | CLUSTER_PWRGATE_EN, gate); + pr_debug("%pOF: cluster %d power gate %#x, now %#x\n", np, i, was, + readl(gate)); + iounmap(gate); + } + } + + return 0; +} +early_initcall(apple_cluster_pwrgate_init); diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 946c95d7a46d3..644354f5ddb06 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -164,6 +164,7 @@ pub(crate) struct ControlState { retired: [u64; 4], retired_count: u32, entropy: EntropySink, + wake_seen: u8, } impl ControlState { @@ -181,6 +182,7 @@ impl ControlState { retry_at: None, retry_ms: ENTROPY_RETRY_MIN_MS, }, + wake_seen: 0, } } @@ -218,7 +220,7 @@ impl ControlState { } else { tag + 1 }; - if !self.tag_in_flight(tag) && !self.is_retired(tag) { + if tag != proto::TAG_SLEEP && !self.tag_in_flight(tag) && !self.is_retired(tag) { self.slots[idx] = Slot { used: true, tag, @@ -230,6 +232,45 @@ impl ControlState { Err(EBUSY) } + /// Claims the sleep request's own tag, which the pool never hands out. A + /// request still in flight under it means the last one was never answered. + pub(crate) fn alloc_sleep(&mut self) -> Result { + let tag = proto::TAG_SLEEP; + if self.tag_in_flight(tag) { + return Err(EBUSY); + } + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + self.reclaim_retired(tag); + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + Ok(idx) + } + + /// Whether any request is waiting for its reply. + pub(crate) fn requests_in_flight(&self) -> bool { + self.slots.iter().any(|s| s.used) + } + + /// Forget the wake messages seen, before asking for them. + pub(crate) fn wake_reset(&mut self) { + self.wake_seen = 0; + } + + /// A control message that isn't a reply: note it if it's a wake message. + pub(crate) fn wake_note(&mut self, ty: u8) { + if let Some(n) = proto::WAKE_TYPES.iter().position(|&t| t == ty) { + self.wake_seen |= 1 << n; + } + } + + /// All the wake messages have come. + pub(crate) fn wake_complete(&self) -> bool { + self.wake_seen == (1 << proto::WAKE_TYPES.len()) - 1 + } + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { self.slots[idx].reply.take() } @@ -419,6 +460,55 @@ mod tests { } } + #[test] + fn the_pool_never_hands_out_the_sleep_tag() { + let mut state = ControlState::new(); + for _ in 0..POOL_LEN * 2 { + let (idx, tag) = state.alloc().unwrap(); + assert_ne!(tag, proto::TAG_SLEEP); + state.release(idx); + } + } + + #[test] + fn a_sleep_request_gets_its_own_answer() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + assert!(state.requests_in_flight()); + assert_eq!(state.alloc_sleep(), Err(EBUSY)); + let reply = proto::ControlReply { + tag: proto::TAG_SLEEP, + data_lo: 0, + }; + assert!(matches!(state.deliver(reply), Delivery::Matched)); + assert!(state.take_reply(idx).is_some()); + state.release(idx); + assert!(!state.requests_in_flight()); + } + + #[test] + fn a_sleep_request_can_follow_one_that_timed_out() { + let mut state = ControlState::new(); + let idx = state.alloc_sleep().unwrap(); + state.abandon(idx); + assert!(state.alloc_sleep().is_ok()); + } + + #[test] + fn a_wake_takes_both_messages() { + let mut state = ControlState::new(); + state.wake_reset(); + assert!(!state.wake_complete()); + state.wake_note(0x33); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[0]); + assert!(!state.wake_complete()); + state.wake_note(proto::WAKE_TYPES[1]); + assert!(state.wake_complete()); + state.wake_reset(); + assert!(!state.wake_complete()); + } + #[test] fn a_request_is_answered() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f4f674f548c5d..cad52f94fc81f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -46,6 +46,7 @@ #define APPLE_ASC_MBOX_A2I_RECV1 0x818 #define APPLE_ASC_MBOX_I2A_CONTROL 0x114 +#define APPLE_ASC_MBOX_CONTROL_RECV_ENABLE BIT(0) #define APPLE_ASC_MBOX_I2A_SEND0 0x820 #define APPLE_ASC_MBOX_I2A_SEND1 0x828 #define APPLE_ASC_MBOX_I2A_RECV0 0x830 @@ -228,6 +229,8 @@ struct apple_mbox_hw { /* Status word counts RX occupancy and free TX slots; paired transfers. */ bool counted_fifo; bool ap_initializes_mailboxes; + /* No interrupts: the consumer calls apple_mbox_poll() while it expects traffic. */ + bool polled; unsigned int control_full; unsigned int control_empty; @@ -340,14 +343,11 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); -static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +static void apple_mbox_ascwrap_v6_run_off(struct apple_mbox *mbox) { void __iomem *cpu_control; u32 val; - if (!mbox->ascwrap_v6_cpu_running) - return; - if (mbox->hw->is_ascwrap_v8) { u32 state, ack; @@ -377,12 +377,24 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) val = readl(cpu_control); writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); } +} +static void apple_mbox_ascwrap_v6_stopped_locked(struct apple_mbox *mbox) +{ mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + if (!mbox->ascwrap_v6_cpu_running) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); + apple_mbox_ascwrap_v6_stopped_locked(mbox); +} + /* * A failed resume leaves the runtime PM error set, and runtime PM then fails * every later resume with -EINVAL without retrying the power domain. The @@ -486,6 +498,44 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return NULL; + + return (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_control); + +/* No lock and no runtime PM: the registers alone, as apple_mbox_ascwrap_v6_stop_cpu() has them. */ +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs || + !READ_ONCE(mbox->ascwrap_v6_cpu_running)) + return; + + apple_mbox_ascwrap_v6_run_off(mbox); +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu_atomic); + +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) + ret = -ENODEV; + else if (mbox->ascwrap_v6_cpu_running) + apple_mbox_ascwrap_v6_stopped_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_cpu_stopped); + static bool apple_mbox_is_g17(const struct apple_mbox *mbox) { return mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX || @@ -704,7 +754,8 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, * it unmasked for them. The handler masks it when it fires, * and removal masks it otherwise. */ - t = wait_for_completion_interruptible_timeout( + /* Not for the freezer's signal: a message half sent can break a handshake. */ + t = wait_for_completion_killable_timeout( &mbox->tx_empty, min(remaining, msecs_to_jiffies(APPLE_MBOX_TX_RECHECK_MS))); if (t < 0) @@ -891,6 +942,12 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) return IRQ_HANDLED; } +bool apple_mbox_send_ready(struct apple_mbox *mbox) +{ + return !(readl_relaxed(mbox->regs + mbox->hw->a2i_control) & mbox->hw->control_full); +} +EXPORT_SYMBOL(apple_mbox_send_ready); + int apple_mbox_poll(struct apple_mbox *mbox) { unsigned long flags; @@ -907,6 +964,49 @@ int apple_mbox_poll(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_poll); +/* RAM-only capability check, before a client sends an irreversible request. */ +bool apple_mbox_poll_one_supported(struct apple_mbox *mbox) +{ + return !mbox->hw->is_ascwrap_v8 && !mbox->hw->counted_fifo && + !mbox->hw->has_irq_controls; +} +EXPORT_SYMBOL_GPL(apple_mbox_poll_one_supported); + +/* + * One message with the receive IRQ held. No MMIO follows its callback, including + * a status reread or IRQ ACK. 0 means empty, 1 consumed, <0 error. This narrow + * primitive is not a receive protocol for ACK/counting/PSD-based transports. + */ +int apple_mbox_poll_one(struct apple_mbox *mbox) +{ + struct apple_mbox_msg msg; + unsigned long flags; + u32 control; + int ret = 0; + + spin_lock_irqsave(&mbox->rx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + ret = -ENODEV; + goto out; + } + if (!apple_mbox_poll_one_supported(mbox)) { + ret = -EOPNOTSUPP; + goto out; + } + if (!mbox->rx) + goto out; + control = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + if (!apple_mbox_rx_empty(mbox, control)) { + msg = apple_mbox_read_msg(mbox); + mbox->rx(mbox, msg, mbox->cookie); + ret = 1; + } +out: + spin_unlock_irqrestore(&mbox->rx_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_poll_one); + /* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ static void apple_mbox_v8_rx_poll_work(struct work_struct *work) { @@ -1015,7 +1115,8 @@ int apple_mbox_start(struct apple_mbox *mbox) goto out_unlock; } - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; out_unlock: @@ -1034,7 +1135,8 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) apple_mbox_v8_rx_stop(mbox); } else { mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1048,6 +1150,25 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +/* + * Bit 0 of the receive control word, read and written back. A client whose + * firmware powers itself down clears it when the firmware has answered, and sets + * it before starting the firmware again. That it gates receiving is inferred. + */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&mbox->rx_lock, flags); + val = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + val = enabled ? val | APPLE_ASC_MBOX_CONTROL_RECV_ENABLE : + val & ~APPLE_ASC_MBOX_CONTROL_RECV_ENABLE; + writel_relaxed(val, mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} +EXPORT_SYMBOL_GPL(apple_mbox_set_recv_enabled); + /* * The mailbox's power domain was off while it stayed started (its consumer * powered its co-processor down and up again around it): set the @@ -1069,13 +1190,15 @@ EXPORT_SYMBOL(apple_mbox_restore); void apple_mbox_hold_rx(struct apple_mbox *mbox) { - disable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + disable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_hold_rx); void apple_mbox_release_rx(struct apple_mbox *mbox) { - enable_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + enable_irq(mbox->irq_recv_not_empty); } EXPORT_SYMBOL(apple_mbox_release_rx); @@ -1116,7 +1239,7 @@ static void apple_mbox_quiesce(void *data) if (was_active) { if (apple_mbox_has_rx_poller(mbox)) apple_mbox_v8_rx_stop(mbox); - else + else if (mbox->irq_recv_not_empty >= 0) disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); @@ -1136,7 +1259,8 @@ static void apple_mbox_quiesce(void *data) if (mbox->irq_send_empty >= 0) synchronize_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); - synchronize_irq(mbox->irq_recv_not_empty); + if (mbox->irq_recv_not_empty >= 0) + synchronize_irq(mbox->irq_recv_not_empty); spin_lock_irqsave(&mbox->rx_lock, flags); had_rx = mbox->rx; @@ -1411,19 +1535,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; } - mbox->irq_recv_not_empty = - platform_get_irq_byname(pdev, "recv-not-empty"); - if (mbox->irq_recv_not_empty < 0) - return mbox->irq_recv_not_empty; + if (mbox->hw->polled) { + mbox->irq_recv_not_empty = -ENXIO; + mbox->irq_send_empty = -ENXIO; + } else { + mbox->irq_recv_not_empty = + platform_get_irq_byname(pdev, "recv-not-empty"); + if (mbox->irq_recv_not_empty < 0) + return mbox->irq_recv_not_empty; - /* - * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. - * apple_mbox_send then polls the A2I control register instead. - */ - mbox->irq_send_empty = - platform_get_irq_byname_optional(pdev, "send-empty"); - if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return mbox->irq_send_empty; + /* + * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. + * apple_mbox_send then polls the A2I control register instead. + */ + mbox->irq_send_empty = + platform_get_irq_byname_optional(pdev, "send-empty"); + if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) + return mbox->irq_send_empty; + } mbox->rx_util_min = -1; mbox->rx_util_min_applied = -1; @@ -1432,21 +1561,24 @@ static int apple_mbox_probe(struct platform_device *pdev) mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); - if (!irqname) - return -ENOMEM; + if (mbox->irq_recv_not_empty >= 0) { + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); + if (!irqname) + return -ENOMEM; - if (apple_mbox_is_g17(mbox)) - ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, - apple_mbox_recv_irq, - IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, - irqname, mbox); - else - ret = devm_request_irq(dev, mbox->irq_recv_not_empty, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); - if (ret) - return ret; + if (apple_mbox_is_g17(mbox)) + ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, + apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_NO_SUSPEND, + irqname, mbox); + else + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (ret) + return ret; + } if (mbox->irq_send_empty >= 0) { irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", @@ -1627,6 +1759,24 @@ static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { .has_irq_controls = false, }; +/* The kind of port the secure RTBuddy proxy has, with the AOP setup port's registers. */ +static const struct apple_mbox_hw apple_mbox_secure_rtbuddy_hw = { + .polled = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_T8140_AOP_SETUP_A2I_CONTROL, + .a2i_send0 = APPLE_T8140_AOP_SETUP_A2I_SEND0, + .a2i_send1 = APPLE_T8140_AOP_SETUP_A2I_SEND1, + + .i2a_control = APPLE_T8140_AOP_SETUP_I2A_CONTROL, + .i2a_recv0 = APPLE_T8140_AOP_SETUP_I2A_RECV0, + .i2a_recv1 = APPLE_T8140_AOP_SETUP_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -1750,6 +1900,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6050-j775d-ascwrap-v6", .data = &apple_mbox_t6050_j775d_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, + { .compatible = "apple,secure-rtbuddy-proxy", .data = &apple_mbox_secure_rtbuddy_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* * t6030 gfx-asc mailbox in the boot.bin DTB: it carries no asc-mailbox-v4 fallback so that diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 3b9b47a6d6213..12d40d05b305f 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) #include @@ -39,6 +41,8 @@ struct apple_mcc { unsigned int nr_planes; bool enabled; bool retained; + /* Turned off for a deep sleep, to be turned back on after it. */ + bool sleep_disabled; }; static void apple_mcc_writel(struct apple_mcc *mcc, unsigned int plane, @@ -236,6 +240,43 @@ static void apple_mcc_shutdown(struct platform_device *pdev) apple_mcc_quiesce(mcc); } +/* + * A deep sleep takes the cache's power with the rest of the SoC: stop allocating and write it + * back first, as at shutdown, and allocate again after. Suspend-to-idle keeps it. + */ +static int apple_mcc_suspend_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + + if (!pm_suspend_via_firmware() || !mcc->enabled || mcc->retained) + return 0; + apple_mcc_quiesce(mcc); + mcc->sleep_disabled = true; + + return 0; +} + +static int apple_mcc_resume_noirq(struct device *dev) +{ + struct apple_mcc *mcc = dev_get_drvdata(dev); + int ret; + + if (!mcc->sleep_disabled) + return 0; + mcc->sleep_disabled = false; + /* Only a performance setting: a cache not ready after the wake stays off. */ + ret = apple_mcc_enable(mcc); + mcc->enabled = !ret; + if (ret) + dev_warn(dev, "system level cache allocation stays off after the sleep: %d\n", ret); + + return 0; +} + +static const struct dev_pm_ops apple_mcc_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_mcc_suspend_noirq, apple_mcc_resume_noirq) +}; + static const struct of_device_id apple_mcc_of_match[] = { { .compatible = "apple,t8132-mcc" }, { .compatible = "apple,t8140-mcc" }, @@ -247,6 +288,7 @@ static struct platform_driver apple_mcc_driver = { .driver = { .name = "apple-mcc", .of_match_table = apple_mcc_of_match, + .pm = pm_sleep_ptr(&apple_mcc_pm_ops), }, .probe = apple_mcc_probe, .remove = apple_mcc_remove, diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 3701f08c18487..28ec03e7e9fb7 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -8,6 +8,10 @@ //! is not frozen either, so entropy requests are held over the same window. //! The notifier itself is in `pm_shim.c`; the capture and entropy handling is //! on [`SepData`]. +//! +//! On a firmware-driven suspend the enclave also saves its state and gates its +//! own power domain when asked, as the sleep powers it down, and has to be +//! started again after a wake. See [`SleepGate`] and `SepData::sleep_enter`. use core::sync::atomic::{AtomicPtr, Ordering}; @@ -23,9 +27,120 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; + fn sep_pm_register( + dev: *mut c_void, + event: unsafe extern "C" fn(bool), + sleep: unsafe extern "C" fn() -> c_int, + wake: unsafe extern "C" fn(), + told: unsafe extern "C" fn() -> bool, + ) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; + fn sep_pm_clear_dev_disable(dev: *mut c_void) -> c_int; + fn sep_pm_dart_restore(dev: *mut c_void); + fn sep_pm_commit_prepare( + dev: *mut c_void, + mbox: *mut c_void, + msg0: u64, + msg1: u32, + reply: u64, + mask: u64, + ) -> c_int; + fn sep_pm_commit_finish(dev: *mut c_void) -> c_int; + pub(crate) static sep_pm_ops: kernel::bindings::dev_pm_ops; +} + +/// Who may use the mailbox around a firmware-driven suspend. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SleepGate { + /// Anyone. + Open = 0, + /// Replies to what the enclave asked, and the sleep request itself. + Draining = 1, + /// Nobody until the wake: the enclave is gated or about to be. + Told = 2, + /// Nobody: a previous wake failed. New firmware sleeps refuse until reboot. + Lost = 3, +} + +impl SleepGate { + pub(crate) fn from_u32(v: u32) -> SleepGate { + match v { + 0 => SleepGate::Open, + 1 => SleepGate::Draining, + 2 => SleepGate::Told, + _ => SleepGate::Lost, + } + } + + pub(crate) fn may_request(self) -> bool { + self == SleepGate::Open + } + + pub(crate) fn may_reply(self) -> bool { + matches!(self, SleepGate::Open | SleepGate::Draining) + } + + /// The enclave may be gated, so a wake has to look. + pub(crate) fn needs_wake(self) -> bool { + matches!(self, SleepGate::Told | SleepGate::Lost) + } +} + +/// Stages `request` for the irreversible commit, with RX held until physical wake. +/// Unsupported providers/transports refuse here; device suspend never sends SLEEP. +pub(crate) fn leave_for_commit( + dev: &kernel::device::Device, + mbox: *mut kernel::bindings::apple_mbox, + request: kernel::soc::apple::mailbox::Message, + reply: u64, + mask: u64, +) -> Result<()> { + // SAFETY: `dev` is the bound SEP device and `mbox` its mailbox, which outlives the sleep: + // `commit_finish` takes the request back at the resume. + let rc = unsafe { + sep_pm_commit_prepare( + dev.as_raw().cast(), + mbox.cast(), + request.msg0, + request.msg1, + reply, + mask, + ) + }; + if rc == 0 { + Ok(()) + } else { + Err(Error::from_errno(rc)) + } +} + +/// Finishes a staged request. A sent request requires a real firmware return; +/// the C boundary terminates an impossible sent-without-sleep unwind. `None` if unstaged. +pub(crate) fn commit_finish(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_commit_finish(dev.as_raw().cast()) }; + if rc < 0 { + None + } else { + Some(rc > 0) + } +} + +/// Puts the enclave's DART back if it lost its state, before the doorbell. +pub(crate) fn dart_restore(dev: &kernel::device::Device) { + // SAFETY: `dev` is the bound SEP device. + unsafe { sep_pm_dart_restore(dev.as_raw().cast()) }; +} + +/// Lets a gated enclave start again. Writes only if the bit is set. +pub(crate) fn clear_dev_disable(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device. + let rc = unsafe { sep_pm_clear_dev_disable(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) } /// Attaches every power domain of a multi-domain SEP node and keeps them on @@ -92,7 +207,41 @@ unsafe extern "C" fn pm_event(entering: bool) { sleep_event(&*sep, entering); } +/// Runs `f` on the registered instance. Only for callbacks that +/// `sep_pm_unregister` waits for or that the bound device excludes. +fn with_sep(none: R, f: impl FnOnce(&SepData) -> R) -> R { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return none; + } + // SAFETY: as in `pm_event`. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + f(&*sep) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's suspend callback. +unsafe extern "C" fn pm_sleep() -> c_int { + with_sep(0, |sep| match sep.sleep_enter() { + Ok(()) => 0, + Err(e) => e.to_errno(), + }) +} + +/// # Safety +/// Called by `pm_shim.c` from the device's resume callback. +unsafe extern "C" fn pm_wake() { + with_sep((), |sep| sep.sleep_exit()); +} + +/// # Safety +/// Called by `pm_shim.c`, whether or not the device suspended. +unsafe extern "C" fn pm_told() -> bool { + with_sep(false, |sep| sep.sleep_gate() == SleepGate::Told) +} + pub(crate) fn register(sep: Arc) -> Result<()> { + let dev: *mut c_void = sep.dev.as_raw().cast(); let ptr = sep.into_foreign(); if SEP @@ -111,7 +260,7 @@ pub(crate) fn register(sep: Arc) -> Result<()> { // SAFETY: the shim stores the callback and registers a notifier block // that lives for the module's lifetime. - let rc = unsafe { sep_pm_register(pm_event) }; + let rc = unsafe { sep_pm_register(dev, pm_event, pm_sleep, pm_wake, pm_told) }; if rc != 0 { let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); if !raw.is_null() { @@ -187,6 +336,33 @@ mod tests { } } + #[test] + fn the_gate_lets_replies_out_until_the_enclave_is_told() { + assert!(SleepGate::Open.may_request()); + assert!(SleepGate::Open.may_reply()); + assert!(!SleepGate::Draining.may_request()); + assert!(SleepGate::Draining.may_reply()); + assert!(!SleepGate::Told.may_request()); + assert!(!SleepGate::Told.may_reply()); + assert!(!SleepGate::Lost.may_request()); + assert!(!SleepGate::Lost.may_reply()); + } + + #[test] + fn only_a_told_or_lost_enclave_needs_waking() { + assert!(!SleepGate::Open.needs_wake()); + assert!(!SleepGate::Draining.needs_wake()); + assert!(SleepGate::Told.needs_wake()); + assert!(SleepGate::Lost.needs_wake()); + } + + #[test] + fn the_gate_round_trips() { + for gate in [SleepGate::Open, SleepGate::Draining, SleepGate::Told, SleepGate::Lost] { + assert_eq!(SleepGate::from_u32(gate as u32), gate); + } + } + #[test] fn going_to_sleep_holds_entropy_then_captures() { let calls = Calls::new(); diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 76846ad5aaac3..c1d415c0bd744 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -3,18 +3,51 @@ * System sleep for the SEP driver: the notifier only forwards "about to sleep" * and "back from sleep" to Rust, which ends any Touch ID capture before the * system suspends (see pm.rs), and the enclave's power domains are kept on - * across sleep. + * across sleep. The device's suspend and resume callbacks are the firmware + * suspend's: the enclave is told, and gates its own domain, whose state word is + * read here. On T8140 at EL2 the sleep driver sends the request at the + * irreversible commit boundary. Only errors before send may refuse. */ #include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include +#include + #include "shim.h" +/* The state word of a PMGR power domain, as pmgr-pwrstate has it. */ +#define SEP_PS_DEV_DISABLE BIT(10) +#define SEP_PS_ACTUAL GENMASK(7, 4) + +/* One-shot precommit test: no SLEEP request or commit coprocessor stop. */ +static bool sleep_refuse_before_send; +module_param(sleep_refuse_before_send, bool, 0644); +MODULE_PARM_DESC(sleep_refuse_before_send, + "Test only: refuse the next prepared deep sleep before sending SEP SLEEP"); + static void (*sep_pm_event)(bool entering); +static int (*sep_pm_sleep)(void); +static void (*sep_pm_wake)(void); +static bool (*sep_pm_told)(void); +/* In the device's .suspend, the only caller that can leave the request to the sleep. */ +static bool sep_pm_suspending; static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) @@ -46,15 +79,420 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(void (*event)(bool entering)) +/* The enclave's own domain is the first one its node lists. */ +static int sep_pm_domain(struct device *dev, struct regmap **regmap, u32 *offset) +{ + struct device_node *np; + int ret; + + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + + *regmap = syscon_node_to_regmap(np->parent); + ret = PTR_ERR_OR_ZERO(*regmap); + if (!ret) + ret = of_property_read_u32(np, "reg", offset); + of_node_put(np); + + return ret; +} + +/* + * Returns 0 once the enclave has gated itself, 1 if the deadline expired, or a + * negative observation error (including -ENOENT if there is no domain to read). + * Keep a read's -ETIMEDOUT distinct from the polling deadline. DEV_DISABLE is + * the evidence: a running enclave idles at actual state 0 with auto-enable on. + */ +int sep_pm_wait_gated(void *ptr, unsigned int timeout_ms) +{ + struct regmap *regmap; + u32 offset, val; + int ret, poll_ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + poll_ret = read_poll_timeout(regmap_read, ret, ret || (val & SEP_PS_DEV_DISABLE), + 1000, timeout_ms * USEC_PER_MSEC, false, + regmap, offset, &val); + if (ret) + return ret; + + return !!poll_ret; +} + +int sep_pm_clear_dev_disable(void *ptr) +{ + struct regmap *regmap; + u32 offset, val; + int ret; + + ret = sep_pm_domain(ptr, ®map, &offset); + if (ret) + return ret; + + ret = regmap_read(regmap, offset, &val); + if (ret || !(val & SEP_PS_DEV_DISABLE)) + return ret; + + return regmap_write(regmap, offset, val & ~SEP_PS_DEV_DISABLE); +} + +/* The enclave's DART, the first one its node lists, with a reference held. */ +static struct device *sep_pm_dart(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + + np = of_parse_phandle(dev->of_node, "iommus", 0); + if (!np) + return NULL; + pdev = of_find_device_by_node(np); + of_node_put(np); + + return pdev ? &pdev->dev : NULL; +} + +void sep_pm_dart_restore(void *ptr) +{ + struct device *dart = sep_pm_dart(ptr); + + if (dart) { + apple_dart_sleep_restore(dart); + put_device(dart); + } +} + +#define SEP_PM_POLL_US 20 +#define SEP_PM_TIMEOUT_US (5000 * USEC_PER_MSEC) + +/* + * .suspend only stages this request and holds RX. The platform sends it after + * the last legitimate refusal. Once sent, only physical sleep/wake is supported; + * a failed/aborted SLEEP is fatal as on the native path, not an in-place restart. + */ +static struct sep_pm_commit { + struct device *dev; + struct device *dart; + struct apple_mbox *mbox; + void __iomem *ps; + struct apple_mbox_msg request; + u64 reply, reply_mask; + void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); + void *cookie; + bool registered, sent, answered, gate_valid; + u8 reply_result; + unsigned int answer_us; + int gate_ret; + u32 state; +} sep_pm_commit; + +/* The answer stops here. Anything else goes to the driver, which reads it after the wake. */ +static void sep_pm_commit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct sep_pm_commit *c = cookie; + + if (c->sent && (msg.msg0 & c->reply_mask) == c->reply) { + /* Control reply result is byte3; the match mask is below it. */ + c->reply_result = (msg.msg0 >> 24) & 0xff; + c->answered = true; + return; + } + if (c->rx) + c->rx(mbox, msg, c->cookie); +} + +static void sep_pm_commit_take_rx(struct sep_pm_commit *c, bool take) +{ + unsigned long flags; + + spin_lock_irqsave(&c->mbox->rx_lock, flags); + if (take) { + c->rx = c->mbox->rx; + c->cookie = c->mbox->cookie; + c->mbox->rx = sep_pm_commit_rx; + c->mbox->cookie = c; + } else { + c->mbox->rx = c->rx; + c->mbox->cookie = c->cookie; + } + spin_unlock_irqrestore(&c->mbox->rx_lock, flags); +} + +/* Nonzero returns are permitted only before the request was sent. */ +static int sep_pm_commit_stop(void *data) +{ + struct sep_pm_commit *c = data; + unsigned int us; + bool prepared = false; + int ret = 0; + + c->sent = false; + c->answered = false; + c->answer_us = 0; + c->reply_result = 0xff; + c->gate_ret = -EINVAL; + c->gate_valid = false; + c->state = 0; + + sep_pm_commit_take_rx(c, true); + /* No SLEEP sent yet: drain what was already waiting. */ + ret = apple_mbox_poll(c->mbox); + if (ret < 0) + goto out; + for (us = 0; !apple_mbox_send_ready(c->mbox); us += SEP_PM_POLL_US) { + if (us >= SEP_PM_TIMEOUT_US) { + ret = -ETIMEDOUT; + goto out; + } + udelay(SEP_PM_POLL_US); + } + ret = apple_dart_sleep_prepare(c->dart); + if (ret) + goto out; + prepared = true; + /* The PM core's earlier check precedes the potentially long preparation. */ + if (pm_wakeup_pending()) { + ret = -EBUSY; + goto out; + } + if (xchg(&sleep_refuse_before_send, false)) { + dev_info(c->dev, "test: precommit refusal before SEP SLEEP; request not sent\n"); + ret = -ECANCELED; + goto out; + } + /* Commit decision: a new wake from here must finish physical sleep/wake. */ + ret = apple_mbox_send(c->mbox, c->request, true); + if (ret) + goto out; + c->sent = true; + while (!c->answered && !((c->state = readl(c->ps)) & SEP_PS_DEV_DISABLE) && + c->answer_us < SEP_PM_TIMEOUT_US) { + ret = apple_mbox_poll_one(c->mbox); + if (ret < 0) + break; + if (!c->answered) { + udelay(SEP_PM_POLL_US); + c->answer_us += SEP_PM_POLL_US; + } + } +out: + /* Callback ownership is RAM only; RX remains held until a real wake. */ + sep_pm_commit_take_rx(c, false); + if (!c->sent) { + if (prepared) + apple_dart_sleep_restore(c->dart); + return ret; + } + /* Preserve the failure reason without another hardware query. */ + if (ret >= 0 && !c->answered) + ret = c->state & SEP_PS_DEV_DISABLE ? -EHOSTDOWN : -ETIMEDOUT; + if (c->answered && c->reply_result) + ret = -EREMOTEIO; + /* Native cmsgSLEEP errors/abort/timeout are fatal, never cancellation. */ + if (ret < 0) + panic("SEP SLEEP failed after send: ret %d reply_valid %d result %#x state %#x", + ret, c->answered, c->reply_result, c->state); + /* + * Native success-only receive close. No mailbox access follows it. The + * held RX IRQ adapts native workloop exclusion; access-until-close is + * the native contract, not proved atomic by the preceding PMGR sample. + */ + apple_mbox_set_recv_enabled(c->mbox, false); + c->gate_ret = readl_poll_timeout_atomic(c->ps, c->state, c->state & SEP_PS_DEV_DISABLE, + SEP_PM_POLL_US, SEP_PM_TIMEOUT_US); + c->gate_valid = true; + if (c->gate_ret) + panic("SEP SLEEP did not gate after send: %d state %#x", c->gate_ret, c->state); + + return 0; +} + +/* + * From .suspend, with the enclave drained and its receive interrupt held: leaves the + * request in @msg0 and @msg1 to the irreversible commit. Its answer is the message + * whose @mask bits read @reply. Unsupported paths must refuse without sending. + */ +int sep_pm_commit_prepare(void *ptr, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask) +{ + struct sep_pm_commit *c = &sep_pm_commit; + struct device *dev = ptr; + struct device_node *np; + struct resource res; + u32 offset; + int ret; + + /* Check capability before resources that legacy providers need not have. */ + if (!READ_ONCE(sep_pm_suspending) || !IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) || + !is_kernel_in_hyp_mode()) + return -EOPNOTSUPP; + if (c->registered) + return -EBUSY; + /* Reject ACK/counting/PSD transports before any request or MMIO. */ + if (!apple_mbox_poll_one_supported(mbox)) + return -EOPNOTSUPP; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!np) + return -ENOENT; + ret = of_property_read_u32(np, "reg", &offset); + if (!ret) + ret = np->parent ? of_address_to_resource(np->parent, 0, &res) : -EINVAL; + of_node_put(np); + if (ret) + return ret; + if (offset + 4 > resource_size(&res)) + return -EINVAL; + + c->dart = sep_pm_dart(dev); + if (!c->dart || !dev_get_drvdata(c->dart)) { + ret = -ENODEV; + goto err; + } + c->ps = ioremap(res.start + offset, 4); + if (!c->ps) { + ret = -ENOMEM; + goto err; + } + c->dev = dev; + c->mbox = mbox; + c->request.msg0 = msg0; + c->request.msg1 = msg1; + c->reply = reply; + c->reply_mask = mask; + c->sent = false; + /* Resume may follow a pre-tell abort: observations belong to this attempt. */ + c->answered = false; + c->answer_us = 0; + c->reply_result = 0xff; + c->gate_ret = -EINVAL; + c->gate_valid = false; + c->state = 0; + ret = apple_sleep_enclave_register(dev, sep_pm_commit_stop, c); + if (ret) + goto err; + c->registered = true; + + return 0; + +err: + if (c->ps) + iounmap(c->ps); + c->ps = NULL; + if (c->dart) + put_device(c->dart); + c->dart = NULL; + return ret; +} + +/* After the sleep, refused or not: 1 if the request went out, 0 if not, <0 if none was left. */ +int sep_pm_commit_finish(void *ptr) +{ + struct sep_pm_commit *c = &sep_pm_commit; + + if (!c->registered || c->dev != ptr) + return -ENOENT; + if (c->sent && !pm_resume_via_firmware()) + panic("SEP SLEEP was sent without a physical sleep/wake"); + apple_sleep_enclave_unregister(c->dev); + c->registered = false; + dev_dbg(c->dev, + "sleep commit finished: sent=%d reply_valid=%d result=%#x answer_us=%u gate_valid=%d gate_ret=%d state=%#x\n", + c->sent, c->answered, c->reply_result, c->answer_us, + c->gate_valid, c->gate_ret, c->state); + iounmap(c->ps); + c->ps = NULL; + put_device(c->dart); + c->dart = NULL; + + return c->sent; +} + +/* + * Only a firmware-driven suspend tells the enclave anything, so every other sleep + * stays as it was. Refusing is safe: the PM core runs the resume callback of a + * device whose suspend had passed when a later one failed, and sleep_enter undoes + * what it did if its own step fails. No SLEEP is sent during device suspend. + */ +static int __maybe_unused sep_pm_suspend(struct device *dev) +{ + int (*sleep)(void) = READ_ONCE(sep_pm_sleep); + int ret; + + if (!pm_suspend_via_firmware() || !sleep) + return 0; + + WRITE_ONCE(sep_pm_suspending, true); + ret = sleep(); + WRITE_ONCE(sep_pm_suspending, false); + + return ret; +} + +/* The last step before the platform's own: the domain has to be down by now. */ +static int __maybe_unused sep_pm_suspend_noirq(struct device *dev) +{ + bool (*told)(void) = READ_ONCE(sep_pm_told); + int ret; + + if (!pm_suspend_via_firmware() || !told || !told()) + return 0; + + ret = sep_pm_wait_gated(dev, 5000); + if (ret > 0) + ret = -ETIMEDOUT; + if (ret == -ENOENT) + return 0; + if (ret) { + dev_err(dev, "the enclave did not gate its power domain: %d\n", ret); + } else { + struct regmap *regmap; + u32 offset, val = 0; + + if (!sep_pm_domain(dev, ®map, &offset)) + regmap_read(regmap, offset, &val); + dev_dbg(dev, "the enclave gated its power domain, state %#x\n", val); + } + + return ret; +} + +static int __maybe_unused sep_pm_resume(struct device *dev) +{ + void (*wake)(void) = READ_ONCE(sep_pm_wake); + + if (wake) + wake(); + + return 0; +} + +const struct dev_pm_ops sep_pm_ops = { + .suspend = pm_sleep_ptr(sep_pm_suspend), + .resume = pm_sleep_ptr(sep_pm_resume), + .suspend_noirq = pm_sleep_ptr(sep_pm_suspend_noirq), +}; + +int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)) { int ret; WRITE_ONCE(sep_pm_event, event); + WRITE_ONCE(sep_pm_sleep, sleep); + WRITE_ONCE(sep_pm_wake, wake); + WRITE_ONCE(sep_pm_told, told); ret = register_pm_notifier(&sep_pm_nb); - if (ret) + if (ret) { WRITE_ONCE(sep_pm_event, NULL); - return ret; + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); + return ret; + } + + return 0; } /* Returns only once no notifier call is still running. */ @@ -62,6 +500,9 @@ void sep_pm_unregister(void) { unregister_pm_notifier(&sep_pm_nb); WRITE_ONCE(sep_pm_event, NULL); + WRITE_ONCE(sep_pm_sleep, NULL); + WRITE_ONCE(sep_pm_wake, NULL); + WRITE_ONCE(sep_pm_told, NULL); } static void sep_pm_release_domains(void *data) diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 41091755a77c6..195452b8f0d02 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -1,15 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include #include #include +#include +#include #include +#include #include +#include +#include +#include /* Join removal and serialize the firmware's untagged power reply stream. */ static DEFINE_MUTEX(pmp_lock); static const void *pmp_data; static struct device *pmp_device; +/* The firmware is stopped for a system sleep. */ +static bool pmp_asleep; int apple_pmp_register(struct device *dev, const void *data) { @@ -66,6 +75,8 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; /* The matched native report must be ready; never use an elapsed-time fallback. */ ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) @@ -104,9 +115,153 @@ int apple_pmp_t6030_gpu_ready(void) guard(mutex)(&pmp_lock); if (!pmp_data) return -ENODEV; + if (pmp_asleep) + return -EAGAIN; ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); if (ret) return ret; return apple_pmp_report_set_t6030_gfx(pmp_device, true); } EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); + +/* + * The PMP's idle is a point of no return: an idled PMP doesn't come back with its power kept, + * and only a real sleep restarts it. Where the sleep driver can take them, its idle + * and stop move to the sleep's last step, after every refusal: a refused sleep then finds it + * still running, and the GPU's request, withdrawn at the commit, reaches a running PMP. + */ +static struct apple_pmp_commit_stop pmp_commit; +static bool pmp_commit_registered, pmp_commit_used; + +/* Interrupts off, one CPU: what apple_pmp_sleep() does, through pointers taken before. */ +static int apple_pmp_commit_stop_fn(void *unused) +{ + struct apple_pmp_commit_stop *c = &pmp_commit; + unsigned int us; + size_t i; + int ret; + + ret = apple_rtkit_sleep_idle_polled(c->rtk, c->control, &us); + apple_rtkit_mailbox_recv_off(c->rtk); + for (i = 0; c->sram && i < c->count; i++) + c->words[i] = readl_relaxed(c->sram + 4 * i); + /* Told to idle, it can't carry on whatever it answered: the sleep goes on. */ + return ret ? 1 : 0; +} + +/* + * Retained J713 power votes have no teardown. On T8140 an unbind leaves the firmware running, + * and a commit with it running reset the SoC: its sleep registration is what refuses + * then, every sleep until reboot, since a rebind can't register again. + */ +bool apple_pmp_keep_bound(void) +{ + return apple_pmp_display_retained_board() || of_machine_is_compatible("apple,t8140"); +} + +/* Whether the sleep's last step takes the PMP: registered with the sleep driver, once. */ +static bool apple_pmp_commit_available(struct device *dev) +{ + void __iomem *asc; + int idx; + + if (pmp_commit_registered) + return true; + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return false; + if (apple_sleep_coproc_register(dev, asc, false, apple_pmp_commit_stop_fn, NULL)) { + iounmap(asc); + return false; + } + pmp_commit_registered = true; + + return true; +} + +static int apple_pmp_commit_prepare(struct device *dev) +{ + int ret; + + ret = apple_pmp_sleep_prepare(pmp_data, &pmp_commit); + if (ret) + return ret; + apple_sleep_coproc_set_ready(dev, true); + pmp_commit_used = true; + dev_dbg(dev, "idle and stop left for the sleep's commit\n"); + + return 0; +} + +static int apple_pmp_suspend_late(struct device *dev) +{ + bool at_commit; + int ret; + + if (!pm_suspend_via_firmware()) + return 0; + /* Retained power votes have no teardown. */ + if (apple_pmp_display_retained_board()) + return -EBUSY; + + guard(mutex)(&pmp_lock); + if (pmp_device != dev || pmp_asleep) + return 0; + at_commit = apple_pmp_commit_available(dev); + /* Idled any earlier, a refusal after it would leave it stopped. */ + if (!at_commit && apple_sleep_commit_stops_coprocs()) { + dev_err(dev, "no stop at the sleep's commit: refusing the sleep\n"); + return -EBUSY; + } + apple_pmp_report_sleep(dev, at_commit); + ret = at_commit ? apple_pmp_commit_prepare(dev) : apple_pmp_sleep(pmp_data); + if (ret) { + apple_pmp_report_wake(dev, false); + return dev_err_probe(dev, ret, "firmware did not go idle for sleep\n"); + } + pmp_asleep = true; + + return 0; +} + +static int apple_pmp_resume_early(struct device *dev) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_asleep) + return 0; + /* Left for the commit: if the sleep was refused before it, the firmware never idled. */ + if (pmp_commit_used) { + bool stopped = apple_sleep_coproc_stopped(dev); + + pmp_commit_used = false; + apple_sleep_coproc_set_ready(dev, false); + if (!pm_resume_via_firmware() && !stopped) { + pmp_asleep = false; + apple_pmp_report_wake(dev, true); + dev_info(dev, "sleep refused before the commit: firmware never stopped\n"); + return 0; + } + } + /* A stopped firmware doesn't restart with its power kept: the next real sleep does. */ + if (!pm_resume_via_firmware()) { + dev_warn(dev, "sleep refused: firmware stays stopped until a real sleep\n"); + return 0; + } + pmp_asleep = false; + ret = apple_pmp_wake(pmp_data); + if (ret) + dev_err(dev, "firmware did not come back from sleep: %d\n", ret); + else + dev_dbg(dev, "firmware back from sleep\n"); + /* Requests go back either way: consumers read them from the report too. */ + apple_pmp_report_wake(dev, true); + + return 0; +} + +const struct dev_pm_ops apple_pmp_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmp_suspend_late, apple_pmp_resume_early) +}; diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 3a9f970bf4893..ff2c1bf973e04 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -71,6 +71,7 @@ const BOOTARGS_UUID_MAGIC: u32 = 0x6469_7575; // "uuid" const BOOTARGS_UUID_VERSION: u32 = 5; const BOOTARGS_OFFSET: usize = 0x22c; const BOOTARGS_SIZE: usize = 0x230; +const ASC_CPU_CONTROL_OFFSET: usize = 0x44; const PMP_ENDPOINT: u8 = 0x20; const PM_COMMAND_MARKER: u64 = pmp_power::MARKER; const PM_COMMAND_SHIFT: u32 = pmp_power::SHIFT; @@ -120,11 +121,19 @@ unsafe impl AsBytes for IovaTableEntry {} // SAFETY: TODO: unsafe impl FromBytes for IovaTableEntry {} +/// A range of the firmware's SRAM as it was when the firmware stopped. +struct SramCopy { + offset: usize, + words: KVVec, +} + struct PmpState { iova_table: Option>, allocs: KVec, value_buf: Option, ioreg_entries: KVec, + /// Held from a system sleep's start until its end. + sram: Option, } impl PmpState { @@ -134,6 +143,7 @@ impl PmpState { allocs: KVec::with_capacity(10, GFP_KERNEL)?, value_buf: None, ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + sram: None, }) } fn find_alloc(&self, addr: u64) -> Option { @@ -169,10 +179,12 @@ struct PmpData { display: AtomicPtr, /// The first failure seen before or after the bridge was published. display_error: AtomicI32, + /// See [`PmpHwConfig::sram_backup`]. + sram_backup: Option<(usize, usize)>, } impl PmpData { - fn new(dev: &platform::Device>) -> Result> { + fn new(dev: &platform::Device>, cfg: &PmpHwConfig) -> Result> { let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = pmp_req.iomap_sized::()?.into_devres()?; let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; @@ -189,6 +201,7 @@ impl PmpData { asc_mmio, display: AtomicPtr::new(core::ptr::null_mut()), display_error: AtomicI32::new(0), + sram_backup: cfg.sram_backup, } ), GFP_KERNEL, @@ -438,6 +451,112 @@ impl PmpData { ) }; } + /// The handle, to wait on the firmware with the lock free: its messages take that lock. + fn rtkit_handle(&self) -> Result<*mut bindings::apple_rtkit> { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + Ok(rtk.as_raw()) + } + /// The ASC's CPU control register. + fn asc_cpu_control(&self) -> Result<*mut kernel::ffi::c_void> { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let asc_mmio = self.asc_mmio.access(bound_dev)?.as_view(); + Ok(MmioBackend::as_ptr(asc_mmio) + .cast::() + .wrapping_add(ASC_CPU_CONTROL_OFFSET) + .cast()) + } + /// Room for what SRAM needs kept, taken first so that running out refuses a sleep cleanly. + fn sram_alloc(&self, sram_size: usize) -> Result> { + let Some((offset, size)) = self.sram_backup else { + return Ok(None); + }; + if !pmp_bootargs::word_range_valid(offset, size, sram_size) { + return Err(EINVAL); + } + let mut words = KVVec::new(); + words.resize(size / 4, 0u32, GFP_KERNEL)?; + Ok(Some(SramCopy { offset, words })) + } + /// Stops the firmware for a system sleep, then keeps what its restart needs from SRAM. + fn sleep(&self) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + let mut copy = self.sram_alloc(io.size())?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + if let Err(error) = to_result(unsafe { bindings::apple_rtkit_sleep_stop(rtk, control) }) { + // SAFETY: as above. + if to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }).is_err() { + self.fail_display(error); + } + return Err(error); + } + if let Some(copy) = copy.as_mut() { + for (idx, word) in copy.words.iter_mut().enumerate() { + // The range was checked against the mapping when it was allocated. + *word = io.try_read32(copy.offset + idx * 4).unwrap_or(0); + } + } + self.state.lock().sram = copy; + Ok(()) + } + /// For a stop at the commit: the RTKit handle, the CPU control and the SRAM copy's room, + /// whose buffer stays where it is once moved into the state. + fn sleep_prepare(&self, stop: &mut bindings::apple_pmp_commit_stop) -> Result { + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let size = pmp_mmio.as_view().relaxed().size(); + let mut copy = self.sram_alloc(size)?; + stop.rtk = self.rtkit_handle()?; + stop.control = self.asc_cpu_control()?; + stop.sram = core::ptr::null_mut(); + stop.words = core::ptr::null_mut(); + stop.count = 0; + if let Some(copy) = copy.as_mut() { + stop.sram = MmioBackend::as_ptr(pmp_mmio) + .cast::() + .wrapping_add(copy.offset) + .cast(); + stop.words = copy.words.as_mut_ptr(); + stop.count = copy.words.len(); + } + self.state.lock().sram = copy; + Ok(()) + } + /// Writes SRAM back after a sleep that took its power, while the firmware is stopped. + fn sram_restore(&self) -> Result { + let Some(copy) = self.state.lock().sram.take() else { + return Ok(()); + }; + // SAFETY: the PM core only calls a bound device. + let bound_dev = unsafe { self.dev.as_bound() }; + let pmp_mmio = self.pmp_mmio.access(bound_dev)?.as_view(); + let io = pmp_mmio.as_view().relaxed(); + for (idx, word) in copy.words.iter().enumerate() { + io.try_write32(*word, copy.offset + idx * 4)?; + } + // The firmware mustn't start ahead of writes still on their way. + let _ = io.try_read32(copy.offset); + dev_dbg!(self.dev, "SRAM restored: {} bytes\n", copy.words.len() * 4); + Ok(()) + } + /// Brings the firmware back after a system sleep. It keeps its endpoints and everything the + /// host gave it, so nothing here repeats what it asked for at probe. + fn wake(&self) -> Result { + self.sram_restore()?; + let rtk = self.rtkit_handle()?; + let control = self.asc_cpu_control()?; + // SAFETY: the handle stays valid until unbind, which can't run during a system sleep, + // and `control` is inside the ASC mapping held by `self`. + to_result(unsafe { bindings::apple_rtkit_sleep_return(rtk, control) }) + } fn recv_message(&self, msg: u64) -> Result<()> { let pm_command = (msg >> PM_COMMAND_SHIFT) & 0xff; if msg & PM_COMMAND_MARKER != 0 && (pm_command == 0x0f || pm_command == 0x10) { @@ -611,6 +730,58 @@ pub unsafe extern "C" fn apple_pmp_send_power_command( 0 } +/// Stops the firmware for a system sleep. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.sleep() { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Takes what the sleep's last step needs to stop the firmware with interrupts off, and the room +/// for the SRAM it keeps. Nothing is stopped here. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver, and `stop` to writable memory. +#[export] +pub unsafe extern "C" fn apple_pmp_sleep_prepare( + data: *const core::ffi::c_void, + stop: *mut bindings::apple_pmp_commit_stop, +) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + // SAFETY: The caller passes a valid pointer. + let stop = unsafe { &mut *stop }; + match data.sleep_prepare(stop) { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Brings the firmware back after a system sleep. A firmware that doesn't come back disables +/// further power commands. +/// +/// # Safety +/// `data` must point to the live PmpData registered by PmpDriver. +#[export] +pub unsafe extern "C" fn apple_pmp_wake(data: *const core::ffi::c_void) -> i32 { + // SAFETY: The C bridge serializes calls against unregister in Drop. + let data = unsafe { &*data.cast::() }; + match data.wake() { + Ok(()) => 0, + Err(error) => { + data.fail_display(error); + error.to_errno() + } + } +} + struct NoBuffer; impl rtkit::Buffer for NoBuffer { fn iova(&self) -> Result { @@ -659,6 +830,9 @@ struct PmpHwConfig { /// When the firmware needs system endpoint 7 (CoreAnalytics) started /// and given its buffer before it acknowledges the IOP power state. coreanalytics: Ep7, + /// The range of the firmware's SRAM, as (offset, size) in bytes, that a system sleep keeps + /// and writes back. `None` where no sleep has been traced. + sram_backup: Option<(usize, usize)>, } /// Whether a PMP firmware needs endpoint 7 to finish its RTKit boot. @@ -676,6 +850,7 @@ const HW_CFG_DEFAULT: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; /// T8140: the DART translates 42 bits and its DMA window starts at 1 TiB. @@ -683,6 +858,7 @@ const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: Some((0x3c000, 0x5c000)), }; /// T6041: the DART window starts at 1 TiB; the boot argument header is @@ -692,6 +868,7 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: true, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 @@ -701,6 +878,7 @@ const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::Always, + sram_backup: None, }; /// T6050: the DART window starts at 1 TiB (ADT dart-pmp0 vm-base) and spans @@ -709,12 +887,14 @@ const HW_CFG_T6050: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: false, coreanalytics: Ep7::No, + sram_backup: None, }; const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, coreanalytics: Ep7::RetainedBoard, + sram_backup: None, }; impl Drop for PmpDriver { @@ -742,9 +922,14 @@ impl platform::Driver for PmpDriver { type Data<'bound> = PmpDriver; fn suppress_bind_attrs() -> bool { - // Retained J713 power votes have no teardown path. - // SAFETY: read-only board identity check; other boards remain unchanged. - unsafe { bindings::apple_pmp_display_retained_board() } + // Retained J713 power votes have no teardown path, and an unbound T8140 PMP still runs. + // SAFETY: read-only machine identity checks; other machines remain unchanged. + unsafe { bindings::apple_pmp_keep_bound() } + } + + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pmp-export.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(bindings::apple_pmp_pm_ops) }) } const OF_ID_TABLE: Option> = Some(&OF_TABLE); @@ -760,7 +945,7 @@ impl platform::Driver for PmpDriver { // yet. unsafe { pdev.dma_set_mask_and_coherent(mask)? }; } - let data = PmpData::new(pdev)?; + let data = PmpData::new(pdev, cfg)?; let pmp_mmio = data.pmp_mmio.access(pdev.as_ref())?.as_view(); let asc_mmio = data.asc_mmio.access(pdev.as_ref())?.as_view(); let node = dev.fwnode().ok_or(EIO)?; diff --git a/drivers/soc/apple/pmp_bootargs.rs b/drivers/soc/apple/pmp_bootargs.rs index c085a323fee47..bd2abbc94029b 100644 --- a/drivers/soc/apple/pmp_bootargs.rs +++ b/drivers/soc/apple/pmp_bootargs.rs @@ -7,6 +7,11 @@ pub(super) fn region_valid(offset: usize, size: usize, sram_size: usize) -> bool size != 0 && offset <= sram_size && size <= sram_size - offset } +/// A range of SRAM that can be copied as 32-bit words. +pub(super) fn word_range_valid(offset: usize, size: usize, sram_size: usize) -> bool { + region_valid(offset, size, sram_size) && offset % 4 == 0 && size % 4 == 0 +} + fn header(bytes: &[u8]) -> Option<(u32, usize)> { let key = u32::from_le_bytes(bytes.get(..4)?.try_into().ok()?); let size = u32::from_le_bytes(bytes.get(4..8)?.try_into().ok()?) as usize; @@ -69,6 +74,15 @@ mod tests { assert!(!region_valid(1, usize::MAX, usize::MAX)); } + #[test] + fn a_word_range_is_aligned_and_inside_the_window() { + assert!(word_range_valid(0x3c000, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c002, 0x5c000, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x5c002, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0x64004, 0xa0000)); + assert!(!word_range_valid(0x3c000, 0, 0xa0000)); + } + #[test] fn larger_sram_uses_its_actual_extent() { assert!(region_valid(0x90000, 0x1000, 0x100000)); diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 8fc7278550d9a..1bdea86a4c51e 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -90,6 +90,32 @@ pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; pub(crate) const TAG_ENTROPY: u8 = 0xE7; +/// The tag of the sleep request, kept out of the pool. +pub(crate) const TAG_SLEEP: u8 = 0x08; + +/// The sleep request's answer, in the bits [`SLEEP_REPLY_MASK`] keeps: the control endpoint, +/// its tag and the reply type. +pub(crate) const SLEEP_REPLY: u64 = (EP_CONTROL as u64) + | ((TAG_SLEEP as u64) << MSG_TAG_SHIFT) + | ((CONTROL_REPLY_TYPE as u64) << MSG_TYPE_SHIFT); +pub(crate) const SLEEP_REPLY_MASK: u64 = (1 << MSG_PARAM_SHIFT) - 1; + +// The answer in the trace, 0x0000_0000_0001_0800. +static_assert!(0x0000_0000_0001_0800 & SLEEP_REPLY_MASK == SLEEP_REPLY); + +/// How long the enclave gets to answer the sleep request. +pub(crate) const SLEEP_TIMEOUT_MS: u32 = 5000; + +/// The first message macOS sends a gated enclave after a sleep. The contents +/// vary from sleep to sleep, so it is taken to be a doorbell. This is one of them. +pub(crate) const WAKE_KICK: Message = Message { + msg0: 0xffff_ffff_a482_9303, + msg1: 0, +}; + +/// The control messages a woken enclave sends without being asked, in order. +pub(crate) const WAKE_TYPES: [u8; 2] = [0x0d, 0x10]; + pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; pub(crate) const TAG_POOL_FIRST: u8 = 0x01; @@ -97,6 +123,7 @@ pub(crate) const TAG_POOL_LAST: u8 = 0x7e; static_assert!(TAG_POOL_FIRST > 0); static_assert!(TAG_POOL_LAST < TAG_ENTROPY); +static_assert!(TAG_SLEEP >= TAG_POOL_FIRST && TAG_SLEEP <= TAG_POOL_LAST); pub(crate) struct ControlOp { ty: u8, @@ -161,6 +188,19 @@ pub(crate) fn op_get_entropy() -> ControlOp { } } +/// Asks the enclave to save its state and gate its own power domain. +pub(crate) fn op_sleep() -> ControlOp { + ControlOp { + ty: 0x0c, + param: 0x02, + data: 0, + expect_reply: true, + timeout_ms: SLEEP_TIMEOUT_MS, + reserved_tag: false, + name: c"SLEEP", + } +} + pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { Message { msg0: u64::from(EP_CONTROL) @@ -282,6 +322,34 @@ mod tests { assert_eq!(f.data_lo, 0x1234_5678); } + #[test] + fn sleep_request_matches_the_trace() { + let msg = encode_control(&op_sleep(), TAG_SLEEP); + assert_eq!(msg.msg0, 0x0000_0000_020c_0800); + assert_eq!(msg.msg1, 0); + } + + #[test] + fn its_answer_matches_the_trace() { + let reply = Message { + msg0: 0x0000_0000_0001_0800, + msg1: 0, + }; + let f = decode(&reply); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, CONTROL_REPLY_TYPE); + assert_eq!(f.tag, TAG_SLEEP); + } + + #[test] + fn wake_messages_match_the_trace() { + for (raw, ty) in [(0x000d_0000u64, WAKE_TYPES[0]), (0x0010_0000u64, WAKE_TYPES[1])] { + let f = decode(&Message { msg0: raw, msg1: 0 }); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.ty, ty); + } + } + #[test] fn only_entropy_uses_the_reserved_tag() { assert!(op_get_entropy().uses_reserved_tag()); diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 07b5840da8fde..59bb373e954ac 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -39,6 +39,12 @@ struct apple_rtkit_helper { * it does not list must not replace storage the firmware still owns. */ bool strict_buffers; + /* The firmware was told the AP is going to sleep. */ + bool ap_idle; + /* It was told, gave no answer, and wasn't told otherwise. */ + bool idle_unanswered; + /* Ends a sleep whose .prepare failed, which gets no .complete. */ + struct notifier_block pm_nb; void __iomem *asc_base; @@ -237,6 +243,8 @@ static int apple_rtkit_helper_inherit_buffers( return count; } +static int apple_rtkit_helper_pm_register(struct apple_rtkit_helper *helper); + static int apple_rtkit_helper_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -339,6 +347,9 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->rtk); if (ret) return ret; + ret = apple_rtkit_helper_pm_register(helper); + if (ret) + return ret; /* There is no safe module-unload boundary for the retained session. */ __module_get(THIS_MODULE); @@ -363,7 +374,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) if (ret != 0) return dev_err_probe(dev, ret, "Failed to wake up coprocessor\n"); - return 0; + return apple_rtkit_helper_pm_register(helper); } static void apple_rtkit_helper_remove(struct platform_device *pdev) @@ -380,16 +391,129 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); } +static int apple_rtkit_helper_ap_power(struct apple_rtkit_helper *helper, bool on) +{ + int ret = apple_rtkit_ap_power(helper->rtk, on); + + if (ret) + dev_err(helper->dev, "AP power notice (%s) not answered: %d\n", + on ? "on" : "idle", ret); + /* + * Nothing is sent to undo an unanswered idle notice, which macOS never does: + * the end of the sleep looks whether a late answer came, and only then sends AP on. + */ + if (ret && !on) + helper->idle_unanswered = true; + return ret; +} + +static int apple_rtkit_helper_ap_idle(struct apple_rtkit_helper *helper) +{ + int ret; + + if (helper->ap_idle) + return 0; + ret = apple_rtkit_helper_ap_power(helper, false); + if (ret) + return ret; + helper->ap_idle = true; + dev_dbg(helper->dev, "AP idle notice answered for a sleep\n"); + return 0; +} + +static void apple_rtkit_helper_ap_on(struct apple_rtkit_helper *helper) +{ + if (!helper->ap_idle) + return; + helper->ap_idle = false; + if (!apple_rtkit_helper_ap_power(helper, true)) + dev_dbg(helper->dev, "AP on notice answered after a sleep\n"); +} + +/* + * macOS tells this coprocessor first of all. Told before any driver stops its own, + * an unanswered notice refuses a sleep that has stopped nothing yet. + */ +static int __maybe_unused apple_rtkit_helper_prepare(struct device *dev) +{ + if (!pm_suspend_via_firmware()) + return 0; + return apple_rtkit_helper_ap_idle(dev_get_drvdata(dev)); +} + +static void apple_rtkit_helper_sleep_end(struct apple_rtkit_helper *helper) +{ + if (helper->idle_unanswered) { + bool late = !apple_rtkit_is_running(helper->rtk); + + helper->idle_unanswered = false; + dev_info(helper->dev, "after the unanswered idle notice: %s\n", + late ? "its answer came late, sending AP on" : + "still no answer, nothing sent"); + helper->ap_idle = late; + } + apple_rtkit_helper_ap_on(helper); +} + +/* A sleep refused before this device's .suspend never calls its .resume. */ +static void __maybe_unused apple_rtkit_helper_complete(struct device *dev) +{ + apple_rtkit_helper_sleep_end(dev_get_drvdata(dev)); +} + +/* The PM core calls no .complete after a failed .prepare: end that sleep here. */ +static int apple_rtkit_helper_pm_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct apple_rtkit_helper *helper = container_of(nb, struct apple_rtkit_helper, pm_nb); + + if (action == PM_POST_SUSPEND && helper->idle_unanswered) + apple_rtkit_helper_sleep_end(helper); + return NOTIFY_DONE; +} + +static void apple_rtkit_helper_pm_unregister(void *data) +{ + struct apple_rtkit_helper *helper = data; + + unregister_pm_notifier(&helper->pm_nb); +} + +static int apple_rtkit_helper_pm_register(struct apple_rtkit_helper *helper) +{ + int ret; + + helper->pm_nb.notifier_call = apple_rtkit_helper_pm_notify; + ret = register_pm_notifier(&helper->pm_nb); + if (ret) + return ret; + return devm_add_action_or_reset(helper->dev, apple_rtkit_helper_pm_unregister, helper); +} + static int __maybe_unused apple_rtkit_helper_suspend(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + /* + * A firmware-driven suspend keeps this coprocessor running. It hears the + * AP's power state and nothing else, so an adopted session stays live. + * That is measured on T8140 and unverified on other SoCs. + */ + if (pm_suspend_via_firmware()) + return apple_rtkit_helper_ap_idle(helper); + /* Only suspend-to-idle has a verified live-session retention contract. */ if (helper->adopted && pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; return 0; } +static int __maybe_unused apple_rtkit_helper_resume(struct device *dev) +{ + apple_rtkit_helper_ap_on(dev_get_drvdata(dev)); + return 0; +} + static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); @@ -399,7 +523,10 @@ static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) } static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { + .prepare = pm_sleep_ptr(apple_rtkit_helper_prepare), + .complete = pm_sleep_ptr(apple_rtkit_helper_complete), .suspend = pm_sleep_ptr(apple_rtkit_helper_suspend), + .resume = pm_sleep_ptr(apple_rtkit_helper_resume), .freeze = pm_sleep_ptr(apple_rtkit_helper_freeze), .poweroff = pm_sleep_ptr(apple_rtkit_helper_freeze), }; diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index fb365ebfe1921..7551fe6aca285 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -63,6 +63,17 @@ struct apple_rtkit { * hand out kdebug buffers, and poll the mailbox while waiting. */ bool sync_syslog_ack; + /* Echo each OSLOG channel-0 counter, as macOS does. */ + bool oslog_echo; + /* A counter queued for the receive work and not yet echoed. */ + u64 oslog_owed; + /* Device interrupts are off: poll the mailbox while waiting. */ + bool poll_waits; + /* The firmware is restarting with its buffers and rings still in place. */ + bool warm_restart; + u32 warm_last_mask; + /* A port that only answers the co-processor, see apple_rtkit_port_handshake(). */ + bool port_only; DECLARE_KFIFO(sync_ack_fifo, struct apple_mbox_msg, 64); struct work_struct sync_ack_work; @@ -93,6 +104,8 @@ struct apple_rtkit { /* Session inherited from the bootloader: INIT geometry never observed. */ bool syslog_inherited; bool crashlog_inherited; + /* The crash log buffer is the firmware's own, mapped here when it crashed. */ + bool crashlog_placed; bool oslog_inherited; /* A crash reported before the crashlog buffer was mapped: the log is diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 65faedc3da557..8535fd432f4b6 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -7,10 +7,14 @@ #include "rtkit-internal.h" #include +#include +#include #include #include #include #include +#include +#include enum { APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */ @@ -45,7 +49,7 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, - /* Sent by some co-processors once idle after an IOP IDLE request. */ + /* Sent by some co-processors once idle after an IOP IDLE request, argument 0. */ APPLE_RTKIT_MGMT_IDLE_NOTIFY = 0xc, }; @@ -63,6 +67,9 @@ enum { #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0) +#define APPLE_RTKIT_POLLED_STEP_US 10 +#define APPLE_RTKIT_POLLED_LOOPS 20000 + #define APPLE_RTKIT_CRASHLOG_CRASH 1 #define APPLE_RTKIT_BUFFER_REQUEST 1 @@ -117,6 +124,14 @@ enum { #define APPLE_RTKIT_CA_EVENT 2 #define APPLE_RTKIT_CA_EVENT_ACK 3 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +/* From the CPU control register. Both bits wait for the firmware's last message to be read. */ +#define APPLE_ASC_CPU_STATUS 4 +#define APPLE_ASC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_ASC_CPU_STATUS_IDLE BIT(5) +#define APPLE_ASC_CPU_STATUS_DOWN (APPLE_ASC_CPU_STATUS_STOPPED | APPLE_ASC_CPU_STATUS_IDLE) + #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 @@ -302,7 +317,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) * request early crashlog startup, need the endpoint before the map is * acknowledged: G17P can request its crash DVA right after. */ - if (apple_rtkit_early_crashlog(rtk) && base == 0 && + if (!rtk->warm_restart && apple_rtkit_early_crashlog(rtk) && base == 0 && test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) { ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); if (ret) { @@ -312,6 +327,15 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + /* macOS's secure side sends each part back unchanged. */ + if (rtk->port_only) { + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, msg); + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) { + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + } + return; + } if (rtk->retained_wake) { apple_rtkit_management_rx_epmap_retained(rtk, msg, base, bitmap); return; @@ -323,11 +347,29 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) else reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; + /* + * macOS's reply to a restarted DCP echoes the map, and adds bit 1 to the + * first part: 0x51d becomes 0x51f. It starts no endpoints. Measured under + * aura, 3 of 3. Why bit 1 isn't known, but ep 1 is the crash log's. + */ + if (rtk->warm_restart) { + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) + reply |= bitmap & rtk->warm_last_mask; + else + reply |= bitmap | BIT_ULL(1); + } + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (rtk->warm_restart) { + rtk->warm_restart = false; + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + return; + } if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) dev_info(rtk->dev, "RTKit: crashlog endpoint not in the endpoint map\n"); @@ -444,7 +486,7 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) apple_rtkit_management_rx_ap_pwr_ack(rtk, msg); break; case APPLE_RTKIT_MGMT_IDLE_NOTIFY: - dev_dbg(rtk->dev, "RTKit: co-processor idle\n"); + dev_dbg(rtk->dev, "RTKit: co-processor idle: 0x%llx\n", msg); break; default: dev_warn( @@ -569,10 +611,14 @@ static void apple_rtkit_free_buffer(struct apple_rtkit *rtk, if (bfr->size == 0) return; - if (rtk->ops->shmem_destroy) + if (bfr == &rtk->crashlog_buffer && rtk->crashlog_placed) { + memunmap(bfr->buffer); + rtk->crashlog_placed = false; + } else if (rtk->ops->shmem_destroy) { rtk->ops->shmem_destroy(rtk->cookie, bfr); - else if (bfr->buffer) + } else if (bfr->buffer) { dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova); + } bfr->buffer = NULL; bfr->iomem = NULL; @@ -804,6 +850,46 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, len, DMA_FROM_DEVICE); } +/* + * The buffer a crashing firmware placed at its own address, mapped if its IOMMU + * translates every page of it to one contiguous range, and acknowledged as a placed + * buffer is. The firmware doesn't run again until a reboot. + */ +static int apple_rtkit_crashlog_map(struct apple_rtkit *rtk, u64 msg) +{ + struct apple_rtkit_shmem *bfr = &rtk->crashlog_buffer; + struct iommu_domain *domain = iommu_get_domain_for_dev(rtk->dev); + dma_addr_t iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + size_t size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; + phys_addr_t phys, page; + size_t off; + void *va; + + if (!domain || !iova || !size) + return -ENODEV; + phys = iommu_iova_to_phys(domain, iova); + for (off = 0; phys && off < size; off += PAGE_SIZE) { + page = iommu_iova_to_phys(domain, iova + off); + if (page != phys + off) { + dev_err(rtk->dev, "RTKit: crash log at %pad isn't contiguous at +%#zx\n", + &iova, off); + return -EINVAL; + } + } + if (!phys) + return -ENOENT; + va = memremap(phys, size, MEMREMAP_WB); + if (!va) + return -ENOMEM; + bfr->buffer = va; + bfr->size = size; + bfr->iova = iova; + bfr->is_mapped = true; + rtk->crashlog_placed = true; + + return apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_CRASHLOG, msg, NULL, false); +} + #define APPLE_RTKIT_CRASHLOG_LATE_POLL_MS 250 #define APPLE_RTKIT_CRASHLOG_LATE_POLLS 20 @@ -887,6 +973,28 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } + /* + * A firmware that never announced this endpoint asks for its buffer only when + * it crashes, so there's no handshake to take the request for. Report the + * crash now, as the log may never come. A client that sets such a buffer up + * itself takes the path below, which also reads a log written later. + */ + if (!rtk->crashlog_buffer.size && !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints) && + !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { + int ret = apple_rtkit_crashlog_map(rtk, msg); + + dev_err(rtk->dev, "RTKit: co-processor has crashed: crash log buffer at %#llx %s (%d)\n", + FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), + ret ? "not mapped" : "mapped, waiting for its log", ret); + if (!rtk->crashed) { + rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + } + return; + } + if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) @@ -918,6 +1026,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) "RTKit: firmware-owned crash log at DVA %pad (0x%zx bytes)\n", &rtk->crashlog_buffer.iova, rtk->crashlog_buffer.size); rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, NULL, 0); return; @@ -938,7 +1047,11 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) "RTKit: Couldn't allocate crashlog shadow buffer\n"); } + /* Its driver heard of the crash when the buffer was asked for. */ + if (rtk->crashed) + return; rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); return; @@ -963,6 +1076,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) crashed: rtk->crashed = true; + apple_sleep_coproc_crashed(rtk->dev); if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, NULL, 0); @@ -1116,6 +1230,22 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); int err; + /* + * A firmware counts its log writes and holds the next counter until the last is + * echoed, and with one outstanding takes no AP power notice: a DCP after a warm + * restart, the MTP and the SMC after about eight sleeps. macOS echoes every one. + */ + if (type == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo) { + /* A polled idle that came first echoed it already. */ + if (cmpxchg(&rtk->oslog_owed, msg, 0) != msg) + return; + dev_dbg(rtk->dev, "RTKit: oslog counter %016llx, echoed\n", msg); + err = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, msg, NULL, false); + if (err) + dev_err(rtk->dev, "RTKit: oslog echo failed: %d\n", err); + return; + } + /* Preserve the tested Neo/M5 and synchronous-log no-ACK policy. */ if (type == APPLE_RTKIT_OSLOG_LOG && (rtk->sync_syslog_ack || of_machine_is_compatible("apple,t8140") || @@ -1236,12 +1366,17 @@ static void apple_rtkit_rx_work(struct work_struct *work) kfree(rtk_work); } -static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +/* An OSLOG counter this firmware waits to see echoed. */ +static bool apple_rtkit_oslog_owed_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return ep == APPLE_RTKIT_EP_OSLOG && + FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg) == APPLE_RTKIT_OSLOG_LOG && rtk->oslog_echo; +} + +static bool apple_rtkit_notification_msg(u8 ep, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - if (!rtk->sync_syslog_ack) - return false; if (ep == APPLE_RTKIT_EP_SYSLOG) return type == APPLE_RTKIT_SYSLOG_LOG; if (ep == APPLE_RTKIT_EP_IOREPORT) @@ -1250,6 +1385,11 @@ static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) return false; } +static bool apple_rtkit_sync_ack_msg(struct apple_rtkit *rtk, u8 ep, u64 msg) +{ + return rtk->sync_syslog_ack && apple_rtkit_notification_msg(ep, msg); +} + static void apple_rtkit_sync_ack_work(struct work_struct *work) { struct apple_rtkit *rtk = @@ -1320,6 +1460,8 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, if (!work) return; + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) + WRITE_ONCE(rtk->oslog_owed, msg.msg0); work->rtk = rtk; work->ep = ep; work->msg = msg.msg0; @@ -1365,6 +1507,82 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +static int apple_rtkit_poll_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->epmap_completion)) + return rtk->boot_result; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return try_wait_for_completion(&rtk->epmap_completion) ? rtk->boot_result : -ETIME; +} + +static int apple_rtkit_port_power(struct apple_rtkit *rtk, unsigned int state, + unsigned int timeout_ms) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + reinit_completion(&rtk->ap_pwr_ack_completion); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state)); + if (ret) + return ret; + + do { + ret = apple_rtkit_poll(rtk); + if (ret < 0) + return ret; + if (try_wait_for_completion(&rtk->ap_pwr_ack_completion)) + return 0; + usleep_range(50, 100); + } while (time_before(jiffies, timeout)); + + return -ETIME; +} + +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + rtk->port_only = true; + + return apple_rtkit_poll_handshake(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_map); + +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + /* The co-processor starts its side of this port itself. */ + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_INIT, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_on); + +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + int ret; + + ret = apple_rtkit_port_map(rtk, timeout_ms); + if (ret) + return ret; + + return apple_rtkit_port_on(rtk, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_handshake); + +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms) +{ + return apple_rtkit_port_power(rtk, APPLE_RTKIT_PWR_STATE_IDLE, timeout_ms); +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_idle); + void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { apple_mbox_hold_rx(rtk->mbox); @@ -1442,6 +1660,10 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) set_bit(system_endpoints[i], rtk->endpoints); + /* apple_rtkit_wake() must not wait for an EPMAP that already happened. */ + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->syslog_inherited = true; @@ -1584,6 +1806,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, rtk->handshake_timeout_ms = APPLE_RTKIT_HANDSHAKE_TIMEOUT_MS; rtk->sync_syslog_ack = of_property_read_bool(dev->of_node, "apple,sync-syslog-ack"); + /* As macOS does; a driver whose firmware shouldn't get them turns it off. */ + rtk->oslog_echo = of_machine_is_compatible("apple,t8140"); INIT_KFIFO(rtk->sync_ack_fifo); INIT_WORK(&rtk->sync_ack_work, apple_rtkit_sync_ack_work); @@ -1613,6 +1837,8 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + rtk->warm_last_mask = U32_MAX; + rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { @@ -1667,15 +1893,18 @@ static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c); /* - * A retained wake is not interrupted by a signal to its caller: it has + * A handshake given up halfway can leave a firmware started that nothing stops. The + * freezer signals every user task as a sleep begins, so only a fatal signal ends the + * wait, and during a system sleep not even that. Nor does a retained wake's: it has * already started the co-processor's CPU, and nothing would undo that. */ static long apple_rtkit_wait_timeout(struct apple_rtkit *rtk, struct completion *c, unsigned long timeout) { - if (READ_ONCE(rtk->retained_wake)) + if (READ_ONCE(rtk->retained_wake) || pm_suspend_target_state == PM_SUSPEND_MEM) return wait_for_completion_timeout(c, timeout); - return wait_for_completion_interruptible_timeout(c, timeout); + + return wait_for_completion_killable_timeout(c, timeout); } /* @@ -1689,7 +1918,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) long t; int ret; - if (!rtk->sync_syslog_ack) + if (!rtk->sync_syslog_ack && !rtk->poll_waits) return apple_rtkit_wait_for_completion(rtk, c); timeout = jiffies + msecs_to_jiffies(rtk->handshake_timeout_ms); @@ -1722,10 +1951,9 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted_with_buffers); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c) { - unsigned long timeout = msecs_to_jiffies(rtk->handshake_timeout_ms); long t; - t = apple_rtkit_wait_timeout(rtk, c, timeout); + t = apple_rtkit_wait_timeout(rtk, c, msecs_to_jiffies(rtk->handshake_timeout_ms)); if (t < 0) return t; else if (t == 0) @@ -1741,6 +1969,63 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, } EXPORT_SYMBOL_GPL(apple_rtkit_set_handshake_timeout); +/* + * Expect HELLO from a firmware being restarted warm: it keeps the buffers it was + * given and its rings, so unlike apple_rtkit_reinit() nothing is freed, and + * endpoints aren't started again. The caller then sets the run bit and calls + * apple_rtkit_boot(). + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk) +{ + unsigned long flags; + int ret; + + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + + reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + + rtk->crashed = false; + rtk->warm_restart = true; + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; + kfifo_reset(&rtk->sync_ack_fifo); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + WRITE_ONCE(rtk->shutting_down, false); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + ret = apple_mbox_start(rtk->mbox); + if (ret) { + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + } else { + apple_mbox_set_recv_enabled(rtk->mbox, true); + } + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_warm_restart); + +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask) +{ + rtk->warm_last_mask = last_mask; +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_warm_epmap_mask); + +int apple_rtkit_port_restart(struct apple_rtkit *rtk) +{ + int ret = apple_rtkit_warm_restart(rtk); + + rtk->warm_restart = false; + rtk->port_only = true; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_port_restart); + int apple_rtkit_reinit(struct apple_rtkit *rtk) { unsigned long flags; @@ -1846,7 +2131,8 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power, + void (*before_ap)(void *data), void *data) { int ret; @@ -1894,16 +2180,27 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) if (rtk->boot_result) return rtk->boot_result; - return early_ap_power ? 0 : - apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (early_ap_power) + return 0; + + if (before_ap) + before_ap(data); + + return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); } int apple_rtkit_boot(struct apple_rtkit *rtk) { - return apple_rtkit_boot_common(rtk, false); + return apple_rtkit_boot_common(rtk, false, NULL, NULL); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data) +{ + return apple_rtkit_boot_common(rtk, false, before_ap, data); +} +EXPORT_SYMBOL_GPL(apple_rtkit_boot_with); + int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { if (rtk->crashed) @@ -1948,26 +2245,66 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poweroff); -int apple_rtkit_idle(struct apple_rtkit *rtk) +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data) { int ret; /* if OFF is used here the co-processor will not wake up again */ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: AP idle not acknowledged: %d\n", ret); return ret; + } + + if (after_ap) + after_ap(data); ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: IOP idle not acknowledged: %d\n", ret); return ret; + } rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE; return 0; } +EXPORT_SYMBOL_GPL(apple_rtkit_idle_with); + +int apple_rtkit_idle(struct apple_rtkit *rtk) +{ + return apple_rtkit_idle_with(rtk, NULL, NULL); +} EXPORT_SYMBOL_GPL(apple_rtkit_idle); +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_off); + +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on) +{ + return apple_rtkit_set_ap_power_state(rtk, on ? APPLE_RTKIT_PWR_STATE_ON : + APPLE_RTKIT_PWR_STATE_IDLE); +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power); + +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on) +{ + int ret; + + rtk->poll_waits = true; + ret = apple_rtkit_ap_power(rtk, on); + rtk->poll_waits = false; + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_power_noirq); + int apple_rtkit_quiesce(struct apple_rtkit *rtk) { int ret; @@ -2012,7 +2349,7 @@ static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) if (ret) return ret; - return apple_rtkit_boot_common(rtk, early_ap_power); + return apple_rtkit_boot_common(rtk, early_ap_power, NULL, NULL); } int apple_rtkit_wake(struct apple_rtkit *rtk) @@ -2027,6 +2364,225 @@ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); +static int apple_rtkit_sleep_idle(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + u32 status; + int ret; + + ret = apple_rtkit_idle(rtk); + if (ret) + return ret; + + ret = readl_poll_timeout(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == APPLE_ASC_CPU_STATUS_DOWN, + 100, 5000); + /* Both idles took, so it doesn't run on: stop it as if the status had shown it. */ + if (ret) + dev_warn(rtk->dev, "RTKit: idle, but not stopped: status %#x\n", status); + + return 0; +} + +/* With interrupts off: room in the FIFO, then the message. */ +static bool apple_rtkit_polled_send(struct apple_rtkit *rtk, struct apple_mbox_msg msg) +{ + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return false; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + + return !apple_mbox_send(rtk->mbox, msg, true); +} + +/* + * Only what an idle needs: the state acks, notifications answered as they come, and OSLOG + * counters echoed, without which the firmware takes no AP power notice. + */ +static void apple_rtkit_polled_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) +{ + struct apple_rtkit *rtk = cookie; + u8 ep = msg.msg1; + + if (apple_rtkit_oslog_owed_msg(rtk, ep, msg.msg0)) { + if (!apple_rtkit_polled_send(rtk, msg)) + dev_err(rtk->dev, "RTKit: oslog echo failed while idling\n"); + return; + } + if (ep == APPLE_RTKIT_EP_MGMT) { + u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg.msg0); + unsigned int state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg.msg0); + + if (type == APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK) + rtk->iop_power_state = state; + else if (type == APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK) + rtk->ap_power_state = state; + } else if (apple_rtkit_notification_msg(ep, msg.msg0) && apple_mbox_send_ready(mbox)) { + apple_mbox_send(mbox, msg, true); + } +} + +static int apple_rtkit_polled_state(struct apple_rtkit *rtk, u8 type, unsigned int *state, + unsigned int *us) +{ + struct apple_mbox_msg msg = { + .msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type) | + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_IDLE), + .msg1 = APPLE_RTKIT_EP_MGMT, + }; + int i; + + for (i = 0; !apple_mbox_send_ready(rtk->mbox); i++) { + if (i == APPLE_RTKIT_POLLED_LOOPS) + return -ETIMEDOUT; + udelay(APPLE_RTKIT_POLLED_STEP_US); + } + apple_mbox_send(rtk->mbox, msg, true); + + for (i = 0; i < APPLE_RTKIT_POLLED_LOOPS; i++) { + apple_mbox_poll(rtk->mbox); + if (READ_ONCE(*state) == APPLE_RTKIT_PWR_STATE_IDLE) + return 0; + udelay(APPLE_RTKIT_POLLED_STEP_US); + *us += APPLE_RTKIT_POLLED_STEP_US; + } + + return -ETIMEDOUT; +} + +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us) +{ + unsigned long flags; + u64 owed; + u32 status; + int ret; + + *us = 0; + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_polled_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + /* What's already waiting is answered first, or the firmware may not go idle. */ + apple_mbox_poll(rtk->mbox); + /* Then a counter left for the receive work, which can't run now. */ + owed = xchg(&rtk->oslog_owed, 0); + if (owed) { + struct apple_mbox_msg echo = { .msg0 = owed, .msg1 = APPLE_RTKIT_EP_OSLOG }; + + if (!apple_rtkit_polled_send(rtk, echo)) + dev_err(rtk->dev, "RTKit: oslog echo failed before the idle\n"); + } + + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + &rtk->ap_power_state, us); + if (!ret) + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + &rtk->iop_power_state, us); + if (!ret) + ret = readl_poll_timeout_atomic(cpu_control + APPLE_ASC_CPU_STATUS, status, + (status & APPLE_ASC_CPU_STATUS_DOWN) == + APPLE_ASC_CPU_STATUS_DOWN, + APPLE_RTKIT_POLLED_STEP_US, 5000); + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_idle_polled); + +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk) +{ + apple_mbox_set_recv_enabled(rtk->mbox, false); +} +EXPORT_SYMBOL_GPL(apple_rtkit_mailbox_recv_off); + +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + ret = apple_rtkit_sleep_idle(rtk, cpu_control); + if (ret) + return ret; + + apple_rtkit_mailbox_off(rtk); + + /* Twice, each time from what it reads back. */ + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + writel(readl(cpu_control) & ~APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_stop); + +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + /* + * No real sleep: a firmware that idled doesn't come back from that with its power kept, + * and stays down until the next sleep restarts it. One whose idle failed still runs. + */ + if (!pm_resume_via_firmware()) { + if (!(readl(cpu_control) & APPLE_ASC_CPU_CONTROL_RUN) || + (rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON) + return -EAGAIN; + /* Its AP idle took and its IOP idle didn't: endpoints stay shut until AP on. */ + if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON) + return apple_rtkit_ap_power(rtk, true); + return 0; + } + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return); + +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret = apple_rtkit_warm_restart(rtk); + + if (ret) + return ret; + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return apple_rtkit_boot(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_power_cycled_return); + +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo) +{ + rtk->oslog_echo = echo; + dev_dbg(rtk->dev, "RTKit: OSLOG counters %s\n", echo ? "echoed" : "dropped"); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_oslog_echo); + +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control) +{ + int ret; + + if (!pm_resume_via_firmware()) + return -EOPNOTSUPP; + + ret = apple_rtkit_warm_restart(rtk); + if (ret) + return ret; + + writel(readl(cpu_control) | APPLE_ASC_CPU_CONTROL_RUN, cpu_control); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_sleep_return_start); + /* * Wake a co-processor whose CPU was stopped after apple_rtkit_idle(), with * its firmware, endpoints and shared buffers kept (and its power domain @@ -2059,7 +2615,7 @@ int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), voi ret = start(ctx); if (ret) goto out; - ret = apple_rtkit_boot_common(rtk, false); + ret = apple_rtkit_boot_common(rtk, false, NULL, NULL); out: WRITE_ONCE(rtk->retained_wake, false); if (ret) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index f9e79924e53a6..4fa31a0e54f47 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #define APPLE_SART_MAX_ENTRIES 16 @@ -90,6 +91,15 @@ struct apple_sart { unsigned long protected_entries; unsigned long used_entries; + /* The registers as they stood before a firmware-driven suspend */ + struct { + u8 flags; + phys_addr_t paddr; + size_t size; + } saved[APPLE_SART_MAX_ENTRIES]; + bool saved_valid; + /* saved[] still holds the last sleep's entries, for apple_sart_replay() */ + bool replay_valid; }; static void sart0_get_entry(struct apple_sart *sart, int index, u8 *flags, @@ -398,6 +408,98 @@ static void apple_sart_shutdown(struct platform_device *pdev) } } +/* + * A SART in a domain that a firmware-driven suspend powers off comes back empty, + * and its coprocessor can't reach its firmware or buffers until the entries are + * back. This includes the bootloader's, which Linux otherwise never writes. + */ +static int apple_sart_suspend_late(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + int i; + + sart->saved_valid = false; + sart->replay_valid = false; + if (pm_suspend_target_state != PM_SUSPEND_MEM || !pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) + sart->ops->get_entry(sart, i, &sart->saved[i].flags, + &sart->saved[i].paddr, &sart->saved[i].size); + sart->saved_valid = true; + + return 0; +} + +static int apple_sart_resume_noirq(struct device *dev) +{ + struct apple_sart *sart = dev_get_drvdata(dev); + unsigned int restored = 0; + int i; + + if (!sart->saved_valid || !pm_resume_via_firmware()) + return 0; + sart->saved_valid = false; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + /* An unused entry stays as the power left it, whatever that reads. */ + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + /* Every one in use, the same or not: macOS rewrites 0 to 7 at every wake. */ + sart->ops->set_entry(sart, i, sart->saved[i].flags, + sart->saved[i].paddr >> sart->ops->paddr_shift, + sart->saved[i].size >> sart->ops->size_shift); + restored++; + } + dev_dbg(dev, "%u entries restored after sleep\n", restored); + sart->replay_valid = true; + + return 0; +} + +/* + * A coprocessor's driver calls this right before its run bit. macOS writes the entries back + * there, config first, then address, then size, so this does the same on SARTv3. Unused + * entries stay as they read. + */ +int apple_sart_replay(struct apple_sart *sart) +{ + int i, bad = 0; + + if (sart->ops != &sart_ops_v3 || !sart->replay_valid) + return -EOPNOTSUPP; + + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + phys_addr_t paddr; + size_t size; + u8 flags; + + if (!sart->saved[i].size && !sart->saved[i].flags) + continue; + writel(sart->saved[i].flags, sart->regs + APPLE_SART3_CONFIG(i)); + writel(sart->saved[i].paddr >> APPLE_SART3_PADDR_SHIFT, + sart->regs + sart->ops->paddr_reg + 4 * i); + writel(sart->saved[i].size >> APPLE_SART3_SIZE_SHIFT, + sart->regs + sart->ops->size_reg + 4 * i); + sart->ops->get_entry(sart, i, &flags, &paddr, &size); + if (flags == sart->saved[i].flags && paddr == sart->saved[i].paddr && + size == sart->saved[i].size) + continue; + bad++; + dev_warn(sart->dev, "entry %d reads %#x %pa+%#zx, wrote %#x %pa+%#zx\n", + i, flags, &paddr, size, sart->saved[i].flags, &sart->saved[i].paddr, + sart->saved[i].size); + } + + return bad; +} +EXPORT_SYMBOL_GPL(apple_sart_replay); + +static const struct dev_pm_ops apple_sart_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_sart_suspend_late, NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, apple_sart_resume_noirq) +}; + static const struct of_device_id apple_sart_of_match[] = { { .compatible = "apple,t6050-sart", @@ -423,6 +525,7 @@ static struct platform_driver apple_sart_driver = { .driver = { .name = "apple-sart", .of_match_table = apple_sart_of_match, + .pm = pm_sleep_ptr(&apple_sart_pm_ops), }, .probe = apple_sart_probe, .shutdown = apple_sart_shutdown, diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index bed228b4d1ca3..5536de938e5ba 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,21 @@ MODULE_PARM_DESC(sensor_spi_mode, struct sep_sensor_lifetime { /* Serializes sensor operations with this device's resource teardown. */ struct mutex lock; + /* The power last written; a failed write counts as on. */ + bool powered; + /* Set across a system sleep: bus and power requests are refused. */ + bool asleep; }; +/* The power line or rails and the bus, with no settling delays. */ +struct sep_sensor_hw { + int (*set_power)(int on); + int (*send)(const void *tx, size_t len); +}; + +/* Puts the sensor in its low-power state; macOS sends it before a sleep. */ +static const u8 sep_sensor_cmd_sleep[] = { 0x80, 0x31, 0x00, 0x07, 0x00, 0x00, 0x00 }; + static DEFINE_MUTEX(sep_sensor_registry_lock); static struct sep_sensor_lifetime *sep_active_sensor; @@ -58,6 +72,7 @@ static struct sep_sensor_lifetime *sep_lock_sensor(void) } static void sep_sensor_irq_teardown_locked(void); +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len); static struct spi_device *sep_spi; static struct gpio_desc *sep_power; @@ -328,6 +343,52 @@ static void sep_release_power(void) sep_power_source = SEP_POWER_NONE; } +static int sep_sensor_set_power_now(int on) +{ + if (sep_power_source == SEP_POWER_SMC_RAILS) + return sep_smc_set_power(on); + if (!sep_power) + return -ENODEV; + gpiod_set_value_cansleep(sep_power, on); + return 0; +} + +static const struct sep_sensor_hw sep_hw = { + .set_power = sep_sensor_set_power_now, + .send = sep_sensor_xfer_tx_locked, +}; + +static int sep_sensor_write_power(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw, int on) +{ + int rc = hw->set_power(on); + + /* A failed write may have taken effect. */ + sensor->powered = rc || on; + return rc; +} + +/* + * As macOS does for a sleep: the sleep command, then the power off. The next + * bring-up's power cycle powers the sensor again, so a resume does not. + * Returns the power-off's error. + */ +static int sep_sensor_sleep_locked(struct sep_sensor_lifetime *sensor, + const struct sep_sensor_hw *hw) +{ + WRITE_ONCE(sensor->asleep, true); + if (!sensor->powered) + return 0; + /* Best effort: only the power-off matters. */ + hw->send(sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + return sep_sensor_write_power(sensor, hw, 0); +} + +static int sep_sensor_awake_locked(struct sep_sensor_lifetime *sensor) +{ + return READ_ONCE(sensor->asleep) ? -ESHUTDOWN : 0; +} + #define SEP_CS_TIMING_SOFTWARE 0 #define SEP_CS_TIMING_HOOK_BYPASSED 1 #define SEP_CS_TIMING_HARDWARE 2 @@ -462,6 +523,35 @@ static void sep_sensor_remove(struct spi_device *spi) mutex_unlock(&sep_sensor_registry_lock); } +static int sep_sensor_suspend(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + bool was_on; + int rc; + + mutex_lock(&sensor->lock); + was_on = sensor->powered; + rc = sep_sensor_sleep_locked(sensor, &sep_hw); + mutex_unlock(&sensor->lock); + /* Failing the sleep would keep the machine awake; the next bring-up recovers. */ + if (rc) + dev_warn(dev, "sep sensor: power-off for sleep failed (%d)\n", rc); + else if (was_on) + dev_dbg(dev, "sep sensor: off for sleep\n"); + return 0; +} + +/* No lock: a capture still waiting on the data-ready line may hold it. */ +static int sep_sensor_resume(struct device *dev) +{ + struct sep_sensor_lifetime *sensor = dev_get_drvdata(dev); + + WRITE_ONCE(sensor->asleep, false); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(sep_sensor_pm_ops, sep_sensor_suspend, sep_sensor_resume); + /* * Its own compatible, deliberately not spidev's: capture must stay in the * kernel, and a spidev node would leak an image to userspace. No @@ -485,6 +575,7 @@ static struct spi_driver sep_sensor_driver = { .driver = { .name = "apple-mesa", .of_match_table = sep_sensor_of_match, + .pm = pm_sleep_ptr(&sep_sensor_pm_ops), }, .id_table = sep_sensor_spi_ids, .probe = sep_sensor_probe, @@ -528,26 +619,16 @@ static int sep_sensor_cs_timing_mode_locked(void) } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -static int sep_sensor_power_cycle_locked(void) +static int sep_sensor_power_cycle_locked(struct sep_sensor_lifetime *sensor) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(0); - - if (rc) - return rc; - msleep(SEP_SENSOR_OFF_DELAY_MS); - rc = sep_smc_set_power(1); - if (rc) - return rc; - msleep(SEP_SENSOR_ON_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; + int rc = sep_sensor_write_power(sensor, &sep_hw, 0); - gpiod_set_value_cansleep(sep_power, 0); + if (rc) + return rc; msleep(SEP_SENSOR_OFF_DELAY_MS); - gpiod_set_value_cansleep(sep_power, 1); + rc = sep_sensor_write_power(sensor, &sep_hw, 1); + if (rc) + return rc; msleep(SEP_SENSOR_ON_DELAY_MS); return 0; } @@ -588,24 +669,13 @@ static const char *sep_sensor_firmware_name_locked(unsigned int index) } /* Powers the sensor on/off, holding the hardware settling delay. */ -static int sep_sensor_power_locked(int on) +static int sep_sensor_power_locked(struct sep_sensor_lifetime *sensor, int on) { - if (sep_power_source == SEP_POWER_SMC_RAILS) { - int rc = sep_smc_set_power(on ? 1 : 0); + int rc = sep_sensor_write_power(sensor, &sep_hw, on ? 1 : 0); - if (rc) - return rc; - msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); - return 0; - } - if (!sep_power) - return -ENODEV; - - gpiod_set_value_cansleep(sep_power, on ? 1 : 0); - if (on) - msleep(SEP_SENSOR_ON_DELAY_MS); - else - msleep(SEP_SENSOR_OFF_DELAY_MS); + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); return 0; } @@ -846,7 +916,7 @@ int sep_sensor_power_cycle(void) if (!sensor) return -ENODEV; - result = sep_sensor_power_cycle_locked(); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_cycle_locked(sensor); mutex_unlock(&sensor->lock); return result; } @@ -894,7 +964,7 @@ int sep_sensor_power(int on) if (!sensor) return -ENODEV; - result = sep_sensor_power_locked(on); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_power_locked(sensor, on); mutex_unlock(&sensor->lock); return result; } @@ -962,7 +1032,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_locked(tx, rx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_locked(tx, rx, len); mutex_unlock(&sensor->lock); return result; } @@ -974,7 +1044,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer_tx_locked(tx, len); + result = sep_sensor_awake_locked(sensor) ?: sep_sensor_xfer_tx_locked(tx, len); mutex_unlock(&sensor->lock); return result; } @@ -986,7 +1056,13 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) if (!sensor) return -ENODEV; - result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + result = sep_sensor_awake_locked(sensor) ?: + sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); mutex_unlock(&sensor->lock); return result; } + +/* Like the module's Rust tests, which need KUnit built in. */ +#if IS_ENABLED(CONFIG_APPLE_SEP_KUNIT_TEST) && IS_BUILTIN(CONFIG_KUNIT) +#include "sensor_shim_test.c" +#endif diff --git a/drivers/soc/apple/sensor_shim_test.c b/drivers/soc/apple/sensor_shim_test.c new file mode 100644 index 0000000000000..a0374e65bbf5f --- /dev/null +++ b/drivers/soc/apple/sensor_shim_test.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* KUnit tests for the fingerprint sensor's system sleep. No hardware is touched. */ + +#include +#include + +struct sep_fake_hw { + /* 's' for a send, '0' and '1' for power writes, in order. */ + char calls[8]; + unsigned int ncalls; + u8 sent[16]; + size_t sent_len; + int send_rc; + int power_rc; +}; + +static struct sep_fake_hw *sep_fake(void) +{ + return kunit_get_current_test()->priv; +} + +static void sep_fake_note(struct sep_fake_hw *f, char call) +{ + if (f->ncalls < ARRAY_SIZE(f->calls)) + f->calls[f->ncalls++] = call; +} + +static int sep_fake_send(const void *tx, size_t len) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, 's'); + f->sent_len = min(len, sizeof(f->sent)); + memcpy(f->sent, tx, f->sent_len); + return f->send_rc; +} + +static int sep_fake_set_power(int on) +{ + struct sep_fake_hw *f = sep_fake(); + + sep_fake_note(f, on ? '1' : '0'); + return f->power_rc; +} + +static const struct sep_sensor_hw sep_fake_hw = { + .set_power = sep_fake_set_power, + .send = sep_fake_send, +}; + +static int sep_sensor_test_init(struct kunit *test) +{ + test->priv = kunit_kzalloc(test, sizeof(struct sep_fake_hw), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, test->priv); + return 0; +} + +static void sep_sensor_sleep_sends_the_command_then_cuts_power(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_EQ(test, f->sent_len, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_MEMEQ(test, f->sent, sep_sensor_cmd_sleep, sizeof(sep_sensor_cmd_sleep)); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_leaves_an_unpowered_sensor_alone(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = false }; + + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_EXPECT_EQ(test, f->ncalls, 0U); + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_sleep_cuts_power_when_the_command_fails(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->send_rc = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), 0); + KUNIT_ASSERT_EQ(test, f->ncalls, 2U); + KUNIT_EXPECT_MEMEQ(test, f->calls, "s0", 2); + KUNIT_EXPECT_FALSE(test, sensor.powered); +} + +static void sep_sensor_sleep_counts_a_failed_power_off_as_on(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = { .powered = true }; + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_sleep_locked(&sensor, &sep_fake_hw), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_TRUE(test, sensor.asleep); +} + +static void sep_sensor_requests_wait_for_the_resume(struct kunit *test) +{ + struct sep_sensor_lifetime sensor = { .powered = true }; + struct device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + sep_sensor_sleep_locked(&sensor, &sep_fake_hw); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), -ESHUTDOWN); + + dev_set_drvdata(dev, &sensor); + KUNIT_EXPECT_EQ(test, sep_sensor_resume(dev), 0); + KUNIT_EXPECT_EQ(test, sep_sensor_awake_locked(&sensor), 0); + /* The resume powers nothing: the next bring-up does. */ + KUNIT_EXPECT_FALSE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, ((struct sep_fake_hw *)test->priv)->ncalls, 2U); +} + +static void sep_sensor_power_writes_are_tracked(struct kunit *test) +{ + struct sep_fake_hw *f = test->priv; + struct sep_sensor_lifetime sensor = {}; + + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), 0); + KUNIT_EXPECT_TRUE(test, sensor.powered); + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), 0); + KUNIT_EXPECT_FALSE(test, sensor.powered); + + f->power_rc = -EIO; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 1), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); + sensor.powered = false; + KUNIT_EXPECT_EQ(test, sep_sensor_write_power(&sensor, &sep_fake_hw, 0), -EIO); + KUNIT_EXPECT_TRUE(test, sensor.powered); +} + +/* Framed like the driver's other commands: 0x80, opcode, then the length at 3. */ +static void sep_sensor_sleep_command_is_framed(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, sizeof(sep_sensor_cmd_sleep), (size_t)7); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[0], (u8)0x80); + KUNIT_EXPECT_EQ(test, sep_sensor_cmd_sleep[1], (u8)0x31); + KUNIT_EXPECT_EQ(test, (size_t)get_unaligned_le16(&sep_sensor_cmd_sleep[3]), + sizeof(sep_sensor_cmd_sleep)); +} + +static struct kunit_case sep_sensor_sleep_cases[] = { + KUNIT_CASE(sep_sensor_sleep_sends_the_command_then_cuts_power), + KUNIT_CASE(sep_sensor_sleep_leaves_an_unpowered_sensor_alone), + KUNIT_CASE(sep_sensor_sleep_cuts_power_when_the_command_fails), + KUNIT_CASE(sep_sensor_sleep_counts_a_failed_power_off_as_on), + KUNIT_CASE(sep_sensor_requests_wait_for_the_resume), + KUNIT_CASE(sep_sensor_power_writes_are_tracked), + KUNIT_CASE(sep_sensor_sleep_command_is_framed), + {} +}; + +static struct kunit_suite sep_sensor_sleep_suite = { + .name = "apple_sep_sensor_sleep", + .init = sep_sensor_test_init, + .test_cases = sep_sensor_sleep_cases, +}; + +kunit_test_suite(sep_sensor_sleep_suite); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 386e32eb5f2a9..206551ac2d4a6 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -61,8 +61,10 @@ use kernel::{ sync::{ aref::ARef, atomic::{ + Acquire, Atomic, - Relaxed, // + Relaxed, + Release, // }, barrier::{ dma_mb, @@ -643,6 +645,12 @@ struct SepData { // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, + // A `pm::SleepGate`. + sleep_gate: Atomic, + + // The mailbox's receive interrupt is held off for the enclave's sleep. + sleep_rx_held: Atomic, + #[pin] ool_xarm: Mutex>, @@ -1083,6 +1091,8 @@ impl SepData { sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), + sleep_gate: Atomic::new(pm::SleepGate::Open as u32), + sleep_rx_held: Atomic::new(false), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), @@ -1338,7 +1348,200 @@ impl SepData { if self.shutting_down.load(Relaxed) { return Err(ENODEV); } - Self::ring(self.mbox.lock().as_ref().ok_or(ENODEV)?, msg) + // Checked with the mailbox lock held: a sender can wait for it for longer + // than the system takes to tell the enclave it's going to sleep. + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_request() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + /// As [`Self::send`], for an answer to something the enclave asked: it + /// waits for these before it can answer the sleep request. + fn send_reply(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.sleep_gate().may_reply() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + + pub(crate) fn sleep_gate(&self) -> pm::SleepGate { + pm::SleepGate::from_u32(self.sleep_gate.load(Acquire)) + } + + /// Under the mailbox lock, so that a sender holding it finishes first. + fn set_sleep_gate(&self, gate: pm::SleepGate) { + let _mbox = self.mbox.lock(); + self.sleep_gate.store(gate as u32, Release); + } + + /// Prepare a firmware-driven suspend without sending SLEEP. The platform + /// owns the irreversible request, after every returnable refusal. + pub(crate) fn sleep_enter(&self) -> Result<()> { + match self.sleep_gate() { + // A failed earlier wake is not repaired by another precommit attempt. + pm::SleepGate::Lost => return Err(EIO), + pm::SleepGate::Open => {} + _ => return Err(EBUSY), + } + // Do not commit a running enclave whose sleep protocol is not ready. + if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { + dev_warn!( + self.dev, + "sleep refused: the enclave did not finish coming up\n" + ); + return Err(EBUSY); + } + + self.set_sleep_gate(pm::SleepGate::Draining); + let idle = control::wait_idle( + proto::CONTROL_TIMEOUT_MS, + 10, + || self.control.lock().requests_in_flight(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_err!( + self.dev, + "sleep refused: a request is still waiting for its reply\n" + ); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(EBUSY); + } + + if let Err(e) = self.sleep_leave_for_commit() { + dev_err!( + self.dev, + "sleep refused: irreversible request could not be staged ({:?})\n", + e + ); + self.set_sleep_gate(pm::SleepGate::Open); + return Err(e); + } + Ok(()) + } + + /// Stage without sending; the platform's commit owns SLEEP with IRQs off. + fn sleep_leave_for_commit(&self) -> Result<()> { + let request = proto::encode_control(&proto::op_sleep(), proto::TAG_SLEEP); + match self.mbox.lock().as_ref() { + Some(mbox) => pm::leave_for_commit( + &self.dev, + mbox.as_raw(), + request, + proto::SLEEP_REPLY, + proto::SLEEP_REPLY_MASK, + ), + None => Err(ENODEV), + }?; + self.sleep_hold_rx(); + dev_dbg!(self.dev, "SEP SLEEP staged for the irreversible commit\n"); + Ok(()) + } + + /// Holds off the receive interrupt, whose handler reads the mailbox. + fn sleep_hold_rx(&self) { + if !self.sleep_rx_held.xchg(true, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.hold_rx(); + } + } + } + + fn sleep_release_rx(&self) { + if self.sleep_rx_held.xchg(false, Relaxed) { + if let Some(mbox) = self.mbox.lock().as_ref() { + mbox.release_rx(); + } + } + } + + /// A refused precommit attempt was never sent; reopen normal admission. + /// A sent request can be restarted only after actual physical sleep/wake. + pub(crate) fn sleep_exit(&self) { + // Staged only: an unsent refusal releases RX without a restart. + match pm::commit_finish(&self.dev) { + Some(true) => self.set_sleep_gate(pm::SleepGate::Told), + Some(false) => self.sleep_release_rx(), + None => {} + } + let gate = self.sleep_gate(); + if gate == pm::SleepGate::Draining { + self.set_sleep_gate(pm::SleepGate::Open); + return; + } + if !gate.needs_wake() { + return; + } + // SAFETY: only reads a flag. + let real = unsafe { bindings::pm_resume_via_firmware() }; + if !real && gate == pm::SleepGate::Lost { + return; + } + if !real { + // commit_finish already terminates a current sent-without-sleep case. + // An old lost state must not cause mailbox or DART accesses here. + self.set_sleep_gate(pm::SleepGate::Lost); + return; + } + let answered = self.sleep_kick(); + match answered { + Ok(()) => { + self.set_sleep_gate(pm::SleepGate::Open); + // One line per sleep it was told of: nothing else says it came back. + dev_info!(self.dev, "the enclave is back from its sleep request\n"); + } + Err(e) => { + dev_err!( + self.dev, + "the enclave did not come back ({:?}); it stays unused until a reboot\n", + e + ); + self.sleep_hold_rx(); + self.set_sleep_gate(pm::SleepGate::Lost); + } + } + } + + /// Starts a gated enclave and waits for the messages that say it's up. + fn sleep_kick(&self) -> Result<()> { + pm::clear_dev_disable(&self.dev)?; + pm::dart_restore(&self.dev); + // Arm completion before the held receive IRQ can deliver an early wake. + self.control.lock().wake_reset(); + self.sleep_release_rx(); + match self.mbox.lock().as_ref() { + Some(mbox) => { + mbox.set_recv_enabled(true); + Self::ring(mbox, proto::WAKE_KICK)?; + } + None => return Err(ENODEV), + } + let mut remaining = time::msecs_to_jiffies(proto::CONTROL_TIMEOUT_MS); + let mut guard = self.control.lock(); + loop { + if guard.wake_complete() { + return Ok(()); + } + if remaining == 0 { + return Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } } /// Puts `msg` in the mailbox once the enclave can see what the CPU wrote @@ -1364,6 +1567,16 @@ impl SepData { return Err(e); } + self.control_wait(op, idx, tag) + } + + /// Waits for the answer to the request sent under `idx` and `tag`. + fn control_wait( + &self, + op: &proto::ControlOp, + idx: usize, + tag: u8, + ) -> Result> { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { @@ -1501,7 +1714,7 @@ impl SepData { } let mbox = self.mbox.lock(); let mbox = mbox.as_ref().ok_or(ENODEV)?; - if !self.control.lock().entropy_may_send() { + if !self.sleep_gate().may_request() || !self.control.lock().entropy_may_send() { return Err(EAGAIN); } Self::ring(mbox, msg) @@ -1935,7 +2148,7 @@ impl SepData { ); } let msg = crate::xarm::encode_xarm_reply(reply); - let _ = self.send(msg); + let _ = self.send_reply(msg); } fn release_deferred_query(&self) { @@ -2647,6 +2860,8 @@ impl SepData { fn on_control(&self, msg: Message, f: proto::Fields) { if f.ty != proto::CONTROL_REPLY_TYPE { + self.control.lock().wake_note(f.ty); + self.control_wq.notify_all(); return; } @@ -2999,6 +3214,11 @@ impl platform::Driver for SepDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + // SAFETY: a static in `pm_shim.c`, never written. + Some(unsafe { &*core::ptr::addr_of!(pm::sep_pm_ops) }) + } + fn probe<'bound>( pdev: &'bound platform::Device>, _info: Option<&'bound Self::IdInfo>, diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 63713828e7f66..0ba2518bb3696 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -19,15 +19,23 @@ #include struct apple_sep_fv_new_file_key; +struct dev_pm_ops; void sep_cancel_work_sync(void *work); void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void (*event)(bool entering)); +int sep_pm_register(void *dev, void (*event)(bool entering), int (*sleep)(void), + void (*wake)(void), bool (*told)(void)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); +int sep_pm_wait_gated(void *dev, unsigned int timeout_ms); +int sep_pm_clear_dev_disable(void *dev); +void sep_pm_dart_restore(void *dev); +int sep_pm_commit_prepare(void *dev, void *mbox, u64 msg0, u32 msg1, u64 reply, u64 mask); +int sep_pm_commit_finish(void *dev); +extern const struct dev_pm_ops sep_pm_ops; /* -- hwrng_shim.c ------------------------------------------------------- */ diff --git a/drivers/soc/apple/sleep-test.c b/drivers/soc/apple/sleep-test.c new file mode 100644 index 0000000000000..90b7661afd198 --- /dev/null +++ b/drivers/soc/apple/sleep-test.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by sleep.c: each matched SoC's data. A word a list names but the SoC's full list + * lacks is skipped silently at the wake, and a duplicate splits its books. + */ +#include + +/* Matched only once its sleep has been tested on hardware. */ +static const char *const apple_sleep_test_tested[] = { + "apple,t8140-sleep", +}; + +static bool apple_sleep_test_has(const struct apple_sleep_words *all, u32 offset) +{ + return sleep_ps_find(all->offset, all->count, offset) >= 0; +} + +static void apple_sleep_test_in(struct kunit *test, const char *compat, const char *list, + const struct apple_sleep_words *all, u32 offset) +{ + KUNIT_EXPECT_TRUE_MSG(test, apple_sleep_test_has(all, offset), + "%s: %s word %#x isn't in its full list", compat, list, offset); +} + +static void apple_sleep_test_list_in(struct kunit *test, const char *compat, const char *list, + const struct apple_sleep_words *all, + const struct apple_sleep_words *words) +{ + int i; + + for (i = 0; i < words->count; i++) + apple_sleep_test_in(test, compat, list, all, words->offset[i]); +} + +static void apple_sleep_test_full(struct kunit *test, const char *compat, const char *list, + const struct apple_sleep_words *all) +{ + int i; + + KUNIT_EXPECT_GT_MSG(test, all->count, 0, "%s: %s is empty", compat, list); + for (i = 0; i < all->count; i++) { + KUNIT_EXPECT_EQ_MSG(test, all->offset[i] % 4, 0, "%s: %s word %#x", compat, list, + all->offset[i]); + KUNIT_EXPECT_EQ_MSG(test, sleep_ps_find(all->offset, all->count, all->offset[i]), i, + "%s: %s has %#x twice", compat, list, all->offset[i]); + } +} + +static void apple_sleep_test_only_tested(struct kunit *test) +{ + const struct of_device_id *id; + int n = 0; + + for (id = apple_sleep_of_match; id->compatible[0]; id++, n++) { + KUNIT_EXPECT_NOT_NULL_MSG(test, id->data, "%s has no data", id->compatible); + KUNIT_ASSERT_LT(test, n, ARRAY_SIZE(apple_sleep_test_tested)); + KUNIT_EXPECT_STREQ(test, id->compatible, apple_sleep_test_tested[n]); + } + KUNIT_EXPECT_EQ(test, n, ARRAY_SIZE(apple_sleep_test_tested)); +} + +static void apple_sleep_test_words(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = apple_sleep_of_match; id->compatible[0]; id++) { + const struct apple_sleep_soc *soc = id->data; + const struct apple_sleep_domains *d = &soc->ps; + const struct apple_sleep_words *all = &soc->ps_all, *aon = &soc->ps_aon_all; + const char *c = id->compatible; + u32 named[] = { + d->disp[0][0], d->disp[0][1], d->disp[0][2], d->disp[1][0], d->disp[1][1], + d->disp[1][2], d->sio_cpu, d->sep, d->usb_aon, d->usb_common, d->usb, + d->ap_tmm, d->gpio, d->ans, d->apcie_gp, d->apcie_sys_gp, d->apcie_st, + d->apcie_sys_st, d->apcie_phy_sw, + }; + int i; + + apple_sleep_test_full(test, c, "ps_all", all); + apple_sleep_test_full(test, c, "ps_aon_all", aon); + for (i = 0; i < ARRAY_SIZE(named); i++) + apple_sleep_test_in(test, c, "named", all, named[i]); + apple_sleep_test_in(test, c, "debug switch", aon, soc->ps_aon_debug_switch); + apple_sleep_test_list_in(test, c, "ps_macos_on", all, &soc->ps_macos_on); + apple_sleep_test_list_in(test, c, "ps_macos_off", all, &soc->ps_macos_off); + apple_sleep_test_list_in(test, c, "ps_wake_order", all, &soc->ps_wake_order); + apple_sleep_test_list_in(test, c, "ps_still_on", all, &soc->ps_still_on); + apple_sleep_test_list_in(test, c, "ps_aon_macos_off", aon, &soc->ps_aon_macos_off); + apple_sleep_test_list_in(test, c, "ps_aon_macos_no_auto", aon, + &soc->ps_aon_macos_no_auto); + apple_sleep_test_list_in(test, c, "ps_aon_wake_order", aon, + &soc->ps_aon_wake_order); + for (i = 0; i < soc->nstray; i++) + apple_sleep_test_in(test, c, soc->stray[i].name, all, soc->stray[i].offset); + for (i = 0; i < soc->ngate_off; i++) + apple_sleep_test_in(test, c, soc->gate_off[i].name, all, + soc->gate_off[i].offset); + for (i = 0; i < soc->ngate_coprocs; i++) { + KUNIT_EXPECT_TRUE(test, soc->gate_coprocs[i].path[0] == '/'); + if (soc->gate_coprocs[i].ps) + apple_sleep_test_in(test, c, soc->gate_coprocs[i].path, all, + soc->gate_coprocs[i].ps); + } + } +} + +/* The books keep one bit for each stray word and one gate for each coprocessor. */ +static void apple_sleep_test_books(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = apple_sleep_of_match; id->compatible[0]; id++) { + const struct apple_sleep_soc *soc = id->data; + + KUNIT_EXPECT_LE(test, soc->nstray, BITS_PER_LONG); + KUNIT_EXPECT_LE(test, soc->ngate_coprocs, + ARRAY_SIZE(((struct apple_sleep *)0)->gate)); + } +} + +static void apple_sleep_test_wake_names(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = apple_sleep_of_match; id->compatible[0]; id++) { + const struct apple_sleep_soc *soc = id->data; + int i, j; + + KUNIT_EXPECT_NE(test, soc->wake_last_word, 0); + for (i = 0; i < soc->nwake_names; i++) { + u32 n = soc->wake_names[i].id; + + KUNIT_EXPECT_LT_MSG(test, n, WAKE_WORDS * 32, "%s", + soc->wake_names[i].name); + if (n / 32 == WAKE_WORDS - 1) + KUNIT_EXPECT_TRUE_MSG(test, soc->wake_last_word & BIT(n % 32), "%s", + soc->wake_names[i].name); + for (j = 0; j < i; j++) + KUNIT_EXPECT_NE(test, soc->wake_names[j].id, n); + } + } +} + +static struct kunit_case apple_sleep_test_cases[] = { + KUNIT_CASE(apple_sleep_test_only_tested), + KUNIT_CASE(apple_sleep_test_words), + KUNIT_CASE(apple_sleep_test_books), + KUNIT_CASE(apple_sleep_test_wake_names), + {} +}; + +static struct kunit_suite apple_sleep_test_suite = { + .name = "apple-sleep-soc-data", + .test_cases = apple_sleep_test_cases, +}; +kunit_test_suite(apple_sleep_test_suite); diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c new file mode 100644 index 0000000000000..73befaeb87d1b --- /dev/null +++ b/drivers/soc/apple/sleep.c @@ -0,0 +1,3563 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC system sleep + * + * On some Apple SoCs a kernel can run directly at EL2 behind the CTXR execute + * window (Documentation/arch/arm64/ctxr-execmem.rst), with no monitor below it. + * So no PSCI answers a request to power a CPU off or to suspend the system, and + * the kernel does both itself. + * + * iBoot locks every core's reset vector to the boot object. A core that comes + * back from a power-off therefore restarts inside the boot object (aura), and + * that has to send it back here. The boot object opts in by publishing a mailbox + * in /chosen/aurora,s2r-mailbox, which also gives its spin-table CPUs the parking + * spin table (arch/arm64/kernel/cpu_ops.c). Without it this driver does nothing: + * + * - A secondary core goes back through its spin-table release address, which + * the boot object keeps, into secondary_holding_pen. + * - The boot core takes cpu_resume from the mailbox: EL2, MMU off. + * + * The register addresses are in the device tree. The ADT has every one of them, + * as /arm-io/pmgr's or /arm-io/wdt's reg entries. Register offsets and order not + * otherwise explained here were measured from a macOS sleep under aura. + * + * What the kernel does in suspend-to-RAM is what macOS does in its last 14 ms, + * with the order kept. What differs between SoCs, their power words and the + * lists macOS was measured with, is each SoC's match data. Only SoCs measured + * and tested that way are matched: on the others this driver doesn't bind, and + * s2idle stays the only sleep. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Must match aura's struct aura_s2r_mailbox. */ +#define AURA_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ + +struct aura_s2r_mailbox { + u64 magic; /* aura, before the kernel starts */ + u64 boot_mpidr; /* aura: the only core that may take resume_pa */ + u64 resume_pa; /* kernel, armed before a commit; aura clears it */ + u64 resumes; /* aura: resume_pa taken this many times */ +}; + +/* PS word */ +#define PS_AUTO_ENABLE BIT(28) +#define PS_DEV_DISABLE BIT(10) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_OFF_TIMEOUT_US 5000 + +/* CPU start block, as aura's smp_start_cpu() uses it */ +#define CPU_START_STOP 0x0 +#define CPU_START_SYS 0x4 +#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) + +/* Power-gate page: reads back this value already. Purpose unknown. */ +#define PWRGATE_ON 0x80000000 + +/* Three clock muxes macOS writes before the commit. */ +#define CLK_MUX_PARKED 0x80100000 +#define CLK_MUX_34 0x0 +#define CLK_MUX_38 0x4 +#define CLK_MUX_3C 0x8 + +/* Cluster control: only these three words, the next one is locked. */ +#define CLUSTER_E 0x0 +#define CLUSTER_P 0x4 +#define CLUSTER_AVE 0x8 +#define CLUSTER_ON 0xf +/* Set when a complex's up sequence ends, and cleared by writing 1. */ +#define CLUSTER_UP_DONE BIT(5) +#define CLUSTER_E_READY BIT(5) +#define CLUSTER_TIMEOUT_US 5000 + +/* CLLT: names from nothing. Bits measured cleared by macOS before the commit. */ +#define CLLT_0 0x30 +#define CLLT_1 0x34 +#define CLLT_2 0x3c +#define CLLT_0_CLEAR GENMASK(7, 0) +#define CLLT_1_CLEAR GENMASK(7, 0) +#define CLLT_2_CLEAR GENMASK(17, 16) + +/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ +#define WDT_WD0_COUNT 0x00 +#define WDT_WD0_CTL 0x0c +#define WDT_WD1_COUNT 0x10 +#define WDT_WD2_COUNT 0x20 +#define WDT_WD2_ALARM 0x28 +#define WDT_WD2_CTL 0x2c +#define WDT_TICKS_PER_S 24000000 +/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ +#define WDT_SLEEP_SECS 5 +/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ +#define WDT_SUSPEND_SECS 120 + +#define AON_AP_DEADLINE 0x08 + +/* Wake sources: a mask, so 0 enables. Status words are sticky. */ +#define WAKE_MASK(word) (0x08 + (word) * 4) +#define WAKE_STATUS(word) (0x28 + (word) * 4) +#define WAKE_WORDS 8 +#define WAKE_SOURCES_MAX 32 + +/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ +#define SPMI_STATUS 0x00 +#define SPMI_STATUS_RX_EMPTY BIT(24) +#define SPMI_CMD 0x04 +#define SPMI_REPLY 0x08 +#define SPMI_NOTIFY_ENABLE 0x40 +#define SPMI_NOTIFY_ENABLE_SLEEP 0x2000001 +#define SPMI_NOTIFY_REPLY BIT(0) +#define SPMI_NOTIFY_ACK 0x80 +#define SPMI_CMD_ACTIVE BIT(15) +#define SPMI_OP_EXT_WRITE(n) ((n) - 1) +#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) +#define SPMI_OP_ZERO_WRITE 0x80 +#define SPMI_OP_SLEEP 0x11 +#define SPMI_OP_WAKEUP 0x13 +#define SPMI_POLL_US 20000 + +/* A port controller on SPMI: registers go through a page select, then a data window. */ +#define HPM_REG_SELECT 0x00 +#define HPM_REG_DATA 0x20 +#define HPM_REG_DATA_CMD 0xa0 +#define HPM_PAGE_CMD1 0x08 +#define HPM_PAGE_DATA1 0x09 +#define HPM_CMD_SSPS 0x53505353 +#define HPM_CMD_COLD_RESET 0x64696147 +#define HPM_SSPS_S3 3 +#define HPM_RESET_MS 1000 +#define HPM_PAGE_POWER_STATUS 0x3f +#define HPM_POWER_CONNECTED BIT(0) +#define HPM_POWER_TYPEC_MODE GENMASK(3, 2) +#define HPM_POWER_TYPEC_PD 3 +#define HPM_CONTRACT_MS 5000 +#define HPM_RESET_TRIES 3 + +/* A PMU register write on the sleep's SPMI bus */ +#define SPMI_CMD_WRITE (BIT(15) | (0xe << 8) | 0x30) +#define PMU_BOOT_STAGE 0xf701 +#define PMU_BOOT_STAGE_SLEEP 0x50 + +/* Three of the memory controller's channels, and what macOS steps down on each. */ +#define MCC_CHANNELS 3 +#define MCC_ENABLES 0x4 +#define MCC_LANES 0x8 +#define MCC_LANES_MASK 0xc +#define MCC_LANE_STROBE(n) BIT(16 + 4 * (n)) +#define MCC_LANE(n) BIT(4 * (n)) + +/* Two words macOS zeroes just before the power gates. */ +#define PTD_A 0x0 +#define PTD_B 0x8 + +/* Perf-state words, each like DCS's. The SoC's is first. */ +#define CLK_PERF_SOC 0x0 +#define SOC_SLEEP_STATE 3 + +/* The PMC's voter agents, each with a word holding its floor for every rail. */ +#define PMC_VOTER_AGENTS 4 +#define PMC_VOTER_STRIDE 0x4000 + +/* DCS: the low nibble asks for a state, and [11:8] is the state it has reached. */ +#define DCS_STATE GENMASK(11, 8) +#define DCS_BUSY BIT(31) +#define DCS_SLEEP_STATE 4 +#define DCS_STEP_STATE 6 +#define DCS_TIMEOUT_US 5000 + +/* + * Word 0 of the wake block. Bit 0 lets the sleep keep DRAM for the USB host's always-on side. + * macOS sets it from the host's start to its stop, not at the commit. + */ +#define WAKE_SLP_DDR 0x00 +#define WAKE_SLP_DDR_USB BIT(0) + +/* + * AMCC error enables. Any error enabled and pending at the commit turns the + * sleep around within 20 ms with no error handler to service it. Linux has none. + * Measured: with macOS's handler hidden, as Linux's position is, 6 of 6 sleeps + * held with both of these off, and bit 25 alone still bounced. Their status + * stays set; only the enables matter. A deep sleep sets them again, so this has + * to be done before every commit. + */ +#define AMCC_WORD0_ENABLE 0x0 +#define AMCC_WORD1_ENABLE 0x8 +#define AMCC_WORD0_DSID_AGE_OUT BIT(25) +#define AMCC_WORD1_UNEXP_RT_HIT BIT(8) + +/* A coprocessor's ASC */ +#define ASC_CPU_CONTROL 0x44 +#define ASC_CPU_CONTROL_RUN BIT(4) +#define ASC_CPU_STATUS 0x48 + +/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ +#define APCIE_AXI_EN0 0x104 +#define APCIE_AXI_EN1 0x108 +#define APCIE_CLK_CTRL 0x0 +#define APCIE_CLK_BUSY BIT(31) +#define APCIE_CLK_RUN BIT(5) +#define APCIE_CLK_OFF BIT(1) +#define APCIE_CLK_OFF2 BIT(2) +#define APCIE_CLK_START 0x4 +#define APCIE_CLK_START_GO BIT(31) +#define APCIE_PHY_COMMON 0x4000 +#define APCIE_PHY_CTRL 0x8000 +#define APCIE_PHY_CTRL_CLK0REQ BIT(0) +#define APCIE_PHY_CTRL_CLK1REQ BIT(1) +#define APCIE_PHY_CTRL_CLK0ACK BIT(2) +#define APCIE_PHY_CTRL_CLK1ACK BIT(3) +#define APCIE_PHY_CTRL_RESET BIT(4) +#define APCIE_PHY_CTRL_UP BIT(27) +#define APCIE_PHY_RUN 0x8004 +#define APCIE_PHY_RUN_EN BIT(0) +#define APCIE_PHY_RUN_EN2 BIT(4) +#define APCIE_PHY_READY 0x8008 +#define APCIE_PHY_IP_PLL_CTL 0x90 +#define APCIE_PHY_IP_PLL_HOLD BIT(16) +#define APCIE_PORT_INTSTAT 0x100 +#define APCIE_PORT_INTMSK 0x104 +#define APCIE_PORT_INT_LINK GENMASK(7, 4) +#define APCIE_PORT_L2 0x88 +#define APCIE_PORT_L2_ENTER 0x11 +#define APCIE_PORT_L2_BUSY BIT(0) +#define APCIE_PORT_PERST_GUARD 0x13c +#define APCIE_PORT_LINKSTS 0x208 +#define APCIE_PORT_LINKSTS_UP BIT(0) +#define APCIE_PORT_APPCLK 0x800 +#define APCIE_PORT_APPCLK_EN BIT(0) +#define APCIE_PORT_APPCLK_CGDIS BIT(8) +#define APCIE_PORT_PERST 0x82c +#define APCIE_PORT_PHY_LINK BIT(31) +#define APCIE_PORT_PHY_EN0 BIT(9) +#define APCIE_PORT_PHY_EN1 BIT(10) +#define APCIE_ENGINE_CTRL 0x0 +#define APCIE_ENGINE_IRQ_MASK 0x8 +#define APCIE_ENGINE_STOP BIT(0) +#define APCIE_FABRIC_STOP 0x4 +#define APCIE_PORT_STOP_US 5000 +#define APCIE_RC_ZERO 0x04 +#define APCIE_RC_READY_START 0x50 +#define APCIE_RC_READY_CFG 0x54 +#define APCIE_RC_READY_STAT 0x58 +#define APCIE_POLL_US 50000 + +/* The coprocessors the last mile stops. Each driver idles its own first. */ +#define SLEEP_COPROCS 4 + +static struct apple_sleep_coproc { + struct device *dev; + void __iomem *asc; + bool pcie_ans; + int (*last_check)(void *data); + int (*last_stop)(void *data); + void *data; + bool ready; + bool stopped; + u32 status; + int stop_ret; + /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ + bool first; + u32 ps; +} apple_sleep_coprocs[SLEEP_COPROCS]; +static DEFINE_MUTEX(apple_sleep_coprocs_lock); + +/* Irreversible enclave stop; nonzero is legal only if SLEEP was not sent. */ +static struct { + struct device *dev; + int (*stop)(void *data); + void *data; +} apple_sleep_enclave; + +/* + * Written with bits 0 and 63 by every core right before its last WFI, measured + * from macOS. This is not CYC_OVRD, which is undefined on this SoC. + */ +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) + +#define SLEEP_DISPLAY_COUNT 2 + +/* CPU clusters: 0 is E, 1 is P */ +#define SLEEP_CLUSTERS 2 + +/* cpm-dvfm */ +#define DVFM_CTL 0x178 +#define DVFM_WORD_0 0x140 +#define DVFM_WORD_1 0x160 +#define DVFM_WORD_2 0x190 +#define DVFM_WORD_3 0x198 +#define DVFM_ENABLE BIT_ULL(63) + +/* In macOS's order, on the way down. */ +static const u32 sleep_dvfm_words[] = { + DVFM_WORD_0, DVFM_WORD_1, DVFM_WORD_2, DVFM_WORD_3, +}; + +/* A power word by its offset, named for the log */ +struct apple_sleep_word { + const char *name; + u32 offset; +}; + +/* Power words by their offsets */ +struct apple_sleep_words { + const u32 *offset; + unsigned int count; +}; + +#define SLEEP_WORDS(a) { .offset = (a), .count = ARRAY_SIZE(a) } + +/* + * A coprocessor that must be bound for a sleep, by its path. @must_stop: it must also have + * stopped by the last step. @ps: its CPU's own domain, where a driver may power it off. + */ +struct apple_sleep_gate_coproc { + const char *path; + bool must_stop; + u32 ps; +}; + +/* A source in the ADT's pmgr/ap-wake-sources: for the log, and to tell the SMC's apart */ +struct apple_sleep_wake_name { + u32 id; + const char *name; +}; + +/* The power words in "ps" that the sleep drives itself */ +struct apple_sleep_domains { + /* CPU, front end and system, of the DCP and then of the external DCP */ + u32 disp[SLEEP_DISPLAY_COUNT][3]; + u32 sio_cpu; + u32 sep; + /* The USB host's complex: always-on, common and the controller */ + u32 usb_aon; + u32 usb_common; + u32 usb; + /* Only the deep path's wake turns it on. */ + u32 ap_tmm; + u32 gpio; + u32 ans; + /* ANS's PCIe block */ + u32 apcie_gp; + u32 apcie_sys_gp; + u32 apcie_st; + u32 apcie_sys_st; + u32 apcie_phy_sw; +}; + +/* + * What differs between SoCs. Power words are offsets in "ps", or in "ps-aon" where named + * so, and the lists keep the order macOS was measured with. + */ +struct apple_sleep_soc { + struct apple_sleep_domains ps; + /* In "ps-aon": held up by a live debug link, and off on macOS */ + u32 ps_aon_debug_switch; + /* In "pwrgate": each pair written as macOS does */ + u32 pwrgate_wake[2]; + u32 pwrgate_cpm[2]; + u32 pwrgate_disp[2]; + /* Every power word, saved at the commit and checked at the wake */ + struct apple_sleep_words ps_all; + struct apple_sleep_words ps_aon_all; + /* On at macOS's commit and left off by Linux */ + struct apple_sleep_words ps_macos_on; + /* Off at macOS's commit, in the order it lowers them */ + struct apple_sleep_words ps_macos_off; + struct apple_sleep_words ps_aon_macos_off; + /* In "ps-aon": on at macOS's commit, with AUTO_ENABLE off */ + struct apple_sleep_words ps_aon_macos_no_auto; + /* Written back on by macOS's wake before any driver runs, in its order */ + struct apple_sleep_words ps_wake_order; + struct apple_sleep_words ps_aon_wake_order; + /* Polled off just before the complex words */ + struct apple_sleep_words ps_still_on; + /* Lowered at the last step if still on, with no user left */ + const struct apple_sleep_word *stray; + unsigned int nstray; + /* Off on macOS by the last step: one still on only warns. */ + const struct apple_sleep_word *gate_off; + unsigned int ngate_off; + const struct apple_sleep_gate_coproc *gate_coprocs; + unsigned int ngate_coprocs; + const struct apple_sleep_wake_name *wake_names; + unsigned int nwake_names; + /* The sources in the wake block's last word */ + u32 wake_last_word; + /* The PCIe host whose port the radio is on, for its PERST#, or NULL */ + const char *radio_pcie; +}; + +struct apple_sleep { + struct device *dev; + const struct apple_sleep_soc *soc; + void __iomem *ps; + void __iomem *pwrgate; + void __iomem *cpu_start; + void __iomem *cllt; + void __iomem *cluster; + void __iomem *commit; + void __iomem *wdt; + void __iomem *aon_misc; + void __iomem *wake; + void __iomem *amcc[4]; + void __iomem *pmc_dcp; + void __iomem *spmi; + void __iomem *pmc_voters; + void __iomem *dcs; + void __iomem *ps_aon; + void __iomem *spmi_a0; + void __iomem *hpm_spmi; + /* The port controller whose cold reset ends the debug link. */ + struct device_node *hpm_np; + u64 siq_cfg; + u32 hpm_sid; + int usb_down_ret; + /* Cut, or found already down, for this sleep. */ + bool usb_down_ran; + /* The PMGR block "ps" maps, for the power controllers' nodes. */ + struct resource ps_res; + /* Its words a driver took down with apple_sleep_domain_set() and hasn't raised. */ + unsigned long *driver_down; + void __iomem *clk_mux; + void __iomem *mcc[MCC_CHANNELS]; + void __iomem *ptd; + void __iomem *clk_perf; + void __iomem *cluster_dcs; + /* As the match found them: one for each word of the SoC's lists */ + u32 *ps_on_saved; + u32 *ps_off_saved; + u32 *ps_aon_saved; + u32 *ps_aon_auto_saved; + /* For each word of ps_all and ps_aon_all: at the commit, as the wake found it, late */ + u32 *ps_commit; + u32 *ps_aon_commit; + u32 *ps_woke; + u32 *ps_aon_woke; + bool *ps_late; + bool *ps_aon_late; + /* The rewake's own books */ + bool *ps_done; + bool *ps_aon_done; + u32 spmi_saved[2]; + bool matched; + void __iomem *dvfm[SLEEP_CLUSTERS]; + struct aura_s2r_mailbox *mbox; + + u32 wake_enable[WAKE_SOURCES_MAX]; + int wake_enable_count; + u32 wake_saved[WAKE_WORDS]; + /* The sources this sleep left enabled. */ + u32 wake_armed[WAKE_WORDS]; + + struct { + struct device_node *np; + void __iomem *asc; + bool bound; + } display[SLEEP_DISPLAY_COUNT]; + + /* Filled in for the report after a return. */ + u32 wake_status[WAKE_WORDS]; + u64 dvfm_ctl[SLEEP_CLUSTERS]; + u64 dvfm_saved[SLEEP_CLUSTERS][ARRAY_SIZE(sleep_dvfm_words)]; + bool dvfm_parked; + u64 resumes_before; + u64 resumes_after; + int cluster_e_ret; + int cluster_ave_ret; + int cluster_ave_commit_ret; + unsigned int ps_rewoken; + unsigned int ps_rewake_timeouts; + u64 cntkctl; + int cpus_stop_ret; + int ans_off_ret; + int pcie_ans_on_ret; + void __iomem *apcie_axi; + void __iomem *apcie_cio3pll; + void __iomem *apcie_clkgen; + void __iomem *apcie_phy; + void __iomem *apcie_rc; + bool apcie_port_stopped; + /* A real sleep left the radio's port down, until its host sets it up again */ + bool apcie_port_down; + void __iomem *apcie_port; + void __iomem *apcie_port_phy; + void __iomem *apcie_engine; + void __iomem *apcie_fabric; + struct apple_tunable *apcie_axi_tunable; + struct apple_tunable *apcie_cio3pll_tunable; + struct apple_tunable *apcie_clkgen_tunable; + struct apple_tunable *apcie_phy_tunable; + void __iomem *apcie_phy_ip; + struct apple_tunable *apcie_phy_ip_pll; + struct apple_tunable *apcie_phy_ip_lanes; + bool pcie_ans_stopped; + /* The words apple_sleep_lower_stray() lowered, one bit each */ + unsigned long stray_lowered; + /* What must be bound for a sleep, and what must have stopped by the last step */ + struct apple_sleep_gate { + const char *name; + /* Compared only, never dereferenced: nodes outlive the gate. */ + const struct device_node *np; + void __iomem *asc; + bool bound; + bool must_stop; + bool sep; + u32 ps; + } gate[16]; + unsigned int gate_count; + /* The AP pins, which a deep sleep resets */ + void __iomem *pins; + u32 *pins_saved; + unsigned long *pins_skip; + unsigned int npins; + unsigned int pin_irq_groups; + bool pins_saved_valid; + bool pins_lost; + /* The radio port's PERST# among them, or -1 */ + int radio_perst; + bool radio_perst_low; + bool returned; + /* This wake came back down the deep path, not the DRAM-kept one. */ + bool deep_ran; + /* This sleep's wake came from the SMC's sources alone. */ + bool smc_woke_alone; +}; + +static struct apple_sleep *apple_sleep; + +static struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev == dev) + return &apple_sleep_coprocs[i]; + + return NULL; +} + +/* A registered coprocessor its driver has readied, whose run bit the last step clears. */ +static bool apple_sleep_coproc_ready_node(const struct device_node *np) +{ + int i; + + for (i = 0; np && i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && + apple_sleep_coprocs[i].dev->of_node == np) + return true; + return false; +} + +static bool sleep_usb_asleep; + +void apple_sleep_usb_set_asleep(bool asleep) +{ + WRITE_ONCE(sleep_usb_asleep, asleep); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_asleep); + +static unsigned long sleep_usb_wake_ports; + +void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ + WRITE_ONCE(sleep_usb_wake_ports, ports); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); + +bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return port < BITS_PER_LONG && (READ_ONCE(sleep_usb_wake_ports) & BIT(port)); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); + +/* The USB host's vote, kept until the word is mapped. Only bit 0 is touched, as on macOS. */ +static DEFINE_SPINLOCK(sleep_slp_ddr_lock); +static bool sleep_slp_ddr_usb; +static void __iomem *sleep_slp_ddr_word; + +/* + * A coprocessor that runs through a sleep crashed. The AOP is the always-on master SLP_DDR keeps + * DRAM up for: after its firmware crashed, the next SLP_DDR wake stalled in the sleep engine, + * while the deep path doesn't need it. Nothing restarts it short of a reboot. + */ +static bool sleep_slp_ddr_crashed; + +static void sleep_slp_ddr_write(void) +{ + u32 val; + + lockdep_assert_held(&sleep_slp_ddr_lock); + if (!sleep_slp_ddr_word) + return; + val = readl(sleep_slp_ddr_word) & ~WAKE_SLP_DDR_USB; + if (sleep_slp_ddr_usb && !sleep_slp_ddr_crashed) + val |= WAKE_SLP_DDR_USB; + writel(val, sleep_slp_ddr_word); +} + +void apple_sleep_usb_slp_ddr(bool up) +{ + guard(spinlock_irqsave)(&sleep_slp_ddr_lock); + sleep_slp_ddr_usb = up; + sleep_slp_ddr_write(); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_slp_ddr); + +/* + * From RTKit, for every firmware crash. One this driver stops and restarts around each sleep + * doesn't run through it; for any other, every later sleep takes the deep path. It may come + * from a poll with interrupts off, so the table is read without its lock: entries only change + * at registration. + */ +void apple_sleep_coproc_crashed(struct device *dev) +{ + if (!apple_sleep || apple_sleep_coproc_find(dev)) + return; + scoped_guard(spinlock_irqsave, &sleep_slp_ddr_lock) { + if (sleep_slp_ddr_crashed) + return; + sleep_slp_ddr_crashed = true; + sleep_slp_ddr_write(); + } + dev_warn(apple_sleep->dev, "%s crashed: later sleeps take the deep path\n", dev_name(dev)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_crashed); + +static bool sleep_debug_link_cut; + +/* Only the debug port's controller took the cold reset. */ +static bool sleep_debug_port_is(struct device *dev) +{ + return apple_sleep && apple_sleep->hpm_np && dev->of_node == apple_sleep->hpm_np; +} + +bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return sleep_debug_port_is(dev) && xchg(&sleep_debug_link_cut, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_was_cut); + +bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return sleep_debug_port_is(dev) && READ_ONCE(sleep_debug_link_cut); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_peek); + +/* The radio's host set its port up again after a real sleep: the next commit may stop it. */ +void apple_sleep_radio_port_up(void) +{ + if (apple_sleep) + WRITE_ONCE(apple_sleep->apcie_port_down, false); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_port_up); + +#define PIN_REG(n) (4 * (n)) +#define PIN_DATA BIT(0) +#define PIN_LOCK BIT(21) +#define PIN_CTL 0xc48 +#define PIN_IRQ(g, w) (0x800 + 0x40 * (g) + 4 * (w)) + +/* Before any device suspends, with the pins' domain on. Skipped pins aren't read. */ +static void sleep_pins_save(struct apple_sleep *s) +{ + unsigned int pin; + + s->pins_lost = false; + s->pins_saved_valid = false; + if (!s->pins) + return; + for (pin = 0; pin < s->npins; pin++) + if (!test_bit(pin, s->pins_skip)) + s->pins_saved[pin] = readl(s->pins + PIN_REG(pin)); + s->pins_saved_valid = true; +} + +/* + * A deep sleep resets every pin, and the pin controller's driver puts back only the + * ones it configured. macOS puts all of them back right before ANS's run bit, and + * ANS's link doesn't come up without that: 1 written at 0xc48, every interrupt + * group's pending bits cleared, then each pin written back as it reads, then as it + * was, its lock bit last. + */ +void apple_sleep_restore_pins(void) +{ + struct apple_sleep *s = apple_sleep; + unsigned int pin, grp, w, changed = 0; + + if (!s || !s->pins_lost) + return; + s->pins_lost = false; + + writel(1, s->pins + PIN_CTL); + for (grp = 0; grp < s->pin_irq_groups; grp++) + for (w = 0; w < DIV_ROUND_UP(s->npins, 32); w++) + writel(U32_MAX, s->pins + PIN_IRQ(grp, w)); + for (pin = 0; pin < s->npins; pin++) { + void __iomem *reg = s->pins + PIN_REG(pin); + u32 want = s->pins_saved[pin], cur; + + if (test_bit(pin, s->pins_skip)) + continue; + cur = readl(reg); + changed += cur != want; + writel(cur, reg); + if (want & PIN_LOCK) + writel(want & ~PIN_LOCK, reg); + writel(want, reg); + } + dev_info(s->dev, "%u of %u AP pins changed by the sleep, all put back\n", changed, + s->npins - bitmap_weight(s->pins_skip, s->npins)); +} +EXPORT_SYMBOL_GPL(apple_sleep_restore_pins); + +/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data) +{ + struct apple_sleep_coproc *c; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_coproc_find(dev)) + return -EEXIST; + c = apple_sleep_coproc_find(NULL); + if (!c) + return -ENOSPC; + c->asc = asc; + c->pcie_ans = pcie_ans; + c->last_check = NULL; + c->last_stop = last_stop; + c->data = data; + c->ready = false; + c->stopped = false; + c->first = false; + c->ps = 0; + c->dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); + +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data) +{ + struct apple_sleep_coproc *c; + struct device_node *np; + u32 ps = 0; + int ret; + + if (!last_stop) + return -EINVAL; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np) { + of_property_read_u32(np, "reg", &ps); + of_node_put(np); + } + ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); + if (ret) + return ret; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) { + c->first = true; + c->ps = ps; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); + +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return -ENOENT; + c->last_check = last_check; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); + +int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data) +{ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + if (!stop) + return -EINVAL; + if (!apple_sleep_commit_stops_coprocs()) + return -EOPNOTSUPP; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev) + return -EBUSY; + apple_sleep_enclave.stop = stop; + apple_sleep_enclave.data = data; + apple_sleep_enclave.dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); + +void apple_sleep_enclave_unregister(struct device *dev) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev == dev) + memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); + +void apple_sleep_coproc_unregister(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) + memset(c, 0, sizeof(*c)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); + +void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return; + c->ready = ready; + c->stopped = false; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); + +bool apple_sleep_coproc_stopped(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + return c && c->stopped; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); + +bool apple_sleep_commit_stops_coprocs(void) +{ + return READ_ONCE(apple_sleep) && pm_suspend_via_firmware(); +} +EXPORT_SYMBOL_GPL(apple_sleep_commit_stops_coprocs); + +/* + * The last thing that can fail on the way to the commit: no CPU but this one is up and + * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only + * through a real sleep, so every refusal has to be behind this, and nothing after it may + * go back. In the order measured on macOS, on ANS itself: whatever its driver does last, + * status and control read, then 0 written twice. + */ +static void apple_sleep_coproc_run_off(struct apple_sleep_coproc *c) +{ + /* Its own last stop cleared its run bits. */ + if (!c->asc) { + c->stopped = true; + return; + } + c->status = readl(c->asc + ASC_CPU_STATUS); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + c->stopped = true; +} + +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +static bool apple_sleep_coproc_lowers(u32 ps) +{ + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && + apple_sleep_coprocs[i].first && apple_sleep_coprocs[i].ps == ps) + return true; + return false; +} + +/* The last refusals, each coprocessor's own: nothing has stopped yet. */ +static int apple_sleep_coprocs_check(struct apple_sleep *s) +{ + int i, ret; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->stopped || !c->last_check) + continue; + ret = c->last_check(c->data); + if (ret) { + dev_err(s->dev, "refused: %s: %d\n", dev_name(c->dev), ret); + return ret; + } + } + + return 0; +} + +/* + * The other cores stop at the commit, and one left awake there keeps the engine waiting + * until WD2 resets the SoC. Each answers here, before the display goes off, and waits for the + * stop or a refusal. + */ +static int apple_sleep_cpus_hold(struct apple_sleep *s) +{ + int ret = spin_table_hold_parked(); + + if (ret) + dev_err(s->dev, "refused: a CPU didn't answer the hold: %d\n", ret); + + return ret; +} + +/* A nonzero result must mean no SLEEP was sent; post-send errors are fatal. */ +static int apple_sleep_enclave_stop(struct apple_sleep *s) +{ + int ret; + + if (!apple_sleep_enclave.stop) + return 0; + ret = apple_sleep_enclave.stop(apple_sleep_enclave.data); + if (ret) + dev_err(s->dev, "refused before SEP SLEEP was sent: %d\n", ret); + + return ret; +} + +/* + * In two passes, after the last refusal: from the first stop only a real sleep restarts the + * GPU, so a failure here goes on to the commit. Those registered first stop whole first, + * before macOS's off words drop under them: the GPU withdraws its PMP request there, and the + * PMP has to see that before its own stop. The rest after the match. + */ +static void apple_sleep_coprocs_stop(struct apple_sleep *s, bool first) +{ + int i; + + if (first) + s->pcie_ans_stopped = false; + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != first) + continue; + s->pcie_ans_stopped |= c->pcie_ans; + if (c->stopped) + continue; + c->stop_ret = c->last_stop ? c->last_stop(c->data) : 0; + apple_sleep_coproc_run_off(c); + } + dsb(sy); +} + +/* MPIDR's cluster is affinity 2, its core affinity 0, as 0.0.n and 1.1.n on T8140. */ +static unsigned int sleep_cpu_cluster(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 2); +} + +static unsigned int sleep_cpu_core(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 0); +} + +static u32 sleep_cpu_start_bit(u64 mpidr) +{ + return BIT(4 * sleep_cpu_cluster(mpidr) + sleep_cpu_core(mpidr)); +} + +static void sleep_deep_wfi_arm(void) +{ + /* A timer left running ends the WFI, and costs power until reset. */ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_ENABLE); +} + +/* A core's last instructions: nothing brings it back but a reset. */ +static void __noreturn sleep_deep_wfi(void) +{ + sleep_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void sleep_cpu_stop(u64 mpidr) +{ + writel(sleep_cpu_start_bit(mpidr), apple_sleep->cpu_start + CPU_START_STOP); + dsb(sy); +} + +static void sleep_cpu_off(u64 mpidr) +{ + sleep_deep_wfi(); +} + +static int sleep_cpu_on(u64 mpidr) +{ + struct apple_sleep *s = apple_sleep; + unsigned int cluster = sleep_cpu_cluster(mpidr); + + writel(sleep_cpu_start_bit(mpidr), s->cpu_start + CPU_START_SYS); + writel(BIT(sleep_cpu_core(mpidr)), s->cpu_start + CPU_START_CLUSTER(cluster)); + + return 0; +} + +static const struct spin_table_power_ops sleep_cpu_power_ops = { + .cpu_stop = sleep_cpu_stop, + .cpu_off = sleep_cpu_off, + .cpu_on = sleep_cpu_on, +}; + +/* + * As macOS lowers a domain: AUTO_ENABLE off first if it's set, then target 0. + * Writing the flags clears them. Values measured, for instance ANS 0x1f0020ff, + * 0x0f0020ff, 0x0f0023f0. + */ +static u32 sleep_ps_lower_start(struct apple_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + + return val; +} + +static void sleep_ps_lower_finish(struct apple_sleep *s, u32 offset, u32 val) +{ + writel((val & ~PS_TARGET) | PS_FLAGS, s->ps + offset); +} + +static void sleep_ps_lower(struct apple_sleep *s, u32 offset) +{ + sleep_ps_lower_finish(s, offset, sleep_ps_lower_start(s, offset)); +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Measured + * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. + */ +static int sleep_ps_raise_start(struct apple_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + u32 now; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + + return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, + PS_OFF_TIMEOUT_US); +} + +static void sleep_ps_raise_finish(struct apple_sleep *s, u32 offset) +{ + void __iomem *reg = s->ps + offset; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, + reg); +} + +static int sleep_ps_raise(struct apple_sleep *s, u32 offset) +{ + int ret = sleep_ps_raise_start(s, offset); + + sleep_ps_raise_finish(s, offset); + + return ret; +} + +/* A domain the hardware idled under AUTO_ENABLE comes back once that's cleared. */ +static bool sleep_ps_wait_on(struct apple_sleep *s, u32 offset, u32 val) +{ + if (!FIELD_GET(PS_TARGET, val)) + return false; + + return !readl_poll_timeout_atomic(s->ps + offset, val, FIELD_GET(PS_ACTUAL, val), 1, + PS_OFF_TIMEOUT_US); +} + +static int sleep_ps_wait_off(struct apple_sleep *s, u32 offset) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, !FIELD_GET(PS_ACTUAL, val), + 10, PS_OFF_TIMEOUT_US); +} + +static bool sleep_ps_on(struct apple_sleep *s, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)); +} + +/* @pd's word in "ps": a power controller node, child of the PMGR that "ps" maps. */ +static int sleep_ps_offset(struct apple_sleep *s, struct device_node *pd) +{ + struct device_node *pmgr __free(device_node) = of_get_parent(pd); + struct resource res; + u32 off; + + if (!pmgr || of_address_to_resource(pmgr, 0, &res) || res.start != s->ps_res.start || + of_property_read_u32(pd, "reg", &off) || off % 4 || + off + 4 > resource_size(&s->ps_res)) + return -EINVAL; + + return off; +} + +int apple_sleep_domain_check(struct device_node *pd) +{ + int off; + + if (!apple_sleep) + return -ENODEV; + off = sleep_ps_offset(apple_sleep, pd); + return off < 0 ? off : 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_domain_check); + +/* + * A power domain lowered or raised by its own driver for a sleep, before the commit or after + * a refused one, as macOS's drivers do through its PMGR. A lower waits for off. Until the + * driver raises it, the PMGR's driver leaves it alone at the wake. + */ +int apple_sleep_domain_set(struct device_node *pd, bool on) +{ + struct apple_sleep *s = apple_sleep; + int off, ret = 0; + u32 val; + + if (!s) + return -ENODEV; + off = sleep_ps_offset(s, pd); + if (off < 0) + return off; + + val = readl(s->ps + off); + if (on) { + if (FIELD_GET(PS_TARGET, val) != 0xf || FIELD_GET(PS_ACTUAL, val) != 0xf) + ret = sleep_ps_raise(s, off); + else if (!(val & PS_AUTO_ENABLE)) + sleep_ps_raise_finish(s, off); + if (!ret) + clear_bit(off / 4, s->driver_down); + return ret; + } + set_bit(off / 4, s->driver_down); + if (FIELD_GET(PS_TARGET, val)) + sleep_ps_lower(s, off); + return sleep_ps_wait_off(s, off); +} +EXPORT_SYMBOL_GPL(apple_sleep_domain_set); + +bool apple_sleep_domain_is_down(struct device_node *pd) +{ + struct apple_sleep *s = apple_sleep; + int off; + + if (!s) + return false; + off = sleep_ps_offset(s, pd); + return off >= 0 && test_bit(off / 4, s->driver_down); +} +EXPORT_SYMBOL_GPL(apple_sleep_domain_is_down); + +/* + * After a refused sleep, the sleep engine's own restart of a firmware's CPU whose domain shares + * a power switch and can't gate alone, as SIO's: each of the device's own domains reset through + * target 4 and 0, released with the domain off, then raised again, its DARTs' state kept. The + * caller has cleared the run bit, and starts the firmware after. + */ +#define PS_BUSY BIT(11) +#define PS_RESET BIT(31) +#define PS_ENGINE_RESET_4 0x8f000244 +#define PS_ENGINE_RESET_0 0x80000000 +#define ASC_CPU_STATUS_READY BIT(1) +#define ENGINE_RESET_PS 4 +#define ENGINE_RESET_DARTS 2 + +static int sleep_engine_reset_wait(struct apple_sleep *s, u32 offset, u32 actual) +{ + u32 val; + + return readl_poll_timeout_atomic(s->ps + offset, val, + FIELD_GET(PS_ACTUAL, val) == actual && !(val & PS_BUSY), + 10, PS_OFF_TIMEOUT_US); +} + +/* A domain of the PMGR block mapped here as "ps", and its offset in it. */ +static bool sleep_ps_domain(struct apple_sleep *s, struct device_node *pd, u32 *offset) +{ + int idx = of_property_match_string(s->dev->of_node, "reg-names", "ps"); + struct device_node *parent = of_get_parent(pd); + struct resource ps, res; + bool ours; + + ours = idx >= 0 && parent && !of_address_to_resource(s->dev->of_node, idx, &ps) && + !of_address_to_resource(parent, 0, &res) && res.start == ps.start && + !of_property_read_u32(pd, "reg", offset); + of_node_put(parent); + + return ours; +} + +/* The @index-th DART in @np's iommus, a reference held, or NULL past the last. */ +static struct device *sleep_iommus_dart(struct device_node *np, int index) +{ + struct of_phandle_args args; + struct platform_device *pdev; + + if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, &args)) + return NULL; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + + return pdev ? &pdev->dev : NULL; +} + +int apple_sleep_engine_reset(struct device *dev) +{ + struct apple_sleep *s = apple_sleep; + struct device *darts[ENGINE_RESET_DARTS] = {}; + u32 ps[ENGINE_RESET_PS], saved[ENGINE_RESET_PS], val; + struct of_phandle_iterator it; + void __iomem *asc; + int i, n = 0, idx, err, ret = 0; + + if (!s) + return -ENODEV; + of_for_each_phandle(&it, err, dev->of_node, "power-domains", NULL, 0) { + if (!sleep_ps_domain(s, it.node, &val)) + continue; + if (n == ENGINE_RESET_PS) { + of_node_put(it.node); + return -E2BIG; + } + ps[n++] = val; + } + if (!n) + return -ENODEV; + + idx = of_property_match_string(dev->of_node, "reg-names", "coproc"); + if (idx < 0) + idx = of_property_match_string(dev->of_node, "reg-names", "asc"); + asc = idx >= 0 ? of_iomap(dev->of_node, idx) : NULL; + if (!asc) + return -ENODEV; + + for (i = 0; i < ENGINE_RESET_DARTS; i++) { + darts[i] = sleep_iommus_dart(dev->of_node, i); + if (!darts[i]) + break; + apple_dart_state_save(darts[i]); + } + + for (i = 0; i < n && !ret; i++) { + saved[i] = readl(s->ps + ps[i]); + writel(PS_ENGINE_RESET_4, s->ps + ps[i]); + ret = sleep_engine_reset_wait(s, ps[i], 4); + if (!ret) { + writel(PS_ENGINE_RESET_0, s->ps + ps[i]); + ret = sleep_engine_reset_wait(s, ps[i], 0); + } + } + if (!ret) { + usleep_range(1000, 2000); + for (i = 0; i < n; i++) + writel(0, s->ps + ps[i]); + for (i = n - 1; i >= 0 && !ret; i--) { + writel(PS_TARGET, s->ps + ps[i]); + ret = sleep_engine_reset_wait(s, ps[i], FIELD_MAX(PS_ACTUAL)); + if (!ret) + writel(saved[i] & ~(PS_FLAGS | PS_RESET | PS_ACTUAL), + s->ps + ps[i]); + } + } + if (!ret) + ret = readl_poll_timeout(asc + ASC_CPU_STATUS, val, val & ASC_CPU_STATUS_READY, 10, + 100 * USEC_PER_MSEC); + + for (i = 0; i < ENGINE_RESET_DARTS && darts[i]; i++) { + apple_dart_state_restore(darts[i]); + put_device(darts[i]); + } + iounmap(asc); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_sleep_engine_reset); + +/* The order macOS does it in, across clusters. */ +static void sleep_dvfm_park(struct apple_sleep *s) +{ + int cluster, word; + + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) { + s->dvfm_ctl[cluster] = readq(s->dvfm[cluster] + DVFM_CTL); + for (word = 0; word < ARRAY_SIZE(sleep_dvfm_words); word++) + s->dvfm_saved[cluster][word] = + readq(s->dvfm[cluster] + sleep_dvfm_words[word]); + } + + writeq(0, s->dvfm[0] + DVFM_CTL); + writeq(0, s->dvfm[1] + DVFM_CTL); + for (word = 0; word < 2; word++) + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + sleep_dvfm_words[word]); + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word] & ~DVFM_ENABLE, + s->dvfm[cluster] + sleep_dvfm_words[word]); + s->dvfm_parked = true; +} + +static void sleep_dvfm_unpark(struct apple_sleep *s) +{ + int cluster, word; + + if (!s->dvfm_parked) + return; + s->dvfm_parked = false; + + writeq(s->dvfm_ctl[0], s->dvfm[0] + DVFM_CTL); + writeq(s->dvfm_ctl[1], s->dvfm[1] + DVFM_CTL); + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + for (word = 2; word < 4; word++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + sleep_dvfm_words[word]); + for (word = 1; word >= 0; word--) + for (cluster = 0; cluster < SLEEP_CLUSTERS; cluster++) + writeq(s->dvfm_saved[cluster][word], + s->dvfm[cluster] + sleep_dvfm_words[word]); +} + +/* + * Every device has suspended by the last step, so a stray word still on has no user: it's + * lowered there, and a sleep with one that won't go off is refused. They come back on after + * the sleep or a refusal, for genpd's books. + */ +static void apple_sleep_stray_restore(struct apple_sleep *s) +{ + const struct apple_sleep_word *stray = s->soc->stray; + int i; + + for (i = s->soc->nstray - 1; i >= 0; i--) + if (s->stray_lowered & BIT(i)) + sleep_ps_raise(s, stray[i].offset); + s->stray_lowered = 0; +} + +static int apple_sleep_lower_stray(struct apple_sleep *s) +{ + const struct apple_sleep_word *stray = s->soc->stray; + int i, ret = 0; + + s->stray_lowered = 0; + for (i = 0; i < s->soc->nstray; i++) { + u32 off = stray[i].offset; + + if (!sleep_ps_on(s, off)) + continue; + /* Its driver's own last stop lowers it, at the commit. */ + if (apple_sleep_coproc_lowers(off)) { + dev_dbg(s->dev, "%s: left on for its driver's last stop\n", + stray[i].name); + continue; + } + sleep_ps_lower(s, off); + s->stray_lowered |= BIT(i); + if (sleep_ps_wait_off(s, off)) { + dev_err(s->dev, "refused: %s won't go off: %#x\n", + stray[i].name, readl(s->ps + off)); + ret = -EBUSY; + } else { + dev_info(s->dev, "%s was on at the last step, lowered\n", + stray[i].name); + } + } + if (ret) + apple_sleep_stray_restore(s); + + return ret; +} + +/* + * The SoC's coprocessors that must be bound for a sleep, the ones that must also have stopped + * by the last step, and the domains macOS has off by then. Taken in process context, since + * the checks run with interrupts off. A driver whose stop failed has already failed the + * suspend, naming itself, so what's left is whether it's bound at all, and what the hardware + * shows. + */ +static void apple_sleep_gate_prepare(struct apple_sleep *s) +{ + const struct apple_sleep_gate_coproc *coprocs = s->soc->gate_coprocs; + int i; + + s->gate_count = 0; + for (i = 0; i < s->soc->ngate_coprocs && s->gate_count < ARRAY_SIZE(s->gate); i++) { + struct device_node *c = of_find_node_by_path(coprocs[i].path); + struct platform_device *pdev = c ? of_find_device_by_node(c) : NULL; + struct apple_sleep_gate *g; + + if (!c) { + dev_warn(s->dev, "%s isn't in this tree\n", coprocs[i].path); + continue; + } + g = &s->gate[s->gate_count++]; + g->name = c->name; + g->np = c; + g->bound = pdev && device_is_bound(&pdev->dev); + g->sep = of_node_name_eq(c, "sep"); + g->must_stop = coprocs[i].must_stop; + g->ps = coprocs[i].ps; + if (g->must_stop && !g->asc) { + int idx = of_property_match_string(c, "reg-names", "coproc"); + + if (idx < 0) + idx = of_property_match_string(c, "reg-names", "asc"); + if (idx >= 0) + g->asc = of_iomap(c, idx); + } + if (pdev) + put_device(&pdev->dev); + of_node_put(c); + } +} + +/* + * Names everything wrong, so one boot shows the whole list. A coprocessor that must stop and + * still runs refuses the sleep; the rest only warns, as a real sleep is the only full test. + */ +static bool apple_sleep_gate_failed(struct apple_sleep *s) +{ + int i, bad = 0, running = 0; + + for (i = 0; i < s->gate_count; i++) { + struct apple_sleep_gate *g = &s->gate[i]; + + /* Even an always-on domain may read actual0: require a prepared protocol. */ + if (g->sep && (!g->bound || !apple_sleep_enclave.stop)) { + dev_err(s->dev, "refused: SEP has no prepared irreversible sleep request\n"); + bad++; + running++; + continue; + } + if (!g->bound) { + dev_err(s->dev, "%s isn't bound\n", g->name); + bad++; + continue; + } + /* A CPU whose domain is off isn't running, and its registers aren't there. */ + if (g->ps && !sleep_ps_on(s, g->ps)) + continue; + if (g->must_stop && (!g->asc || + ((readl(g->asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN) && + !apple_sleep_coproc_ready_node(g->np)))) { + dev_err(s->dev, "%s didn't stop: control %#x status %#x\n", g->name, + g->asc ? readl(g->asc + ASC_CPU_CONTROL) : 0, + g->asc ? readl(g->asc + ASC_CPU_STATUS) : 0); + bad++; + running++; + } + } + + for (i = 0; i < s->soc->ngate_off; i++) { + const struct apple_sleep_word *w = &s->soc->gate_off[i]; + + if (sleep_ps_on(s, w->offset)) { + dev_err(s->dev, "%s is on: %#x\n", w->name, readl(s->ps + w->offset)); + bad++; + } + } + + if (running) { + dev_err(s->dev, "refused: %d coprocessors that must stop or gate are still running\n", + running); + return true; + } + if (bad) + dev_warn(s->dev, "going ahead with %d problems\n", bad); + else + dev_info(s->dev, "%u coprocessors up and stopped\n", s->gate_count); + + return false; +} + +static void sleep_wake_unmask(u32 *val, u32 id) +{ + if (id / 32 < WAKE_WORDS) + val[id / 32] &= ~BIT(id % 32); +} + +/* + * Mask every source but the ones asked for. These words read all zero until + * something writes them, which leaves every source enabled, listed in the ADT or + * not. macOS reads all ones outside the ADT's ap-wake-sources (measured). The AOP's + * source 0 is masked with the rest: that held under macOS with its handler hidden, + * as did leaving it enabled with the AMCC enables off. The two together are unmeasured. + */ +static void sleep_wake_masks(struct apple_sleep *s, u32 *val) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + val[i] = i == WAKE_WORDS - 1 ? s->soc->wake_last_word : U32_MAX; + for (i = 0; i < s->wake_enable_count; i++) + sleep_wake_unmask(val, s->wake_enable[i]); +} + +/* The last checks, then the stray words lowered: what has stopped may not return until a wake. */ +static int apple_sleep_refuse(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + bool refused = false, idled = false; + int i; + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev) + continue; + if (c->ready) { + idled |= c->pcie_ans; + } else { + /* Nothing may lower a coprocessor's domain under its driver. */ + dev_err(s->dev, "refused: %s is running and its driver has not quiesced it\n", + dev_name(c->dev)); + refused = true; + } + } + + if (num_online_cpus() != 1 || + (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { + dev_err(s->dev, "refused: wrong CPU, or other CPUs online\n"); + refused = true; + } + + /* Only a driver takes the display to where its domain can be lowered. */ + if (!s->display[0].bound) { + dev_err(s->dev, "refused: no driver bound to the display coprocessor\n"); + refused = true; + } + + /* ANS's link needs these after a real power loss. */ + if (idled && !s->apcie_phy_tunable) { + dev_err(s->dev, "refused: no tunables to bring ANS's PCIe block back\n"); + refused = true; + } + + /* Only ANS's driver can stop it, and it doesn't come back without a real sleep. */ + if (!idled && (sleep_ps_on(s, d->ans) || sleep_ps_on(s, d->apcie_sys_gp))) { + dev_err(s->dev, "refused: ANS or PCIe is on with no driver to stop it\n"); + refused = true; + } + + /* The engine waits for DISP_CPU with no timeout, so this must never slip. */ + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + u32 ps = d->disp[i][0]; + + if (s->display[i].bound && sleep_ps_on(s, ps) && + (readl(s->display[i].asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + dev_err(s->dev, "refused: a bound display coprocessor is still running\n"); + refused = true; + } + } + + if (apple_sleep_gate_failed(s)) + refused = true; + + if (refused) + return -EBUSY; + + return apple_sleep_lower_stray(s); +} + +static void sleep_wake_arm(struct apple_sleep *s) +{ + u32 val[WAKE_WORDS]; + int i; + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_saved[i] = readl(s->wake + WAKE_MASK(i)); + sleep_wake_masks(s, val); + for (i = 0; i < WAKE_WORDS; i++) { + writel(val[i], s->wake + WAKE_MASK(i)); + s->wake_armed[i] = ~val[i]; + } +} + +static void sleep_wake_restore(struct apple_sleep *s) +{ + int i; + + for (i = 0; i < WAKE_WORDS; i++) + writel(s->wake_saved[i], s->wake + WAKE_MASK(i)); +} + +/* + * The sleep engine waits on DISP_CPU with no timeout, so the display domains + * must be off. A coprocessor's driver stops its firmware first, and nothing here + * talks to the firmware. Otherwise no driver is bound, and the bootloader left the + * firmware running: on the DCP, stop its CPU cold. That's unproven. The coprocessor + * restarts from its reset vector after any sleep, and the engine itself clears the + * DCP's run bit on the way down. + * + * Domain order and values as macOS's: DCPEXT then DCP, CPU, front end, system. + * The rest of the display's domains go off with these, by nothing Linux or macOS + * writes. The coprocessors' secure mailboxes are not used here. Whether that + * holds is open. + */ +static int sleep_display_off(struct apple_sleep *s) +{ + const u32 (*domains)[3] = s->soc->ps.disp; + int i, ret = 0; + + for (i = SLEEP_DISPLAY_COUNT - 1; i >= 0; i--) { + void __iomem *asc = s->display[i].asc; + + /* The engine doesn't wait for DISPEXT0. */ + if (i == 1 && !s->display[i].bound) + continue; + + if (sleep_ps_on(s, domains[i][0]) && asc && + (readl(asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + writel(0, asc + ASC_CPU_CONTROL); + writel(0, asc + ASC_CPU_CONTROL); + } + + /* Reads as a count of the DCPs powered, and macOS ends at 0. */ + writel(i, s->pmc_dcp); + if (i == 0) { + if (readl(s->ps + domains[0][0]) & PS_AUTO_ENABLE) + writel(readl(s->ps + domains[0][0]) & + ~(PS_AUTO_ENABLE | PS_FLAGS), s->ps + domains[0][0]); + writel(3, s->pwrgate + s->soc->pwrgate_disp[0]); + writel(0, s->pwrgate + s->soc->pwrgate_disp[1]); + } + sleep_ps_lower(s, domains[i][0]); + sleep_ps_lower(s, domains[i][1]); + sleep_ps_lower(s, domains[i][2]); + } + + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + int err; + + if (i == 1 && !s->display[i].bound) + continue; + + err = sleep_ps_wait_off(s, domains[i][0]); + + if (err) { + dev_err(s->dev, "refused: display domain %#x stayed on: %#x\n", + domains[i][0], readl(s->ps + domains[i][0])); + ret = err; + } + } + + return ret; +} + +/* One command on the port controller's bus, polled: no other CPU, and interrupts are off. */ +static int sleep_hpm_cmd(struct apple_sleep *s, u32 op, u32 extra, const u32 *in, u32 *out) +{ + void __iomem *b = s->hpm_spmi; + u32 status, reply; + int ret; + + while (!(readl(b + SPMI_STATUS) & SPMI_STATUS_RX_EMPTY)) + readl(b + SPMI_REPLY); + + writel(extra << 16 | SPMI_CMD_ACTIVE | s->hpm_sid << 8 | op, b + SPMI_CMD); + if (in) + writel(*in, b + SPMI_CMD); + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, SPMI_POLL_US); + if (ret) + return ret; + reply = readl(b + SPMI_REPLY); + writel(1, b + SPMI_NOTIFY_ACK); + if (out) { + ret = readl_poll_timeout_atomic(b + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 10, + SPMI_POLL_US); + if (ret) + return ret; + *out = readl(b + SPMI_REPLY); + } + + /* A zero write's reply carries the value written. */ + return op == SPMI_OP_ZERO_WRITE || (reply & 0xff) == op ? 0 : -EIO; +} + +static int sleep_hpm_select(struct apple_sleep *s, u32 page) +{ + u32 val = ~0U; + int i, ret; + + ret = sleep_hpm_cmd(s, SPMI_OP_ZERO_WRITE, page << 8, NULL, NULL); + for (i = 0; !ret && i < 2000; i++) { + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(1), HPM_REG_SELECT, NULL, &val); + if (!ret && (val & 0xff) == page) + return 0; + /* A select it dropped, waking, is sent again. page | 0x80 is one in progress. */ + if (!ret && (val & 0xff) != (page | 0x80) && i % 20 == 19) { + ret = sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_ZERO_WRITE, page << 8, NULL, NULL); + } + udelay(10); + } + + return ret ?: -ETIMEDOUT; +} + +/* + * What macOS sends the other port controller about a second before a sleep, measured + * in three runs: system power state 3, then the bus's sleep command. + */ +static int sleep_hpm_sleep(struct apple_sleep *s) +{ + u32 data = HPM_SSPS_S3, cmd = HPM_CMD_SSPS, val = ~0U; + int i, ret; + + ret = sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_DATA1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_WRITE(1), HPM_REG_DATA_CMD, &data, NULL); + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA_CMD, &cmd, NULL); + /* Selected afresh for each read, as macOS does. */ + for (i = 0; !ret && val && i < 100; i++) { + udelay(1000); + ret = sleep_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); + } + if (!ret && val) + ret = -ETIMEDOUT; + /* Asleep either way: a failure here only means it wasn't told first. */ + sleep_hpm_cmd(s, SPMI_OP_SLEEP, 0, NULL, NULL); + + return ret; +} + +/* + * The partner takes the reset as an unplug, attaches again and renegotiates its contract. + * Left to finish across the commit, that sometimes failed: neither side had a contract or + * saw the other, and no VDM got through until a replug. Returns the ms it took. + */ +static int sleep_hpm_wait_contract(struct apple_sleep *s) +{ + bool wake = true; + u32 val = 0; + int ms, ret; + + for (ms = 0; ms < HPM_CONTRACT_MS; ms += 10) { + ret = wake ? sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL) : 0; + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_POWER_STATUS); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); + if (!ret && (val & HPM_POWER_CONNECTED) && + FIELD_GET(HPM_POWER_TYPEC_MODE, val) == HPM_POWER_TYPEC_PD) + return ms; + wake = ret; + mdelay(10); + } + + return -ETIMEDOUT; +} + +/* A live debug link holds the debug switch domain on. */ +static bool sleep_debug_link_up(struct apple_sleep *s) +{ + return s->hpm_spmi && + FIELD_GET(PS_ACTUAL, readl(s->ps_aon + s->soc->ps_aon_debug_switch)); +} + +/* + * The debug link stays up with the ATC0 domains off, and it holds the debug domains + * active, which macOS has idle. The only known way to end it is the port controller's + * cold reset. To the SMC that's a cable pulled: it records the port's events as wake + * reasons once it has been warned of the sleep, and wakes the AP right after the commit. + * So the SMC's driver warns it only after this. tipd rearms the link after a refusal. + */ +static bool apple_sleep_usb_down(struct apple_sleep *s) +{ + u32 cmd = HPM_CMD_COLD_RESET; + u32 enable; + int i, ret, ms; + + if (s->usb_down_ran) + return false; + s->usb_down_ran = true; + + if (!sleep_debug_link_up(s)) + return false; + + dev_info(s->dev, "debug link: cutting it for the sleep\n"); + enable = readl(s->hpm_spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + /* A reset that the partner doesn't come back from is done again. */ + for (i = 1; i <= HPM_RESET_TRIES; i++) { + ret = sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL); + if (!ret) + ret = sleep_hpm_select(s, HPM_PAGE_CMD1); + if (!ret) + ret = sleep_hpm_cmd(s, SPMI_OP_EXT_WRITE(4), HPM_REG_DATA, &cmd, NULL); + if (ret) + break; + WRITE_ONCE(sleep_debug_link_cut, true); + mdelay(HPM_RESET_MS); + ms = sleep_hpm_wait_contract(s); + if (ms >= 0) { + dev_info(s->dev, "debug link: contract back %d ms after reset %d\n", + HPM_RESET_MS + ms, i); + break; + } + dev_warn(s->dev, "debug link: no contract with the partner after reset %d\n", i); + } + if (!ret) + ret = sleep_hpm_sleep(s); + writel(1, s->hpm_spmi + SPMI_NOTIFY_ACK); + writel(enable, s->hpm_spmi + SPMI_NOTIFY_ENABLE); + s->usb_down_ret = ret; + + return true; +} + +bool apple_sleep_debug_link_cut(void) +{ + return apple_sleep ? apple_sleep_usb_down(apple_sleep) : false; +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); + +bool apple_sleep_debug_link_up(void) +{ + struct apple_sleep *s = apple_sleep; + + return s && !s->usb_down_ran && sleep_debug_link_up(s); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_up); + +/* + * macOS takes the wake for a lid closing during the sleep as a dark one: every driver wakes, + * the display stays off, and the root domain sleeps again as soon as it sees the lid closed. + */ +void apple_sleep_lid_closed_wake(void) +{ + struct apple_sleep *s = apple_sleep; + + if (!s || !s->smc_woke_alone) + return; + dev_info(s->dev, "the lid closed during the sleep: dark wake\n"); + pm_set_dark_wake(); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_wake); + +/* + * The SMC woke the AP for the port alone, and this sleep cut the debug link: the cut's own + * events. Sleep again. The link stays down until tipd rearms it at a full wake, so the next + * entry doesn't cut it. + */ +void apple_sleep_debug_link_cut_wake(void) +{ + struct apple_sleep *s = apple_sleep; + + if (!s || !s->smc_woke_alone || !READ_ONCE(sleep_debug_link_cut)) + return; + dev_info(s->dev, "debug link: its cut woke the sleep: dark wake\n"); + pm_set_dark_wake(); +} +EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut_wake); + +/* + * The radio's link and the engine that starts it, in macOS's order, measured on one + * sleep: the link into L2, the engine stopped, PERST# asserted, the PHY's clocks off. + * macOS does it 40 ms before ANS's idle. Here it follows, so that a refused sleep finds + * the link untouched. A link that's down or a radio that's gone only makes the poll time + * out, and the power goes next either way. + */ +static void sleep_apcie_port_stop(struct apple_sleep *s) +{ + void __iomem *port = s->apcie_port; + void __iomem *phy = s->apcie_port_phy; + u32 val; + + writel(readl(port + APCIE_PORT_INTMSK) | APCIE_PORT_INT_LINK, port + APCIE_PORT_INTMSK); + writel(APCIE_PORT_INT_LINK, port + APCIE_PORT_INTSTAT); + + writel(readl(port + APCIE_PORT_APPCLK) | APCIE_PORT_APPCLK_CGDIS, port + APCIE_PORT_APPCLK); + /* A loader can leave the port ready with the link never trained. */ + if (readl(port + APCIE_PORT_LINKSTS) & APCIE_PORT_LINKSTS_UP) { + writel(APCIE_PORT_L2_ENTER, port + APCIE_PORT_L2); + readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), + 10, APCIE_PORT_STOP_US); + } + /* The radio keeps its power and firmware, in reset until its host releases it. */ + if (s->radio_perst >= 0 && sleep_ps_on(s, s->soc->ps.gpio)) { + void __iomem *reg = s->pins + PIN_REG(s->radio_perst); + + val = readl(reg); + writel(s->radio_perst_low ? val & ~PIN_DATA : val | PIN_DATA, reg); + } + writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); + + if (s->apcie_engine && s->apcie_fabric) { + writel(0, s->apcie_engine + APCIE_ENGINE_IRQ_MASK); + writel(3, s->apcie_fabric + APCIE_FABRIC_STOP); + writel(readl(s->apcie_engine + APCIE_ENGINE_CTRL) | APCIE_ENGINE_STOP, + s->apcie_engine + APCIE_ENGINE_CTRL); + } + udelay(1000); + + /* What this bit does isn't known. macOS sets it around PERST#. */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(0, port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), port + APCIE_PORT_PERST_GUARD); + + writel(readl(phy) & ~APCIE_PORT_PHY_EN1, phy); + writel(readl(phy) & ~APCIE_PORT_PHY_EN0, phy); + writel(readl(phy) | APCIE_PHY_CTRL_RESET, phy); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK0REQ, phy); + readl_poll_timeout_atomic(phy, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK1REQ, phy); + + writel(readl(port + APCIE_PORT_APPCLK) & ~APCIE_PORT_APPCLK_EN, port + APCIE_PORT_APPCLK); + writel(readl(port + APCIE_PORT_INTSTAT), port + APCIE_PORT_INTSTAT); +} + +static void sleep_apcie_port_stop_once(struct apple_sleep *s) +{ + /* Only with the port set up, by the loader or by its host after a wake, or it aborts. */ + if (s->apcie_port_stopped || s->apcie_port_down || !s->apcie_port || !s->apcie_port_phy || + !sleep_ps_on(s, s->soc->ps.apcie_gp) || !sleep_ps_on(s, s->soc->ps.apcie_phy_sw)) + return; + s->apcie_port_stopped = true; + sleep_apcie_port_stop(s); +} + +/* A poll here only paces the steps: the power goes next whatever it reads. */ +static void sleep_apcie_phy_off(struct apple_sleep *s) +{ + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + + writel(readl(ctrl) | APCIE_PHY_CTRL_RESET, ctrl); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK1REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK1ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK0REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_UP, ctrl); + writel(readl(run) & ~APCIE_PHY_RUN_EN2, run); + writel(readl(run) & ~APCIE_PHY_RUN_EN, run); +} + +static void sleep_apcie_clk_off(struct apple_sleep *s) +{ + void __iomem *clk = s->apcie_clkgen + APCIE_CLK_CTRL; + u32 val; + + writel(readl(clk) & ~APCIE_CLK_RUN, clk); + writel(readl(clk) | APCIE_CLK_OFF, clk); + readl_poll_timeout_atomic(clk, val, !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + writel(readl(clk) | APCIE_CLK_OFF2, clk); +} + +/* + * ANS's own link runs on this block, and macOS takes it down with the domains. Its + * order, measured on one sleep, in the same microseconds as the power words. A block's + * registers fault with its domain off, and a loader may have left any of these off. + */ +static void apple_sleep_lower_apcie(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + u32 val; + + sleep_apcie_port_stop_once(s); + /* The PHY's registers need both domains on. */ + if (s->apcie_phy && sleep_ps_on(s, d->apcie_phy_sw) && sleep_ps_on(s, d->apcie_sys_st)) + sleep_apcie_phy_off(s); + sleep_ps_lower(s, d->apcie_phy_sw); + + if (s->apcie_clkgen && sleep_ps_on(s, d->apcie_sys_st)) + sleep_apcie_clk_off(s); + sleep_ps_lower(s, d->apcie_sys_st); + sleep_ps_lower(s, d->apcie_st); + + /* + * The AXI enables are the block's link into the fabric. macOS reads the domain on + * before clearing them: powered off with them set, SoC NI 1 latches a PCIe receive + * link error once the block's last domain goes off. + */ + val = sleep_ps_lower_start(s, d->apcie_sys_gp); + if (s->apcie_axi && sleep_ps_wait_on(s, d->apcie_sys_gp, val)) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) & ~BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + sleep_ps_lower_finish(s, d->apcie_sys_gp, val); + + sleep_ps_lower(s, d->apcie_gp); + sleep_ps_lower(s, d->ans); +} + +/* Read-modify-write in table order, with no write where nothing changes. */ +static void sleep_tunable_apply_changed(void __iomem *regs, const struct apple_tunable *t) +{ + size_t i; + + for (i = 0; i < t->sz; i++) { + void __iomem *reg = regs + t->values[i].offset; + u32 old = readl(reg); + u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); + + if (val != old) + writel(val, reg); + } +} + +/* + * The power loss takes the PHY's PLL and lane settings with it, and ANS's link can't + * train without them. Applied as macOS does: the PLL table, then the lanes, then the + * PLL's hold released. That takes about the settling time the PHY needs here. + */ +static void sleep_apcie_phy_ip_apply(struct apple_sleep *s) +{ + void __iomem *ctl = s->apcie_phy_ip + APCIE_PHY_IP_PLL_CTL; + + sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); + sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); + writel(readl(ctl) & ~APCIE_PHY_IP_PLL_HOLD, ctl); +} + +/* + * Brought up from power-off in macOS's order, measured on one sleep: all of it follows + * the domains on the waking core, well before ANS runs. Whether ANS needs each step is + * inferred. Nothing else does this if no host driver is bound. + */ +static int apple_sleep_raise_apcie(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + void __iomem *ctrl = s->apcie_phy + APCIE_PHY_CTRL; + void __iomem *run = s->apcie_phy + APCIE_PHY_RUN; + u32 val; + int ret; + + ret = sleep_ps_raise(s, d->ans); + ret |= sleep_ps_raise(s, d->apcie_gp); + ret |= sleep_ps_raise_start(s, d->apcie_sys_gp); + if (!ret) { + writel(readl(s->apcie_axi + APCIE_AXI_EN0) | BIT(0), s->apcie_axi + APCIE_AXI_EN0); + writel(readl(s->apcie_axi + APCIE_AXI_EN1) | BIT(0), s->apcie_axi + APCIE_AXI_EN1); + } + sleep_ps_raise_finish(s, d->apcie_sys_gp); + ret |= sleep_ps_raise(s, d->apcie_st); + ret |= sleep_ps_raise(s, d->apcie_sys_st); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_axi, s->apcie_axi_tunable); + apple_tunable_apply(s->apcie_cio3pll, s->apcie_cio3pll_tunable); + apple_tunable_apply(s->apcie_clkgen, s->apcie_clkgen_tunable); + writel(readl(s->apcie_clkgen + APCIE_CLK_START) | APCIE_CLK_START_GO, + s->apcie_clkgen + APCIE_CLK_START); + /* Both off bits: the lowering sets them, and only a power-off clears OFF. */ + writel(readl(s->apcie_clkgen + APCIE_CLK_CTRL) & ~(APCIE_CLK_OFF2 | APCIE_CLK_OFF), + s->apcie_clkgen + APCIE_CLK_CTRL); + ret = readl_poll_timeout_atomic(s->apcie_clkgen + APCIE_CLK_CTRL, val, + !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + if (ret) + return ret; + + ret = sleep_ps_raise(s, d->apcie_phy_sw); + if (ret) + return ret; + + apple_tunable_apply(s->apcie_phy, s->apcie_phy_tunable); + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK0REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK1REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) & ~APCIE_PHY_CTRL_RESET, ctrl); + + writel(readl(run) | APCIE_PHY_RUN_EN, run); + sleep_apcie_phy_ip_apply(s); + writel(readl(run) | APCIE_PHY_RUN_EN2, run); + ret = readl_poll_timeout_atomic(s->apcie_phy + APCIE_PHY_READY, val, val & BIT(0), 1, + APCIE_POLL_US); + if (ret) + return ret; + writel(readl(s->apcie_phy + APCIE_PHY_COMMON) | BIT(0), s->apcie_phy + APCIE_PHY_COMMON); + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + + writel(0, s->apcie_rc + APCIE_RC_ZERO); + writel(0x140, s->apcie_rc + APCIE_RC_READY_CFG); + writel(1, s->apcie_rc + APCIE_RC_READY_START); + + return readl_poll_timeout_atomic(s->apcie_rc + APCIE_RC_READY_STAT, val, val & BIT(0), 1, + APCIE_POLL_US); +} + +/* About 4 ms after the other cores stop on macOS: deepest first, ANS last. */ +static void apple_sleep_lower_io(struct apple_sleep *s) +{ + const struct apple_sleep_domains *d = &s->soc->ps; + + /* + * USB's complex, deepest first. macOS's host lowers the controller's and the common + * domain in its own suspend, so here they're only lowered for a host that didn't. A host + * that sleeps in place keeps the always-on one, as macOS's does. + */ + if (FIELD_GET(PS_TARGET, readl(s->ps + d->usb))) + sleep_ps_lower(s, d->usb); + if (FIELD_GET(PS_TARGET, readl(s->ps + d->usb_common))) + sleep_ps_lower(s, d->usb_common); + if (!READ_ONCE(sleep_usb_asleep)) + sleep_ps_lower(s, d->usb_aon); + if (s->pcie_ans_stopped) { + apple_sleep_lower_apcie(s); + s->ans_off_ret = sleep_ps_wait_off(s, d->ans); + } +} + +static void apple_sleep_set_wd2(struct apple_sleep *s, unsigned int secs) +{ + writel(0, s->wdt + WDT_WD2_CTL); + writel(0, s->wdt + WDT_WD2_COUNT); + writel(secs * WDT_TICKS_PER_S, s->wdt + WDT_WD2_ALARM); + writel(1, s->wdt + WDT_WD2_CTL); +} + +#define PS_PARENT_MISSING BIT(11) +#define PS_AUTO_STATE GENMASK(27, 24) +#define PS_REWAKE_PASSES 2 + +static void sleep_ps_save(struct apple_sleep *s) +{ + const struct apple_sleep_words *all = &s->soc->ps_all, *aon = &s->soc->ps_aon_all; + int i; + + for (i = 0; i < all->count; i++) + s->ps_commit[i] = readl(s->ps + all->offset[i]); + for (i = 0; i < aon->count; i++) + s->ps_aon_commit[i] = readl(s->ps_aon + aon->offset[i]); +} + +static int sleep_ps_find(const u32 *all, int n, u32 offset) +{ + int i; + + for (i = 0; i < n; i++) + if (all[i] == offset) + return i; + return -1; +} + +/* + * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Never + * DEV_DISABLE, which keeps it off: the SEP's is set at the commit, and macOS's wake writes + * that word without it. The hardware owns the auto state field, so it stays as read. + */ +static int sleep_ps_restore(void __iomem *reg, u32 saved) +{ + u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_PARENT_MISSING | PS_DEV_DISABLE | + PS_FLAGS | PS_ACTUAL)) | (readl(reg) & PS_AUTO_STATE); + u32 now; + int ret; + + /* + * Off under AUTO_ENABLE at the commit, as the SEP's is: the hardware powers it when + * it's wanted, and forcing it on doesn't. macOS writes that word once, AUTO set. + */ + if ((saved & PS_AUTO_ENABLE) && !FIELD_GET(PS_ACTUAL, saved)) { + writel(keep | PS_AUTO_ENABLE, reg); + return 0; + } + + writel(keep, reg); + ret = readl_poll_timeout_atomic(reg, now, + FIELD_GET(PS_ACTUAL, now) == FIELD_GET(PS_TARGET, saved), 1, + PS_OFF_TIMEOUT_US); + if (saved & PS_AUTO_ENABLE) + writel(keep | PS_AUTO_ENABLE, reg); + + return ret; +} + +/* Whether the domain at @offset was on as the wake found it. */ +static bool sleep_ps_woke_on(struct apple_sleep *s, u32 offset) +{ + const struct apple_sleep_words *all = &s->soc->ps_all; + int i = sleep_ps_find(all->offset, all->count, offset); + + return i >= 0 && FIELD_GET(PS_ACTUAL, s->ps_woke[i]); +} + +/* Linux had it on and the wake left it otherwise. With AUTO_ENABLE, off is allowed. */ +static bool sleep_ps_dropped(u32 saved, u32 woke) +{ + u32 control = PS_AUTO_ENABLE | PS_DEV_DISABLE | PS_TARGET; + + if (!FIELD_GET(PS_TARGET, saved)) + return false; + if ((saved ^ woke) & control) + return true; + return !(saved & PS_AUTO_ENABLE) && + FIELD_GET(PS_ACTUAL, woke) != FIELD_GET(PS_TARGET, saved); +} + +/* + * Every domain back as it was at the commit, before any driver's resume reads its block: + * macOS's wake words first, in its order, whatever they read, if Linux had them on. Then + * any other domain the wake left otherwise, a second time for one whose parent came later. + */ +static void sleep_ps_rewake(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + const u32 *all = soc->ps_all.offset, *aon = soc->ps_aon_all.offset; + unsigned int n = soc->ps_all.count, naon = soc->ps_aon_all.count; + bool *redo = s->ps_late, *redo_aon = s->ps_aon_late; + bool *done = s->ps_done, *done_aon = s->ps_aon_done; + int i, j, pass; + + memset(done, 0, n * sizeof(*done)); + memset(done_aon, 0, naon * sizeof(*done_aon)); + memset(s->ps_late, 0, n * sizeof(*s->ps_late)); + memset(s->ps_aon_late, 0, naon * sizeof(*s->ps_aon_late)); + s->ps_rewoken = 0; + s->ps_rewake_timeouts = 0; + for (i = 0; i < n; i++) + s->ps_woke[i] = readl(s->ps + all[i]); + for (i = 0; i < naon; i++) + s->ps_aon_woke[i] = readl(s->ps_aon + aon[i]); + + for (j = 0; j < soc->ps_wake_order.count; j++) { + i = sleep_ps_find(all, n, soc->ps_wake_order.offset[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, s->ps_commit[i])) + continue; + done[i] = true; + s->ps_rewoken++; + if (sleep_ps_restore(s->ps + all[i], s->ps_commit[i])) + redo[i] = true; + } + for (j = 0; j < soc->ps_aon_wake_order.count; j++) { + i = sleep_ps_find(aon, naon, soc->ps_aon_wake_order.offset[j]); + if (i < 0 || !FIELD_GET(PS_TARGET, s->ps_aon_commit[i])) + continue; + done_aon[i] = true; + s->ps_rewoken++; + if (sleep_ps_restore(s->ps_aon + aon[i], s->ps_aon_commit[i])) + redo_aon[i] = true; + } + + for (pass = 0; pass < PS_REWAKE_PASSES; pass++) { + for (i = 0; i < n; i++) { + if (!redo[i] && + (done[i] || !sleep_ps_dropped(s->ps_commit[i], s->ps_woke[i]))) + continue; + if (!done[i]) { + done[i] = true; + s->ps_rewoken++; + } + redo[i] = sleep_ps_restore(s->ps + all[i], s->ps_commit[i]); + } + for (i = 0; i < naon; i++) { + if (!redo_aon[i] && + (done_aon[i] || + !sleep_ps_dropped(s->ps_aon_commit[i], s->ps_aon_woke[i]))) + continue; + if (!done_aon[i]) { + done_aon[i] = true; + s->ps_rewoken++; + } + redo_aon[i] = sleep_ps_restore(s->ps_aon + aon[i], s->ps_aon_commit[i]); + } + } + + for (i = 0; i < n; i++) + s->ps_rewake_timeouts += redo[i]; + for (i = 0; i < naon; i++) + s->ps_rewake_timeouts += redo_aon[i]; +} + +static void sleep_ps_rewake_report(struct apple_sleep *s) +{ + const struct apple_sleep_words *all = &s->soc->ps_all, *aon = &s->soc->ps_aon_all; + int i; + + dev_dbg(s->dev, "power words: %u written back, %u not on in time\n", s->ps_rewoken, + s->ps_rewake_timeouts); + for (i = 0; i < all->count; i++) + if (s->ps_late[i]) + dev_err(s->dev, "power word %#x not on in time: %#x at the commit, %#x now\n", + all->offset[i], s->ps_commit[i], + readl(s->ps + all->offset[i])); + for (i = 0; i < aon->count; i++) + if (s->ps_aon_late[i]) + dev_err(s->dev, "power word aon %#x not on in time: %#x at the commit, %#x now\n", + aon->offset[i], s->ps_aon_commit[i], + readl(s->ps_aon + aon->offset[i])); +} + +/* As macOS writes the PMU: the reply's vector masked, then acknowledged, so none is left raised. */ +static void sleep_pmu_write(struct apple_sleep *s, u16 reg, u8 val) +{ + u32 status, enable; + + enable = readl(s->spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, s->spmi + SPMI_NOTIFY_ENABLE); + writel(((u32)reg << 16) | SPMI_CMD_WRITE, s->spmi + SPMI_CMD); + writel(val, s->spmi + SPMI_CMD); + if (!readl_poll_timeout_atomic(s->spmi + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) + readl(s->spmi + SPMI_REPLY); + writel(1, s->spmi + SPMI_NOTIFY_ACK); + writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); +} + +static void apple_sleep_match(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + int i, raised = 0, lowered = 0, failed = 0; + + writel(0, s->wdt + WDT_WD0_COUNT); + writel(0, s->wdt + WDT_WD1_COUNT); + sleep_pmu_write(s, PMU_BOOT_STAGE, PMU_BOOT_STAGE_SLEEP); + + s->spmi_saved[0] = readl(s->spmi + SPMI_NOTIFY_ENABLE); + s->spmi_saved[1] = readl(s->spmi_a0); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi + SPMI_NOTIFY_ENABLE); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi_a0); + + for (i = 0; i < soc->ps_macos_on.count; i++) { + u32 offset = soc->ps_macos_on.offset[i]; + + s->ps_on_saved[i] = readl(s->ps + offset); + /* State 4 reads as on, and macOS has these at f. */ + if (FIELD_GET(PS_ACTUAL, s->ps_on_saved[i]) == 0xf) + continue; + /* Measured at the commit with AUTO_ENABLE clear, unlike after a wake. */ + if (sleep_ps_raise_start(s, offset)) + failed++; + raised++; + } + for (i = 0; i < soc->ps_macos_off.count; i++) { + u32 offset = soc->ps_macos_off.offset[i]; + + s->ps_off_saved[i] = readl(s->ps + offset); + if (!sleep_ps_on(s, offset)) + continue; + sleep_ps_lower(s, offset); + lowered++; + } + for (i = 0; i < soc->ps_aon_macos_off.count; i++) { + void __iomem *reg = s->ps_aon + soc->ps_aon_macos_off.offset[i]; + u32 val = readl(reg); + + s->ps_aon_saved[i] = val; + if (!FIELD_GET(PS_ACTUAL, val)) + continue; + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); + lowered++; + } + for (i = 0; i < soc->ps_aon_macos_no_auto.count; i++) { + void __iomem *reg = s->ps_aon + soc->ps_aon_macos_no_auto.offset[i]; + u32 val = readl(reg); + + s->ps_aon_auto_saved[i] = val; + writel(val & ~(PS_AUTO_ENABLE | PS_FLAGS), reg); + } + s->matched = true; + + for (i = 0; i < soc->ps_macos_off.count; i++) { + u32 offset = soc->ps_macos_off.offset[i]; + + if (sleep_ps_wait_off(s, offset)) + dev_dbg(s->dev, "still on, macOS has it off: %x = %x\n", offset, + readl(s->ps + offset)); + } + dev_dbg(s->dev, "macOS's power words: raised %d, %d of them failed, lowered %d\n", + raised, failed, lowered); +} + +/* Back as they were, after a refusal past the match or a real wake. */ +static void apple_sleep_match_undo(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + int i; + + if (!s->matched) + return; + s->matched = false; + + for (i = soc->ps_aon_macos_no_auto.count - 1; i >= 0; i--) + writel(s->ps_aon_auto_saved[i] & ~PS_FLAGS, + s->ps_aon + soc->ps_aon_macos_no_auto.offset[i]); + for (i = soc->ps_aon_macos_off.count - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_aon_saved[i])) + writel(s->ps_aon_saved[i] & ~PS_FLAGS, + s->ps_aon + soc->ps_aon_macos_off.offset[i]); + for (i = soc->ps_macos_off.count - 1; i >= 0; i--) + if (FIELD_GET(PS_ACTUAL, s->ps_off_saved[i])) + sleep_ps_raise(s, soc->ps_macos_off.offset[i]); + for (i = soc->ps_macos_on.count - 1; i >= 0; i--) { + u32 val = s->ps_on_saved[i]; + + if (!FIELD_GET(PS_ACTUAL, val)) + sleep_ps_lower(s, soc->ps_macos_on.offset[i]); + else if (FIELD_GET(PS_ACTUAL, val) != 0xf) + writel(val & ~PS_FLAGS, s->ps + soc->ps_macos_on.offset[i]); + } + writel(s->spmi_saved[0], s->spmi + SPMI_NOTIFY_ENABLE); + writel(s->spmi_saved[1], s->spmi_a0); +} + +/* The safety net, as macOS arms it. */ +static void apple_sleep_arm_wd2(struct apple_sleep *s) +{ + writel(0, s->wdt + WDT_WD0_CTL); + apple_sleep_set_wd2(s, WDT_SLEEP_SECS); + writel(0, s->aon_misc + AON_AP_DEADLINE); +} + +static int sleep_perf_wait(void __iomem *reg, unsigned int state) +{ + u32 val; + + return readl_poll_timeout_atomic(reg, val, + (val & (DCS_BUSY | DCS_STATE)) == + FIELD_PREP(DCS_STATE, state), + 1, DCS_TIMEOUT_US); +} + +static int sleep_dcs_wait(struct apple_sleep *s, unsigned int state) +{ + return sleep_perf_wait(s->dcs, state); +} + +/* + * As macOS does at about -18 ms, on each channel in turn: the enables down a bit at a time, + * then each of four lanes strobed and cleared, two lanes on every channel before the + * next two. Each write is built from a fresh read, as the trace has it. + */ +static void apple_sleep_mcc_stop(struct apple_sleep *s) +{ + int ch, pass, n; + u32 val; + + for (ch = 0; ch < MCC_CHANNELS; ch++) { + val = readl(s->mcc[ch] + MCC_ENABLES); + for (n = 0; val && n < 32; n++) { + val &= val - 1; + writel(val, s->mcc[ch] + MCC_ENABLES); + val = readl(s->mcc[ch] + MCC_ENABLES); + } + } + + for (pass = 0; pass < 2; pass++) { + for (ch = 0; ch < MCC_CHANNELS; ch++) { + void __iomem *mcc = s->mcc[ch]; + + for (n = 2 * pass; n < 2 * pass + 2; n++) { + writel(readl(mcc + MCC_LANES_MASK), mcc + MCC_LANES_MASK); + writel(readl(mcc + MCC_LANES) | MCC_LANE_STROBE(n), + mcc + MCC_LANES); + writel(readl(mcc + MCC_LANES) & ~MCC_LANE(n), mcc + MCC_LANES); + } + } + } +} + +/* + * The engine's first step polls for DCS at this state and has no way to get it there. + * macOS has every voter at 0 by now, which is how it lands there, and steps it down itself + * on the path without the PMC. Both, here, in that order. Nothing can fail by now. + */ +static void apple_sleep_dcs_down(struct apple_sleep *s) +{ + unsigned int state; + int agent; + + for (agent = 0; agent < PMC_VOTER_AGENTS; agent++) + if (readl(s->pmc_voters + agent * PMC_VOTER_STRIDE)) + writel(0, s->pmc_voters + agent * PMC_VOTER_STRIDE); + if (!sleep_dcs_wait(s, DCS_SLEEP_STATE)) + return; + + state = FIELD_GET(DCS_STATE, readl(s->dcs)); + if (state > DCS_STEP_STATE) { + writel(DCS_STEP_STATE, s->dcs); + sleep_dcs_wait(s, DCS_STEP_STATE); + } + if (state > DCS_SLEEP_STATE) { + writel(DCS_SLEEP_STATE, s->dcs); + sleep_dcs_wait(s, DCS_SLEEP_STATE); + } +} + +/* The boot core's last WFI. An event that ends it is taken, and the WFI taken again. */ +static void __noreturn sleep_boot_wfi(struct apple_sleep *s) +{ + sleep_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + aic_sleep_ack(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +/* XNU polls these off just before its complex words. The sleep goes on whatever they read. */ +static void apple_sleep_wait_still_on(struct apple_sleep *s) +{ + const struct apple_sleep_words *words = &s->soc->ps_still_on; + int i; + + for (i = 0; i < words->count; i++) + sleep_ps_wait_off(s, words->offset[i]); +} + +/* + * The last mile, in macOS's order. It starts after every refusal: this can't be + * undone, and returning now would leave the clusters armed off. No printing, and + * under 33 ms from the complex words to the WFI (the engine's bound for the + * cores to go idle). A hang from here ends at the watchdog. + * + * What's left out on purpose: + * - The two writes of 0 to 0x302800000: reads 0, changes nothing. + * - The RTC alarm: macOS arms it after the commit. rtcwake has armed it long + * before this, with no SPMI traffic near the engine. + */ +static int apple_sleep_commit(unsigned long unused) +{ + struct apple_sleep *s = apple_sleep; + int plane, agent; + u32 val; + + /* + * The other cores stop first, as on macOS. Each answered the hold in .enter, and they + * stay coherent until the commit. + */ + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + s->cpus_stop_ret = spin_table_stop_parked(); + + apple_sleep_lower_io(s); + apple_sleep_arm_wd2(s); + apple_sleep_wait_still_on(s); + + /* The SoC comes down before memory does, as on macOS. */ + if (sleep_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE)) { + writel(SOC_SLEEP_STATE, s->clk_perf + CLK_PERF_SOC); + sleep_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE); + } + apple_sleep_dcs_down(s); + apple_sleep_mcc_stop(s); + + /* Gates, PTD, the first voter twice, gates: macOS's order, measured once. */ + for (agent = 0; agent < 2; agent++) { + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[1]); + } + writeq(0, s->ptd + PTD_A); + writeq(0, s->ptd + PTD_B); + for (agent = 0; agent < 2; agent++) { + readl(s->pmc_voters); + writel(0, s->pmc_voters); + readl(s->pmc_voters + 4); + } + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[1]); + + /* Already off since .prepare, as on macOS. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_commit_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, 5000); + writel(0, s->cluster + CLUSTER_E); + writel(0, s->cluster + CLUSTER_P); + + writel(readl(s->ps + s->soc->ps.sio_cpu) & ~PS_AUTO_ENABLE, s->ps + s->soc->ps.sio_cpu); + + /* The SoC refuses the first, on macOS too. */ + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_38); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_3C); + writel(CLK_MUX_PARKED, s->clk_mux + CLK_MUX_34); + + writel(readl(s->cllt + CLLT_0) & ~CLLT_0_CLEAR, s->cllt + CLLT_0); + writel(readl(s->cllt + CLLT_1) & ~CLLT_1_CLEAR, s->cllt + CLLT_1); + writel(readl(s->cllt + CLLT_2) & ~CLLT_2_CLEAR, s->cllt + CLLT_2); + + /* + * Two runs put these writes after the CLLT's: 5 to 6 ms before the commit + * in one, and the CLLT at 5.75 ms in the other. No run has hooked both. + */ + sleep_dvfm_park(s); + + sleep_ps_save(s); + for (plane = 0; plane < ARRAY_SIZE(s->amcc); plane++) { + void __iomem *amcc = s->amcc[plane]; + + writel(readl(amcc + AMCC_WORD0_ENABLE) & ~AMCC_WORD0_DSID_AGE_OUT, + amcc + AMCC_WORD0_ENABLE); + writel(readl(amcc + AMCC_WORD1_ENABLE) & ~AMCC_WORD1_UNEXP_RT_HIT, + amcc + AMCC_WORD1_ENABLE); + } + + /* aura reads this with the MMU off. */ + s->resumes_before = READ_ONCE(s->mbox->resumes); + WRITE_ONCE(s->mbox->resume_pa, __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dsb(sy); + + writel(1, s->commit); + + /* As on macOS: a raised line would end the last WFI, and the engine waits for it. */ + aic_sleep_mask_all(); + + /* Each core's deep WFI comes after the commit, this one last. No SPMI from here on. */ + spin_table_release_stopped(); + sleep_boot_wfi(s); +} + +/* Back on the boot core through the reset vector, MMU on, in macOS's order. */ +static void apple_sleep_return(struct apple_sleep *s) +{ + u32 val; + int i; + + /* + * The reset leaves the timer control at its EL2 default, without the event stream + * or EL0's counter. The timer driver's CPU PM exit puts it back too, later. + */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + + /* This core takes the AIC's IRQs, and a reset may have changed that. */ + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + + spin_table_stop_finished(); + aic_sleep_restore(); + + writel(CLUSTER_ON, s->cluster + CLUSTER_E); + s->cluster_e_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_E, val, + val & CLUSTER_E_READY, 1, + CLUSTER_TIMEOUT_US); + writel(CLUSTER_ON, s->cluster + CLUSTER_P); + + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_wake[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_wake[1]); + + sleep_ps_rewake(s); + + /* Only C2S turns AP_TMM on. */ + s->deep_ran = sleep_ps_woke_on(s, s->soc->ps.ap_tmm); + + val = readl(s->ps + s->soc->ps.sep); + if (val & PS_DEV_DISABLE) + writel(val & ~PS_DEV_DISABLE, s->ps + s->soc->ps.sep); + + /* The complex comes back on, and macOS turns it off. */ + writel(0, s->cluster + CLUSTER_AVE); + s->cluster_ave_ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, + 1, CLUSTER_TIMEOUT_US); + + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[0]); + writel(PWRGATE_ON, s->pwrgate + s->soc->pwrgate_cpm[1]); + + sleep_dvfm_unpark(s); + + /* macOS stops it here. Keep a longer one over device resume, until .end. */ + apple_sleep_set_wd2(s, WDT_SUSPEND_SECS); + + /* + * What the commit lowered for a driver comes back before anything uses it, in + * macOS's order, measured about 14 ms after this point. With no driver, genpd's + * books decide, as they do for every other domain. + */ + if (s->pcie_ans_stopped) + s->pcie_ans_on_ret = apple_sleep_raise_apcie(s); + s->apcie_port_down = true; + s->pins_lost = s->pins_saved_valid; + + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + s->resumes_after = READ_ONCE(s->mbox->resumes); + + for (i = 0; i < WAKE_WORDS; i++) + s->wake_status[i] = readl(s->wake + WAKE_STATUS(i)); + s->returned = true; +} + +static int apple_sleep_enter(suspend_state_t state) +{ + struct apple_sleep *s = apple_sleep; + int ret; + + ret = apple_sleep_refuse(s); + if (ret) + return ret; + ret = apple_sleep_cpus_hold(s); + /* As on macOS, the display goes off before the GPU. */ + if (!ret) + ret = sleep_display_off(s); + if (!ret) + ret = apple_sleep_coprocs_check(s); + /* Cover the platform's own preparation, after the PM core's early check. */ + if (!ret && pm_wakeup_pending()) + ret = -EBUSY; + if (!ret) + ret = apple_sleep_enclave_stop(s); + if (ret) { + dev_dbg(s->dev, "refused before any commit coprocessor stop: %d\n", ret); + spin_table_unhold_parked(); + apple_sleep_stray_restore(s); + return ret; + } + apple_sleep_coprocs_stop(s, true); + + sleep_wake_arm(s); + apple_sleep_match(s); + + /* Acknowledge a leftover done flag, as the DCS word has none on macOS. */ + if (readl(s->cluster_dcs) & CLUSTER_UP_DONE) + writel(readl(s->cluster_dcs), s->cluster_dcs); + apple_sleep_coprocs_stop(s, false); + /* Done by now unless no SMC driver is bound. */ + apple_sleep_usb_down(s); + + s->cntkctl = read_sysreg(cntkctl_el1); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, apple_sleep_commit); + if (ret) + panic("Apple committed sleep returned without physical wake: %d", ret); + apple_sleep_return(s); + + apple_sleep_match_undo(s); + apple_sleep_stray_restore(s); + sleep_wake_restore(s); + + return ret; +} + +/* From here to .end a hang resets the machine. Nobody has to be there to recover it. */ +static int apple_sleep_begin(suspend_state_t state) +{ + /* The last sleep's cut is its port controller's to take, by its .complete. */ + WRITE_ONCE(sleep_debug_link_cut, false); + /* The PM core also calls .begin for s2idle when no s2idle ops are installed. */ + if (state != PM_SUSPEND_MEM) + return 0; + + if (apple_sleep) { + apple_sleep->apcie_port_stopped = false; + apple_sleep->usb_down_ran = false; + apple_sleep->smc_woke_alone = false; + sleep_pins_save(apple_sleep); + } + apple_sleep_set_wd2(apple_sleep, WDT_SUSPEND_SECS); + pm_set_suspend_via_firmware(); + + return 0; +} + +static void apple_sleep_end(void) +{ + if (pm_suspend_via_firmware()) + writel(0, apple_sleep->wdt + WDT_WD2_CTL); +} + +/* + * macOS turns this complex off at boot and again at the end of each wake, so at the commit + * its word is already 0 and the engine has no sequence to run for it. Turning it off at the + * commit instead puts that sequence, with its untimed wait, right before the CPU complexes'. + */ +static void apple_sleep_ave_off(struct apple_sleep *s) +{ + u32 val; + int ret; + + /* Target 0, and the leftover done flag from iBoot's up, as XNU's first write does. */ + writel(CLUSTER_UP_DONE, s->cluster + CLUSTER_AVE); + ret = readl_poll_timeout_atomic(s->cluster + CLUSTER_AVE, val, !val, 1, CLUSTER_TIMEOUT_US); + if (ret) + dev_warn(s->dev, "AVE complex didn't go off: %#x\n", val); +} + +static int apple_sleep_prepare(void) +{ + struct apple_sleep *s = apple_sleep; + int i; + + apple_sleep_ave_off(s); + + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + struct platform_device *pdev; + + s->display[i].bound = false; + if (!s->display[i].np) + continue; + pdev = of_find_device_by_node(s->display[i].np); + if (pdev) { + s->display[i].bound = device_is_bound(&pdev->dev); + put_device(&pdev->dev); + } + } + apple_sleep_gate_prepare(s); + + return 0; +} + +/* Only the SMC's sources woke the SoC. */ +static bool apple_sleep_smc_woke_alone(struct apple_sleep *s) +{ + const struct apple_sleep_wake_name *names = s->soc->wake_names; + u32 smc[WAKE_WORDS] = {}; + bool any = false; + int i; + + /* SMC.CPUWakeupAP and SMC.OutboxNotEmpty, by their names. */ + for (i = 0; i < s->soc->nwake_names; i++) { + u32 id = names[i].id; + + if (strstarts(names[i].name, "SMC.") && id / 32 < WAKE_WORDS) + smc[id / 32] |= BIT(id % 32); + } + for (i = 0; i < WAKE_WORDS; i++) { + u32 woke = s->wake_status[i] & s->wake_armed[i]; + + if (woke & ~smc[i]) + return false; + any |= !!woke; + } + + return any; +} + +static void apple_sleep_report(void) +{ + struct apple_sleep *s = apple_sleep; + const struct apple_sleep_wake_name *names = s->soc->wake_names; + int i; + + s->smc_woke_alone = false; + if (!s->returned) + return; + s->returned = false; + s->smc_woke_alone = apple_sleep_smc_woke_alone(s); + + dev_info(s->dev, "back from sleep, boot object resumes %llu -> %llu\n", + s->resumes_before, s->resumes_after); + if (s->deep_ran) + dev_info(s->dev, "the deep path ran\n"); + for (i = 0; i < s->soc->nwake_names; i++) { + u32 id = names[i].id; + + if (s->wake_status[id / 32] & BIT(id % 32)) + dev_info(s->dev, "wake source: %s\n", names[i].name); + } + sleep_ps_rewake_report(s); + + for (i = 0; i < SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (c->dev && c->stopped && c->stop_ret) + dev_err(s->dev, "%s stop reported %d; physical sleep completed\n", + dev_name(c->dev), c->stop_ret); + } + if (s->cpus_stop_ret) + dev_err(s->dev, "other CPUs did not stop: %d\n", s->cpus_stop_ret); + if (s->cluster_e_ret || s->cluster_ave_ret || s->cluster_ave_commit_ret) + dev_err(s->dev, "cluster power timed out: E %d AVE %d, AVE at the commit %d\n", + s->cluster_e_ret, s->cluster_ave_ret, s->cluster_ave_commit_ret); + if (s->ans_off_ret) + dev_err(s->dev, "ANS did not read off at the commit: %d\n", s->ans_off_ret); + if (s->usb_down_ret) + dev_err(s->dev, "debug link: the stop failed: %d\n", s->usb_down_ret); + if (s->pcie_ans_stopped && s->pcie_ans_on_ret) + dev_err(s->dev, "ANS's PCIe block did not come up after the wake: %d\n", + s->pcie_ans_on_ret); +} + +static const struct platform_suspend_ops apple_sleep_ops = { + .valid = suspend_valid_only_mem, + .begin = apple_sleep_begin, + .prepare = apple_sleep_prepare, + .enter = apple_sleep_enter, + .wake = apple_sleep_report, + .end = apple_sleep_end, +}; + +static int apple_sleep_read_wake_sources(struct apple_sleep *s) +{ + struct device_node *np = s->dev->of_node; + int count; + + count = of_property_read_variable_u32_array(np, "apple,wake-enable", s->wake_enable, + 1, WAKE_SOURCES_MAX); + if (count < 0) + return dev_err_probe(s->dev, count, "no wake source enabled\n"); + s->wake_enable_count = count; + + return 0; +} + +static int apple_sleep_map_displays(struct apple_sleep *s) +{ + int i; + + for (i = 0; i < SLEEP_DISPLAY_COUNT; i++) { + struct device_node *np; + int index; + + np = of_parse_phandle(s->dev->of_node, "apple,display-coprocessors", i); + if (!np) + return dev_err_probe(s->dev, -ENODEV, "no display coprocessor %d\n", i); + s->display[i].np = np; + + index = of_property_match_string(np, "reg-names", "coproc"); + if (index < 0) + return dev_err_probe(s->dev, index, "%pOF has no coproc registers\n", np); + s->display[i].asc = of_iomap(np, index); + if (!s->display[i].asc) + return dev_err_probe(s->dev, -ENOMEM, "cannot map %pOF\n", np); + } + + return 0; +} + +static void __iomem *apple_sleep_map(struct device *dev, const char *name) +{ + struct device_node *np = dev->of_node; + struct resource res; + void __iomem *base; + int index; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return IOMEM_ERR_PTR(index); + if (of_address_to_resource(np, index, &res)) + return IOMEM_ERR_PTR(-EINVAL); + + /* Shared with the drivers of these blocks, so not claimed. */ + base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!base) + return IOMEM_ERR_PTR(-ENOMEM); + + return base; +} + +/* The register blocks every sleep writes, by their reg-names. */ +static const struct { + const char *name; + size_t offset; +} apple_sleep_regs[] = { + { "ps", offsetof(struct apple_sleep, ps) }, + { "pwrgate", offsetof(struct apple_sleep, pwrgate) }, + { "cpu-start", offsetof(struct apple_sleep, cpu_start) }, + { "cllt", offsetof(struct apple_sleep, cllt) }, + { "cluster", offsetof(struct apple_sleep, cluster) }, + { "commit", offsetof(struct apple_sleep, commit) }, + { "wdt", offsetof(struct apple_sleep, wdt) }, + { "aon-misc", offsetof(struct apple_sleep, aon_misc) }, + { "wake", offsetof(struct apple_sleep, wake) }, + { "pmc-dcp", offsetof(struct apple_sleep, pmc_dcp) }, + { "spmi", offsetof(struct apple_sleep, spmi) }, + { "pmc-voters", offsetof(struct apple_sleep, pmc_voters) }, + { "dcs", offsetof(struct apple_sleep, dcs) }, + { "ps-aon", offsetof(struct apple_sleep, ps_aon) }, + { "spmi-a0", offsetof(struct apple_sleep, spmi_a0) }, + { "clk-mux", offsetof(struct apple_sleep, clk_mux) }, + { "mcc0", offsetof(struct apple_sleep, mcc[0]) }, + { "mcc1", offsetof(struct apple_sleep, mcc[1]) }, + { "mcc2", offsetof(struct apple_sleep, mcc[2]) }, + { "ptd", offsetof(struct apple_sleep, ptd) }, + { "clk-perf", offsetof(struct apple_sleep, clk_perf) }, + { "cluster-dcs", offsetof(struct apple_sleep, cluster_dcs) }, + { "dvfm-e", offsetof(struct apple_sleep, dvfm[0]) }, + { "dvfm-p", offsetof(struct apple_sleep, dvfm[1]) }, + { "amcc0", offsetof(struct apple_sleep, amcc[0]) }, + { "amcc1", offsetof(struct apple_sleep, amcc[1]) }, + { "amcc2", offsetof(struct apple_sleep, amcc[2]) }, + { "amcc3", offsetof(struct apple_sleep, amcc[3]) }, +}; + +static int apple_sleep_map_regs(struct apple_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_sleep_regs); i++) { + void __iomem **field = (void __iomem **)((u8 *)s + apple_sleep_regs[i].offset); + + *field = apple_sleep_map(s->dev, apple_sleep_regs[i].name); + if (IS_ERR(*field)) + return dev_err_probe(s->dev, PTR_ERR(*field), "no %s registers\n", + apple_sleep_regs[i].name); + } + + return 0; +} + +static int apple_sleep_map_mailbox(struct apple_sleep *s) +{ + struct device_node *chosen; + u64 pa; + int ret; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return -ENODEV; + ret = of_property_read_u64(chosen, "aurora,s2r-mailbox", &pa); + of_node_put(chosen); + if (ret) { + dev_info(s->dev, "no aurora,s2r-mailbox from the boot object: no deep sleep\n"); + return -ENODEV; + } + + s->mbox = devm_memremap(s->dev, pa, sizeof(*s->mbox), MEMREMAP_WB); + if (IS_ERR(s->mbox)) + return PTR_ERR(s->mbox); + + /* aura wrote it with the MMU off, so nothing of it can be in a cache. */ + if (s->mbox->magic != AURA_S2R_MAGIC) + return dev_err_probe(s->dev, -EINVAL, "bad mailbox magic %llx\n", + s->mbox->magic); + /* Probe can run on any CPU. Logical CPU 0 is always the boot CPU. */ + if (s->mbox->boot_mpidr != cpu_logical_map(0)) + return dev_err_probe(s->dev, -EINVAL, + "mailbox is for CPU %llx, not this boot CPU\n", + s->mbox->boot_mpidr); + + return 0; +} + +static struct apple_tunable *apple_sleep_apcie_tunable(struct apple_sleep *s, + struct device_node *np, + const char *name, const char *prop) +{ + int idx = of_property_match_string(np, "reg-names", name); + struct apple_tunable *t; + struct resource res; + + if (idx < 0 || of_address_to_resource(np, idx, &res)) + return NULL; + t = devm_apple_tunable_parse(s->dev, np, prop, &res); + + return IS_ERR(t) ? NULL : t; +} + +static void __iomem *apple_sleep_apcie_block(struct apple_sleep *s, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable) +{ + int idx = of_property_match_string(np, "reg-names", name); + + if (idx < 0) + return NULL; + if (prop) { + *tunable = apple_sleep_apcie_tunable(s, np, name, prop); + if (!*tunable) + return NULL; + } + + return of_iomap(np, idx); +} + +/* + * Its registers go with PCIe's node, which a test tree may have disabled, and a loader + * publishes the tunables there. Missing ones only refuse a real sleep with ANS stopped. + */ +static void apple_sleep_map_apcie(struct apple_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,apcie", 0); + struct device_node *engine; + + if (!np) + return; + s->apcie_axi = apple_sleep_apcie_block(s, np, "axi2af", "apple,tunable-axi2af", + &s->apcie_axi_tunable); + s->apcie_cio3pll = apple_sleep_apcie_block(s, np, "cio3pll", "apple,tunable-cio3pll", + &s->apcie_cio3pll_tunable); + s->apcie_clkgen = apple_sleep_apcie_block(s, np, "clkgen", "apple,tunable-clkgen", + &s->apcie_clkgen_tunable); + s->apcie_phy = apple_sleep_apcie_block(s, np, "phy-block", "apple,tunable-phy", + &s->apcie_phy_tunable); + s->apcie_rc = apple_sleep_apcie_block(s, np, "rc", NULL, NULL); + s->apcie_phy_ip = apple_sleep_apcie_block(s, np, "phy-ip", "apple,tunable-phy-ip-pll", + &s->apcie_phy_ip_pll); + s->apcie_phy_ip_lanes = apple_sleep_apcie_tunable(s, np, "phy-ip", + "apple,tunable-phy-ip-auspma"); + if (!s->apcie_axi || !s->apcie_cio3pll || !s->apcie_clkgen || !s->apcie_phy || + !s->apcie_rc || !s->apcie_phy_ip || !s->apcie_phy_ip_lanes) { + dev_info(s->dev, + "no PCIe tunables for ANS: suspend-to-RAM with it stopped will be refused\n"); + s->apcie_phy_tunable = NULL; + s->apcie_phy = NULL; + s->apcie_clkgen = NULL; + s->apcie_axi = NULL; + } + + s->apcie_port = apple_sleep_apcie_block(s, np, "port0", NULL, NULL); + s->apcie_port_phy = apple_sleep_apcie_block(s, np, "phy0", NULL, NULL); + engine = of_parse_phandle(np, "apple,piodma", 0); + if (engine) { + s->apcie_engine = apple_sleep_apcie_block(s, engine, "engine", NULL, NULL); + s->apcie_fabric = apple_sleep_apcie_block(s, engine, "fabric", NULL, NULL); + of_node_put(engine); + } + of_node_put(np); +} + +/* The radio port's PERST# on the AP pins @np, from the ports of the SoC's enabled PCIe host. */ +static void apple_sleep_find_radio_perst(struct apple_sleep *s, struct device_node *np) +{ + struct device_node *host, *port; + struct of_phandle_args args; + + if (!s->soc->radio_pcie) + return; + host = of_find_compatible_node(NULL, NULL, s->soc->radio_pcie); + if (!host) + return; + if (of_device_is_available(host)) { + for_each_available_child_of_node(host, port) { + if (of_parse_phandle_with_args(port, "reset-gpios", "#gpio-cells", 0, + &args)) + continue; + if (args.np == np && args.args_count >= 2 && args.args[0] < s->npins && + !test_bit(args.args[0], s->pins_skip)) { + s->radio_perst = args.args[0]; + s->radio_perst_low = args.args[1] & GPIO_ACTIVE_LOW; + } + of_node_put(args.np); + } + } + of_node_put(host); +} + +/* macOS leaves the skipped pins alone, and reading one may fault. */ +static void apple_sleep_map_pins(struct apple_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,pinctrl", 0); + u32 npins, pin; + int i, n; + + s->radio_perst = -1; + if (!np) + return; + if (!of_device_is_available(np) || of_property_read_u32(np, "apple,npins", &npins) || + !npins || npins > 512) + goto out; + s->pins_saved = devm_kcalloc(s->dev, npins, sizeof(*s->pins_saved), GFP_KERNEL); + s->pins_skip = devm_bitmap_zalloc(s->dev, npins, GFP_KERNEL); + if (!s->pins_saved || !s->pins_skip) + goto out; + n = of_property_count_u32_elems(s->dev->of_node, "apple,pinctrl-skip-pins"); + for (i = 0; i < n; i++) + if (!of_property_read_u32_index(s->dev->of_node, "apple,pinctrl-skip-pins", i, + &pin) && pin < npins) + set_bit(pin, s->pins_skip); + s->pin_irq_groups = of_irq_count(np); + s->npins = npins; + s->pins = of_iomap(np, 0); + if (s->pins) + apple_sleep_find_radio_perst(s, np); +out: + of_node_put(np); +} + +static void apple_sleep_map_debug_port(struct apple_sleep *s) +{ + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,debug-port", 0); + struct device_node *bus; + + if (!np) + return; + bus = of_get_parent(np); + if (bus && !of_property_read_u32_index(np, "reg", 0, &s->hpm_sid)) + s->hpm_spmi = of_iomap(bus, 0); + of_node_put(bus); + /* Kept, for as long as the driver: no unbind. */ + s->hpm_np = np; +} + +/* The books kept for each word of the SoC's lists */ +static int apple_sleep_alloc_books(struct apple_sleep *s) +{ + const struct apple_sleep_soc *soc = s->soc; + unsigned int n = soc->ps_all.count, naon = soc->ps_aon_all.count; + struct device *dev = s->dev; + + /* One bit, and one gate, for each */ + if (soc->nstray > BITS_PER_LONG || soc->ngate_coprocs > ARRAY_SIZE(s->gate)) + return dev_err_probe(dev, -EINVAL, "the SoC's lists are too long\n"); + + s->ps_on_saved = devm_kcalloc(dev, soc->ps_macos_on.count, sizeof(u32), GFP_KERNEL); + s->ps_off_saved = devm_kcalloc(dev, soc->ps_macos_off.count, sizeof(u32), GFP_KERNEL); + s->ps_aon_saved = devm_kcalloc(dev, soc->ps_aon_macos_off.count, sizeof(u32), + GFP_KERNEL); + s->ps_aon_auto_saved = devm_kcalloc(dev, soc->ps_aon_macos_no_auto.count, sizeof(u32), + GFP_KERNEL); + s->ps_commit = devm_kcalloc(dev, n, sizeof(u32), GFP_KERNEL); + s->ps_woke = devm_kcalloc(dev, n, sizeof(u32), GFP_KERNEL); + s->ps_late = devm_kcalloc(dev, n, sizeof(bool), GFP_KERNEL); + s->ps_done = devm_kcalloc(dev, n, sizeof(bool), GFP_KERNEL); + s->ps_aon_commit = devm_kcalloc(dev, naon, sizeof(u32), GFP_KERNEL); + s->ps_aon_woke = devm_kcalloc(dev, naon, sizeof(u32), GFP_KERNEL); + s->ps_aon_late = devm_kcalloc(dev, naon, sizeof(bool), GFP_KERNEL); + s->ps_aon_done = devm_kcalloc(dev, naon, sizeof(bool), GFP_KERNEL); + if (!s->ps_on_saved || !s->ps_off_saved || !s->ps_aon_saved || !s->ps_aon_auto_saved || + !s->ps_commit || !s->ps_woke || !s->ps_late || !s->ps_done || !s->ps_aon_commit || + !s->ps_aon_woke || !s->ps_aon_late || !s->ps_aon_done) + return -ENOMEM; + + return 0; +} + +static int apple_sleep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_sleep *s; + int ret; + + /* Below EL2 a monitor owns all of this. */ + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + s = devm_kzalloc(dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = dev; + s->soc = of_device_get_match_data(dev); + if (!s->soc) + return -ENODEV; + + /* Without the boot object's half, a wake would rerun its boot flow. */ + ret = apple_sleep_map_mailbox(s); + if (ret) + return ret; + + ret = apple_sleep_map_regs(s); + if (ret) + return ret; + ret = of_address_to_resource(dev->of_node, + of_property_match_string(dev->of_node, "reg-names", "ps"), + &s->ps_res); + if (ret) + return ret; + s->driver_down = devm_bitmap_zalloc(dev, resource_size(&s->ps_res) / 4, GFP_KERNEL); + if (!s->driver_down) + return -ENOMEM; + ret = apple_sleep_alloc_books(s); + if (ret) + return ret; + scoped_guard(spinlock_irqsave, &sleep_slp_ddr_lock) { + sleep_slp_ddr_word = s->wake + WAKE_SLP_DDR; + sleep_slp_ddr_write(); + } + + ret = apple_sleep_read_wake_sources(s); + if (ret) + return ret; + ret = apple_sleep_map_displays(s); + if (ret) + return ret; + apple_sleep_map_apcie(s); + apple_sleep_map_pins(s); + apple_sleep_map_debug_port(s); + + platform_set_drvdata(pdev, s); + + ret = spin_table_set_power_ops(&sleep_cpu_power_ops); + if (ret) + return dev_err_probe(dev, ret, "cannot control CPU power\n"); + + /* Last: the exported helpers take it as a probed driver, and a failure frees s. */ + apple_sleep = s; + /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ + suspend_set_ops(&apple_sleep_ops); + + dev_info(dev, "deep sleep %s: boot object resumes %llu\n", + mem_sleep_current == PM_SUSPEND_MEM ? "is the default" : + "available, not the default", READ_ONCE(s->mbox->resumes)); + + return 0; +} + +/* + * T8140 (MacBook Neo). Power words from the ADT's /arm-io/pmgr, measured from macOS's sleep + * under aura. CPU clusters: 0 is E, 1 is P, ADT /arm-io/pmgr reg[5], [7], [9] and [14], [16], + * [18]. + */ +#define T8140_PS_DISP_CPU 0x10000 +#define T8140_PS_DISP_FE 0x318 +#define T8140_PS_DISP_SYS 0x280 +#define T8140_PS_DISPEXT0_CPU 0x328 +#define T8140_PS_DISPEXT0_FE 0x320 +#define T8140_PS_DISPEXT0_SYS 0x288 +#define T8140_PS_SIO_CPU 0x2a8 +#define T8140_PS_SEP 0xc00 +#define T8140_PS_ATC0_USB_AON 0x3e0 +#define T8140_PS_ATC0_COMMON 0x3e8 +#define T8140_PS_ATC0_USB 0x3f0 +#define T8140_PS_AP_TMM 0x1a0 +#define T8140_PS_GPIO 0x120 +#define T8140_PS_ANS 0x278 +#define T8140_PS_APCIE_GP 0x258 +#define T8140_PS_APCIE_SYS_GP 0x2a0 +#define T8140_PS_APCIE_ST 0x310 +#define T8140_PS_APCIE_SYS_ST 0x3a0 +#define T8140_PS_APCIE_PHY_SW 0x3c0 + +/* + * Every power word in the ADT, read just before ANS's stop so a run shows the whole chip. + */ +static const u32 t8140_ps_all[] = { + 0x0, 0x8, 0x10, 0x18, 0x30, 0x38, 0x50, 0x58, 0x60, 0x100, 0x108, 0x110, 0x118, + 0x120, 0x128, 0x130, 0x138, 0x140, 0x148, 0x150, 0x158, 0x160, 0x168, 0x170, 0x178, + 0x180, 0x188, 0x190, 0x198, 0x1a0, 0x1a8, 0x1b0, 0x1b8, 0x1c0, 0x1c8, 0x1d0, 0x1d8, + 0x1e0, 0x1e8, 0x1f0, 0x1f8, 0x200, 0x208, 0x210, 0x218, 0x220, 0x228, 0x230, 0x238, + 0x240, 0x248, 0x250, 0x258, 0x260, 0x268, 0x270, 0x278, 0x280, 0x288, 0x290, 0x298, + 0x2a0, 0x2a8, 0x2b0, 0x2b8, 0x2c0, 0x2c8, 0x2d0, 0x2d8, 0x2e0, 0x2e8, 0x2f0, 0x2f8, + 0x300, 0x308, 0x310, 0x318, 0x320, 0x328, 0x330, 0x338, 0x340, 0x348, 0x350, 0x358, + 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x390, 0x398, 0x3a0, 0x3a8, 0x3b0, 0x3b8, + 0x3c0, 0x3c8, 0x3d0, 0x3d8, 0x3e0, 0x3e8, 0x3f0, 0xc00, 0x4000, 0x4008, 0x4010, + 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, 0x4050, 0x4058, 0x4060, + 0x8000, 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, + 0xc000, 0xc008, 0xc010, 0x10000, 0x10008, 0x10010, 0x10018, 0x10020, 0x10028, + 0x10030, 0x10038, 0x10040, +}; + +static const u32 t8140_ps_aon_all[] = { + 0x0, 0x58, 0x60, 0x68, 0x70, 0x78, 0x80, 0x88, 0x90, 0x98, 0xa0, 0xa8, 0xb0, + 0x4000, 0x4008, 0x4010, 0x4018, 0x4020, 0x4028, 0x4030, 0x4038, 0x4040, 0x4048, + 0x4050, 0x4058, 0x4060, 0x4068, 0x4070, 0x4078, 0x4080, 0x4088, 0x4090, 0x4098, + 0x40a0, 0x40a8, 0x40b0, 0x40b8, 0x40c0, 0x40c8, 0x40d0, 0x40d8, 0x40e0, 0x40e8, + 0x40f0, 0x40f8, 0x4100, 0x4108, 0x4110, 0x4118, 0x4120, 0x4128, 0x4130, 0x4138, + 0x4140, 0x4148, 0x4150, 0x4158, 0x4160, 0x4168, 0x4170, 0x4178, 0x4180, 0x8000, + 0x8008, 0x8010, 0x8018, 0x8020, 0x8028, 0x8030, 0x8038, 0x8040, 0x8048, 0x8050, + 0x8058, 0x8060, 0x8068, 0x8070, 0x8078, +}; + +/* What macOS has on at the commit and Linux leaves off, in its order: nothing uses them here. */ +static const u32 t8140_ps_macos_on[] = { + 0x2f0, 0x2f8, 0x358, 0x360, 0x368, 0x370, 0x378, 0x380, 0x388, 0x118, +}; + +/* + * The power words macOS's wake writes back on before any driver runs, in its order + * (logs/wakerec-20261105-154137Z). The engine leaves domains like GPIO dark, and genpd still + * has them on, so nothing else would. + */ +static const u32 t8140_ps_wake_order[] = { + 0xc00, 0x110, 0x118, 0x120, 0x168, 0x178, 0x2a8, 0x2f0, 0x2f8, 0x300, 0x358, 0x360, + 0x368, 0x370, 0x378, 0x380, 0x388, 0x3d0, 0x3d8, +}; + +static const u32 t8140_ps_aon_wake_order[] = { 0x78, 0x88, 0x8030, 0x8038 }; + +/* What macOS has off, highest first, as it lowers them. Domains with their own stop aren't here. */ +static const u32 t8140_ps_macos_off[] = { + 0x4060, 0x4058, 0x4050, 0x4048, 0x4040, 0x4038, 0x4030, 0x4028, 0x4020, 0x4018, + 0x4010, 0x4008, 0x4000, 0x398, 0x390, 0x350, 0x348, 0x340, 0x338, 0x330, + 0x308, 0x2e8, 0x2e0, 0x2d8, 0x2d0, 0x2c8, 0x2c0, 0x2b8, 0x2b0, 0x298, + 0x290, 0x270, 0x268, 0x250, 0x248, 0x240, 0x238, 0x230, 0x1a0, 0x190, + 0x188, 0x180, 0x170, 0x160, 0x158, 0x150, 0x148, 0x140, 0x108, +}; + +static const u32 t8140_ps_aon_macos_off[] = { + 0x8028, 0x4140, 0x40f8, 0x80, +}; + +/* The AOP's SPMI words: on, with AUTO_ENABLE off on macOS. */ +static const u32 t8140_ps_aon_macos_no_auto[] = { + 0x4078, 0x4080, +}; + +static const u32 t8140_ps_still_on[] = { + T8140_PS_ATC0_COMMON, T8140_PS_DISP_SYS, T8140_PS_DISPEXT0_SYS, T8140_PS_ANS, + 0x270 /* AVD_SYS */, 0x240 /* VENC0_SYS */, 0x248 /* VENC1_SYS */, 0x1f0 /* GFX */, + 0x230 /* JPG */, T8140_PS_APCIE_SYS_GP, +}; + +/* + * What XNU polls just before its complex words, and panics on if one is on, besides the + * display's, ANS's and ATC0's, which the commit lowers itself. + */ +static const struct apple_sleep_word t8140_stray_words[] = { + { "AVD_SYS", 0x270 }, { "VENC0_SYS", 0x240 }, { "VENC1_SYS", 0x248 }, + { "GFX", 0x1f0 }, { "JPG", 0x230 }, +}; + +/* + * A commit with the PMP and SIO still running reset the SoC 5.5 s later. The DCPs' drivers + * power their CPUs off at DPMS off. + */ +static const struct apple_sleep_gate_coproc t8140_gate_coprocs[] = { + { "/soc/dcp@412e00000", true, T8140_PS_DISP_CPU }, + { "/soc/dcp@416e00000", true, T8140_PS_DISPEXT0_CPU }, + { "/soc/pmp@300500000", true }, { "/soc/dma-controller@385e00000", true }, + { "/soc/nvme@3cdcc0000" }, { "/soc/mtp@314600000" }, + { "/soc/aop@310c00000" }, { "/soc/smc@30c600000" }, + { "/soc/sep@282600000" }, { "/soc/gpu@482600000" }, +}; + +static const struct apple_sleep_word t8140_gate_off[] = { + { "ane_sys", 0x290 }, { "isp_sys", 0x268 }, { "prores", 0x250 }, + { "avd_sys", 0x270 }, { "jpg", 0x230 }, { "venc0_sys", 0x240 }, +}; + +static const struct apple_sleep_wake_name t8140_wake_names[] = { + { 0, "AOP.AICWakeupAP" }, { 5, "AOP.CPUWakeupAP" }, + { 42, "AOP.OutboxNotEmpty" }, { 62, "AOP.SPMI0.SW3" }, + { 66, "AOP.SPMI1.SW3" }, { 70, "ATC0_PMGR_USB_WAKEUP" }, + { 80, "MTP.DOCK.CHANNELS.AP0.IRQ" }, { 99, "MTP.OUTBOX.N.EMPTY_IRQ" }, + { 109, "NUB.APWakeupTime" }, { 110, "NUB.APWatchdog" }, + { 125, "NUB.GPIO.IRQ6" }, { 131, "NUB.SPMI0.SW3" }, + { 136, "NUB.SPMI1.SW3" }, { 140, "NUB.WakeupAP" }, + { 145, "SMC.CPUWakeupAP" }, { 165, "SMC.OutboxNotEmpty" }, + { 211, "AOP.TrustedOutboxNotEmpty" }, { 227, "AOP2.OutboxNotEmpty" }, + { 231, "AOP2.TrustedOutboxNotEmpty" }, { 236, "AOP2.CPUWakeupAP" }, +}; + +static const struct apple_sleep_soc t8140_sleep_soc = { + .ps = { + .disp = { + { T8140_PS_DISP_CPU, T8140_PS_DISP_FE, T8140_PS_DISP_SYS }, + { T8140_PS_DISPEXT0_CPU, T8140_PS_DISPEXT0_FE, T8140_PS_DISPEXT0_SYS }, + }, + .sio_cpu = T8140_PS_SIO_CPU, + .sep = T8140_PS_SEP, + .usb_aon = T8140_PS_ATC0_USB_AON, + .usb_common = T8140_PS_ATC0_COMMON, + .usb = T8140_PS_ATC0_USB, + .ap_tmm = T8140_PS_AP_TMM, + .gpio = T8140_PS_GPIO, + .ans = T8140_PS_ANS, + .apcie_gp = T8140_PS_APCIE_GP, + .apcie_sys_gp = T8140_PS_APCIE_SYS_GP, + .apcie_st = T8140_PS_APCIE_ST, + .apcie_sys_st = T8140_PS_APCIE_SYS_ST, + .apcie_phy_sw = T8140_PS_APCIE_PHY_SW, + }, + .ps_aon_debug_switch = 0xb0, + .pwrgate_wake = { 0x000, 0x040 }, + .pwrgate_cpm = { 0x060, 0x068 }, + .pwrgate_disp = { 0xc8c, 0xcd0 }, + .ps_all = SLEEP_WORDS(t8140_ps_all), + .ps_aon_all = SLEEP_WORDS(t8140_ps_aon_all), + .ps_macos_on = SLEEP_WORDS(t8140_ps_macos_on), + .ps_macos_off = SLEEP_WORDS(t8140_ps_macos_off), + .ps_aon_macos_off = SLEEP_WORDS(t8140_ps_aon_macos_off), + .ps_aon_macos_no_auto = SLEEP_WORDS(t8140_ps_aon_macos_no_auto), + .ps_wake_order = SLEEP_WORDS(t8140_ps_wake_order), + .ps_aon_wake_order = SLEEP_WORDS(t8140_ps_aon_wake_order), + .ps_still_on = SLEEP_WORDS(t8140_ps_still_on), + .stray = t8140_stray_words, + .nstray = ARRAY_SIZE(t8140_stray_words), + .gate_off = t8140_gate_off, + .ngate_off = ARRAY_SIZE(t8140_gate_off), + .gate_coprocs = t8140_gate_coprocs, + .ngate_coprocs = ARRAY_SIZE(t8140_gate_coprocs), + .wake_names = t8140_wake_names, + .nwake_names = ARRAY_SIZE(t8140_wake_names), + /* 21 sources. Measured: macOS reads 0x001fef77 there. */ + .wake_last_word = GENMASK(20, 0), + .radio_pcie = "apple,t8140-pcie", +}; + +/* + * Only SoCs whose sleep was measured and tested. On every other the driver doesn't bind, and + * s2idle stays the only sleep. + */ +static const struct of_device_id apple_sleep_of_match[] = { + { .compatible = "apple,t8140-sleep", .data = &t8140_sleep_soc }, + { } +}; + +static struct platform_driver apple_sleep_driver = { + .driver = { + .name = "apple-sleep", + .of_match_table = apple_sleep_of_match, + .suppress_bind_attrs = true, + }, + .probe = apple_sleep_probe, +}; +builtin_platform_driver(apple_sleep_driver); + +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP_KUNIT_TEST) +#include "sleep-test.c" +#endif diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 58ada41ed96ec..b855d2bc82c83 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -12,7 +12,9 @@ */ #include +#include #include +#include #include #include #include @@ -23,8 +25,10 @@ #include #include #include +#include #include #include +#include /* SPMI Controller Registers */ #define SPMI_STATUS_REG 0 @@ -96,8 +100,11 @@ struct apple_spmi { struct irq_domain *irqd; raw_spinlock_t irq_mask_lock; DECLARE_BITMAP(irq_mask_cache, SPMI_MAX_PERIPHERAL_IRQS); + /* The sources a driver uses, enabled or not. */ + DECLARE_BITMAP(irq_used, SPMI_MAX_PERIPHERAL_IRQS); int irq; bool notify_irq; + bool notify_parked; bool prev_fail; }; @@ -339,8 +346,11 @@ static struct irq_chip apple_spmi_irq_chip = { static int apple_spmi_irq_domain_map(struct irq_domain *irqd, unsigned int irq, irq_hw_number_t hw) { + struct apple_spmi *spmi = irqd->host_data; + irq_domain_set_info(irqd, irq, hw, &apple_spmi_irq_chip, irqd->host_data, handle_edge_irq, NULL, NULL); + set_bit(hw, spmi->irq_used); return 0; } @@ -394,7 +404,9 @@ static void apple_spmi_irq_domain_free(struct irq_domain *irqd, unsigned int vir for (i = 0; i < nr_irqs; i++) { struct irq_data *d = irq_domain_get_irq_data(irqd, virq + i); + struct apple_spmi *spmi = irqd->host_data; + clear_bit(d->hwirq, spmi->irq_used); irq_set_handler(virq + i, NULL); irq_domain_reset_irq_data(d); } @@ -411,19 +423,44 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) struct apple_spmi *spmi = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); u32 notify = spmi->hw->num_irqs; - bool handled = false; + bool handled = false, held = false, raw_pending = false; unsigned long val, offset, bit; chained_irq_enter(chip, desc); val = readl(spmi->regs + SPMI_IRQ_ACK_BASE + (notify / 32) * 4); + raw_pending |= !!val; if (val & BIT(notify % 32)) { apple_spmi_irq_ack_raw(spmi, notify); complete(&spmi->fifo_rx); handled = true; } + /* Other queues' responses, such as the sleep's own commands at the debug link's cut. */ + if (val & ~BIT(notify % 32)) { + writel(val & ~BIT(notify % 32), spmi->regs + SPMI_IRQ_ACK_BASE + (notify / 32) * 4); + handled = true; + } for (offset = 0; offset < notify / 8; offset += sizeof(val)) { + unsigned long stray; + val = readq(spmi->regs + SPMI_IRQ_ACK_BASE + offset); + raw_pending |= !!val; + /* + * After a real sleep, sources no driver uses come up pending (on the Neo the + * USB-PD controllers' even lines, latched by the debug link's cut), and the + * interrupt fires for them with nothing of ours to run. Clear them, as probe does. + * A used source that is masked, as all are until the resume enables them, stays + * pending for its unmask: it isn't a bad interrupt either. + */ + stray = val & ~spmi->irq_used[offset / sizeof(val)]; + if (stray) { + writel(lower_32_bits(stray), spmi->regs + SPMI_IRQ_ACK_BASE + offset); + writel(upper_32_bits(stray), spmi->regs + SPMI_IRQ_ACK_BASE + offset + 4); + handled = true; + } + if (val & spmi->irq_used[offset / sizeof(val)] & + ~spmi->irq_mask_cache[offset / sizeof(val)]) + held = true; /** * because of other masters in the bus, we're going to get a multitude of * interrupts we're not interested in. irq_resolve_mapping isn't very @@ -436,8 +473,19 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) handled = true; } } - if (!handled) + /* + * Native dispatch completes normally when every sampled bank is empty. + * Keep the diagnostic for unclassified nonempty samples, including a + * mask transition; an empty parent needs no invented controller ACK. + */ + if (!handled && !held && raw_pending) handle_bad_irq(desc); + /* + * The acks are posted writes: let them reach the controller before the parent is + * unmasked, or it fires again for what was just cleared. + */ + if (handled) + readl(spmi->regs + SPMI_IRQ_ACK_BASE); chained_irq_exit(chip, desc); } @@ -485,6 +533,63 @@ static int apple_spmi_init_irq(struct platform_device *pdev, return 0; } +/* + * Every command asks for an interrupt on its response, on the vector this + * driver waits on. The system's wake logic watches this queue's interrupt line + * for the PMU's RTC alarm, so that vector has to be quiet and clear before a + * firmware-driven suspend, or the commit may read our own traffic as a wake. + * macOS clears the same enable bit at about the same point (measured under aura). + * Any command sent from here on polls for its response. + */ +static int apple_spmi_suspend_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + unsigned long flags; + + if (!spmi->notify_irq || !pm_suspend_via_firmware()) + return 0; + + guard(mutex)(&spmi->fifo_lock); + spmi->notify_irq = false; + spmi->notify_parked = true; + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + apple_spmi_irq_mask_raw(spmi, spmi->hw->num_irqs); + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + + return 0; +} + +static int apple_spmi_resume_noirq(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + u32 words[SPMI_MAX_PERIPHERAL_IRQS / 32]; + unsigned long flags; + size_t i; + + if (!spmi->notify_parked) + return 0; + + guard(mutex)(&spmi->fifo_lock); + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + /* Whether the enables survive a real sleep has not been measured. */ + if (pm_resume_via_firmware()) { + bitmap_to_arr32(words, spmi->irq_mask_cache, spmi->hw->num_irqs); + for (i = 0; i < spmi->hw->num_irqs / 32; i++) + writel(words[i], spmi->regs + SPMI_IRQ_MASK_BASE + i * 4); + } + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); + apple_spmi_irq_unmask_raw(spmi, spmi->hw->num_irqs); + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); + spmi->notify_parked = false; + spmi->notify_irq = true; + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(apple_spmi_pm_ops, apple_spmi_suspend_noirq, + apple_spmi_resume_noirq); + static int apple_spmi_probe(struct platform_device *pdev) { struct apple_spmi *spmi; @@ -547,6 +652,7 @@ static struct platform_driver apple_spmi_driver = { .driver = { .name = "apple-spmi", .of_match_table = apple_spmi_match_table, + .pm = pm_sleep_ptr(&apple_spmi_pm_ops), }, }; module_platform_driver(apple_spmi_driver); diff --git a/drivers/tty/serial/apple_dockchannel.h b/drivers/tty/serial/apple_dockchannel.h index 97fcccc6a5a57..4fcecee3b81c5 100644 --- a/drivers/tty/serial/apple_dockchannel.h +++ b/drivers/tty/serial/apple_dockchannel.h @@ -24,13 +24,20 @@ */ #define DOCKCHANNEL_NEVER_WAIT "apple,tx-never-wait" -/* Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. */ -static inline void dockchannel_wait_putc(void __iomem *data, unsigned char c) +/* + * Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. With the + * debug host gone the FIFO stays full: after one character timed out, the next + * are tried once each (*@no_reader) until there's room again, so printing on the + * way into and out of a sleep doesn't hold anything up. + */ +static inline void dockchannel_wait_putc(void __iomem *data, bool *no_reader, + unsigned char c) { - unsigned int retries = DOCKCHANNEL_TX_RETRIES; + unsigned int retries = *no_reader ? 1 : DOCKCHANNEL_TX_RETRIES; while (readl(data + DOCKCHANNEL_TX_FREE) == 0 && --retries) cpu_relax(); + *no_reader = !retries; if (!retries) return; diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 6c427189d34be..5004f17501e47 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -16,13 +16,15 @@ static struct dockchannel_tx dockchannel_early_tx; static bool dockchannel_early_never_wait; +/* The last character timed out: see dockchannel_wait_putc(). */ +static bool dockchannel_early_no_reader; static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) { if (dockchannel_early_never_wait) dockchannel_console_putc(port->membase, &dockchannel_early_tx, c); else - dockchannel_wait_putc(port->membase, c); + dockchannel_wait_putc(port->membase, &dockchannel_early_no_reader, c); } /* The DT's DockChannel UART node asks for DOCKCHANNEL_NEVER_WAIT (flat DT). */ diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index d4255cb949848..7150cf1cca350 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -109,6 +109,9 @@ static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) dcuart_irq_update(dcuart, bits, false); } +/* The last console character timed out: see dockchannel_wait_putc(). */ +static bool dcuart_no_reader; + static void dcuart_putc(struct uart_port *port, unsigned char c) { struct dcuart_port *dcuart = container_of(port, struct dcuart_port, port); @@ -116,7 +119,7 @@ static void dcuart_putc(struct uart_port *port, unsigned char c) if (dcuart->never_wait) dockchannel_console_putc(port->membase, &dcuart->console_tx, c); else - dockchannel_wait_putc(port->membase, c); + dockchannel_wait_putc(port->membase, &dcuart_no_reader, c); } static void dcuart_write(struct console *co, const char *s, diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 8e8a5f59b319b..0ea6a9a729bc5 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -2280,7 +2280,9 @@ int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg) usb_set_device_state(rhdev, USB_STATE_SUSPENDED); hcd->state = HC_STATE_SUSPENDED; - if (!PMSG_IS_AUTO(msg)) + /* A suspend after a raced autoresume finds them released already. */ + if (!PMSG_IS_AUTO(msg) && + !test_and_set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); @@ -2321,11 +2323,12 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) return 0; } - if (!PMSG_IS_AUTO(msg)) { + if (!PMSG_IS_AUTO(msg) && test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { status = usb_phy_roothub_resume(hcd->self.sysdev, hcd->phy_roothub); if (status) return status; + clear_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); } if (!hcd->driver->bus_resume) @@ -2368,7 +2371,11 @@ int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg) } } else { hcd->state = old_state; - usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + /* Only release what this resume took: an autoresume took nothing. */ + if (!PMSG_IS_AUTO(msg)) { + usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub); + set_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags); + } dev_dbg(&rhdev->dev, "bus %s fail, err %d\n", "resume", status); if (status != -ESHUTDOWN) @@ -3084,8 +3091,16 @@ void usb_remove_hcd(struct usb_hcd *hcd) usb_deregister_bus(&hcd->self); hcd_buffer_destroy(hcd); - usb_phy_roothub_power_off(hcd->phy_roothub); - usb_phy_roothub_exit(hcd->phy_roothub); + /* + * A root hub whose resume failed still has its PHYs released by the suspend, + * powered off and, without wakeup, exited too. Don't release them twice. + */ + if (!test_bit(HCD_FLAG_PHY_SUSPENDED, &hcd->flags)) { + usb_phy_roothub_power_off(hcd->phy_roothub); + usb_phy_roothub_exit(hcd->phy_roothub); + } else if (device_may_wakeup(hcd->self.sysdev)) { + usb_phy_roothub_exit(hcd->phy_roothub); + } usb_put_invalidate_rhdev(hcd); hcd->flags = 0; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 30b5c5fc4d071..7821ccb4686e5 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1002,10 +1002,13 @@ struct dwc3_scratchpad_array { * need to be passed on to glue layer * @pre_set_role: Notify glue of role switch notifications * @pre_run_stop: Notify run stop enable/disable information to glue + * @pre_host_resume: Before xHCI touches the core at a system resume; an error keeps it away, + * and a positive value says the core lost its state and was set up again */ struct dwc3_glue_ops { void (*pre_set_role)(struct dwc3 *dwc, enum usb_role role); void (*pre_run_stop)(struct dwc3 *dwc, bool is_on); + int (*pre_host_resume)(struct dwc3 *dwc); }; /** diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index 7a04604ddd4a1..69abb0f32653f 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -38,10 +38,19 @@ static void dwc3_apple_non_role_test(struct kunit *test) } } +/* A role that arrives during a sleep decides what comes back after it. */ +static void dwc3_apple_role_during_sleep_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST), DWC3_APPLE_HOST); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE), DWC3_APPLE_DEVICE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE), DWC3_APPLE_NO_CABLE); +} + static struct kunit_case dwc3_apple_state_test_cases[] = { KUNIT_CASE(dwc3_apple_host_left_to_xhci_test), KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), KUNIT_CASE(dwc3_apple_non_role_test), + KUNIT_CASE(dwc3_apple_role_during_sleep_test), {} }; diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index 42c13d900428e..bb8d827fdec4c 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -9,6 +9,7 @@ #include #include +#include /* See the comment at the top of dwc3-apple.c for how these states are used. */ enum dwc3_apple_state { @@ -47,4 +48,22 @@ static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target } } +/** + * dwc3_apple_state_for_role() - state a role asks for while the core is stopped for sleep + * @role: the role asked for + * + * Return: the state to bring the controller up in after the sleep. + */ +static inline enum dwc3_apple_state dwc3_apple_state_for_role(enum usb_role role) +{ + switch (role) { + case USB_ROLE_HOST: + return DWC3_APPLE_HOST; + case USB_ROLE_DEVICE: + return DWC3_APPLE_DEVICE; + default: + return DWC3_APPLE_NO_CABLE; + } +} + #endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f66370e6b5f88..c3b02c3602d0a 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -9,16 +9,23 @@ */ #include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include +#include +#include +#include #include #include "dwc3-apple-state.h" @@ -91,6 +98,15 @@ * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device * may wake the system, as a halted xHC could not signal the wakeup. * + * A deep sleep takes the controller's power domain away. Behind a fixed hub on T8140 the host + * sleeps in place, as macOS's does: xHCI suspends and resumes as usual, and the PHY takes the + * core through hibernation in between, so the hub and its devices stay. The core comes back + * without its registers and is set up again in place. With its always-on side kept, xHCI + * restores its saved state and the hub resumes from U3, with no reset; without it xHCI starts + * over, and the devices reset-resume. Anywhere else, or when the PHY can't, the glue stops + * everything before the freezer runs, the way a cable disconnect does, and brings it back + * once the system has resumed. See dwc3_apple_pm_notify(). + * * Runtime PM (T6040 and T8132, see dwc3_apple_core_probe()): the glue holds a reference from * the reset deassert to the end of the shutdown, so with no cable the controller can * runtime-suspend and its power domain drop, its DARTs (which save and restore their state) and, @@ -128,6 +144,7 @@ * @rpm_held: The glue holds a runtime PM reference for the running core * @pre_core_rpm: Runtime PM is the glue's own until the first core bring-up * @dt_dis_u3_susphy: The device tree keeps USB3 PHY suspend off itself + * @sleep_ddr: The host votes for DRAM in a sleep while it runs (apple,sleep-ddr) * @drd_regs: The controller's optional blocks outside dwc3-core (drd-cfg, drd-bulk, drd-ausbc), * mapped for their tunables * @tunables: Controller tunables from the device tree, see dwc3_apple_tunable_list @@ -144,6 +161,8 @@ struct dwc3_apple { struct phy *usb3_phy; struct usb_role_switch *role_sw; bool fixed_hub; + /* The root port the fixed hub sits on, its node's reg. */ + u32 hub_port; bool host_only; struct mutex lock; @@ -159,6 +178,21 @@ struct dwc3_apple { bool rpm_held; bool pre_core_rpm; bool dt_dis_u3_susphy; + bool sleep_ddr; + struct notifier_block pm_nb; + /* Kept through a deep sleep, with the PHY's hibernation, and the GUID it had. */ + bool sleep_in_place; + u32 sleep_guid; + /* The core lost its registers in that sleep and was set up again in place. */ + bool sleep_reinit; + /* Looks for the fixed hub once a host restarted after a sleep has had time to find it. */ + struct delayed_work hub_check; + /* The core didn't come back from a sleep or a power cycle: nothing may touch it. */ + bool sleep_dead; + /* Stopped for a deep sleep: the state to come back to, and the wakeup setting. */ + bool sleep_stopped; + bool sleep_wakeup; + enum dwc3_apple_state sleep_state; void __iomem *drd_regs[DWC3_APPLE_NR_DRD_REGS]; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; @@ -210,6 +244,27 @@ static inline void dwc3_apple_mask(struct dwc3_apple *appledwc, u32 offset, u32 dwc3_apple_writel(appledwc, offset, reg); } +/* + * macOS's T8140 controller reset sets GCTL's hibernation enable and a power-down scale + * of 0xd. Without the enable the core's power management accepts a + * D0/D3 request but saves and restores nothing, so the PHY's hibernation for a sleep + * needs it. + */ +#define DWC3_APPLE_T8140_PWRDNSCALE 0xd + +static void dwc3_apple_setup_hibernation(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 reg; + + if (!of_device_is_compatible(appledwc->dev->of_node, "apple,t8140-dwc3")) + return; + reg = dwc3_readl(dwc, DWC3_GCTL); + reg &= ~DWC3_GCTL_PWRDNSCALE_MASK; + reg |= DWC3_GCTL_PWRDNSCALE(DWC3_APPLE_T8140_PWRDNSCALE) | DWC3_GCTL_GBLHIBERNATIONEN; + dwc3_writel(dwc, DWC3_GCTL, reg); +} + static void dwc3_apple_setup_cio(struct dwc3_apple *appledwc) { dwc3_apple_writel(appledwc, APPLE_DWC3_CIO_LFPS_OFFSET, APPLE_DWC3_CIO_LFPS_OFFSET_VALUE); @@ -361,6 +416,105 @@ static void dwc3_apple_set_ptrcap(struct dwc3_apple *appledwc, u32 mode) dwc3_set_prtcap(&appledwc->dwc, mode, false); } +/* Written to the core's GUID scratch register before a sleep in place, read back after it. */ +#define DWC3_APPLE_SLEEP_GUID 0x534c5045 + +/* macOS waits up to 100 ms for this after the core's D0, and only logs a timeout. */ +#define DWC3_APPLE_CFG_STATUS 0x2c +#define DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY BIT(30) +#define DWC3_APPLE_PIPE_PHY_READY_US 100000 + +/* + * The core is powered but its registers are at their reset values. Set it up again the way + * dwc3_apple_core_start() and dwc3_apple_set_role() do, in place: the PHYs and the xHCI + * device stay, so xHCI can restore its saved state, or start over. Runs in the resume, + * while nothing else touches the core. + */ +static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + int i, ret; + + ret = dwc3_core_init(dwc); + if (ret) + return ret; + /* dwc3_core_init() took PHY references the core holds already. */ + for (i = 0; i < dwc->num_usb3_ports; i++) { + phy_power_off(dwc->usb3_generic_phy[i]); + phy_exit(dwc->usb3_generic_phy[i]); + } + for (i = 0; i < dwc->num_usb2_ports; i++) { + phy_power_off(dwc->usb2_generic_phy[i]); + phy_exit(dwc->usb2_generic_phy[i]); + } + + dwc3_apple_setup_cio(appledwc); + dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); + dwc3_apple_set_ptrcap(appledwc, DWC3_GCTL_PRTCAP_HOST); + dwc->dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; + dwc3_enable_susphy(dwc, true); + /* A PHY that came back in a USB3 mode routes the PIPE to it now, before xHCI starts. */ + ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_HOST); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed after the sleep, USB2 only, err=%d\n", + ret); + + return 0; +} + +/* + * Runs before xHCI touches the core after a sleep in place. A core that may be unpowered must + * be left alone, as an access could be an SError: dwc3_apple_sleep_kept() deals with it once + * the system is back. A core that came back without its registers is set up again. xHCI then + * restores its saved state, as macOS's does, unless the always-on side lost power too: then + * it's told it lost its state (a positive return). A sleep refused before the hibernation + * left the core as it was. + */ +static int dwc3_apple_pre_host_resume(struct dwc3 *dwc) +{ + struct dwc3_apple *appledwc = to_dwc3_apple(dwc); + bool lost; + u32 val; + int ret; + + if (!READ_ONCE(appledwc->sleep_in_place)) + return 0; + + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + lost = ret == APPLE_ATCPHY_SLEEP_POWER_LOST; + if (ret == -EBUSY || lost) + ret = 0; + if (!ret && appledwc->drd_regs[0] && /* drd-cfg */ + readl_poll_timeout(appledwc->drd_regs[0] + DWC3_APPLE_CFG_STATUS, val, + val & DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY, 100, + DWC3_APPLE_PIPE_PHY_READY_US)) + dev_warn(appledwc->dev, "PIPE PHY not ready after the sleep: %#x\n", val); + if (!ret && dwc3_readl(dwc, DWC3_GUID) != DWC3_APPLE_SLEEP_GUID) { + ret = dwc3_apple_core_reinit(appledwc); + if (!ret && !lost) { + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core lost its registers in the sleep: set up again, xHCI restores its state\n"); + return 0; + } + if (!ret) { + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core and its always-on side lost power in the sleep: set up again, xHCI starts over\n"); + return 1; + } + } + if (ret) + dev_err(appledwc->dev, "not resuming xHCI, the core didn't come back: %d\n", ret); + + return ret; +} + +static const struct dwc3_glue_ops dwc3_apple_glue_ops = { + .pre_host_resume = dwc3_apple_pre_host_resume, +}; + static bool dwc3_apple_host_before_core_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { @@ -378,6 +532,7 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc, enum dwc3_apple_st WARN_ON_ONCE(appledwc->state != DWC3_APPLE_PROBE_PENDING); appledwc->dwc.dev = appledwc->dev; + appledwc->dwc.glue_ops = &dwc3_apple_glue_ops; probe_data.dwc = &appledwc->dwc; probe_data.res = appledwc->mmio_resource; probe_data.ignore_clocks_and_resets = true; @@ -622,6 +777,7 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st dwc3_apple_host_before_core_init(appledwc, state)) dwc3_apple_setup_cio(appledwc); dwc3_apple_apply_tunables(appledwc); + dwc3_apple_setup_hibernation(appledwc); appledwc->core_active = true; appledwc->usb2_power_off_pending = BIT(appledwc->dwc.num_usb2_ports) - 1; appledwc->usb3_power_off_pending = BIT(appledwc->dwc.num_usb3_ports) - 1; @@ -762,6 +918,9 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st } appledwc->state = state; + /* macOS votes USB_SLP_DDR from here until the host's Off, through every sleep. */ + if (state == DWC3_APPLE_HOST && appledwc->sleep_ddr) + apple_sleep_usb_slp_ddr(true); /* * A connected cable leaves the Apple DWC3 core and its PHY as one live, * stateful unit. The glue driver tears that unit down explicitly on a @@ -784,6 +943,152 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st return ret; } +static int dwc3_apple_exit(struct dwc3_apple *appledwc); + +/* A hub that hasn't enumerated by then didn't come back. */ +#define DWC3_APPLE_HUB_CHECK_MS 5000 + +/* Any way a fixed-hub host came out of a sleep, its hub must be there a little later. */ +static void dwc3_apple_check_hub_later(struct dwc3_apple *appledwc) +{ + if (appledwc->fixed_hub) + mod_delayed_work(system_freezable_wq, &appledwc->hub_check, + msecs_to_jiffies(DWC3_APPLE_HUB_CHECK_MS)); +} + +/* + * Bring a controller up again after a sleep that stopped it or lost it. A fixed-hub host + * starts as at probe, where the core has no role yet, so dwc3_core_soft_reset() runs too: + * rebinding the driver brought the J700's hub back where a restart without it didn't. + */ +static int dwc3_apple_start_after_sleep(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +{ + int ret; + + lockdep_assert_held(&appledwc->lock); + if (appledwc->fixed_hub) + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, state); + if (ret) { + dev_err(appledwc->dev, "Failed to start after sleep, err=%d\n", ret); + return ret; + } + dwc3_apple_check_hub_later(appledwc); + return 0; +} + +/* Whether the fixed hub is on its USB2 root port. Takes the xHCI device's lock. */ +static bool dwc3_apple_hub_present(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root; + struct usb_hcd *hcd; + bool present = true; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return true; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + present = usb_hub_find_child(root, appledwc->hub_port); + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return present; +} + +/* Whether @udev or a device below it can wake the system. Takes the hubs' locks. */ +static bool dwc3_apple_can_wake(struct usb_device *udev) +{ + struct usb_device *child; + bool wake = device_may_wakeup(&udev->dev); + int port1; + + if (wake || !udev->maxchild) + return wake; + + usb_lock_device(udev); + usb_hub_for_each_child(udev, port1, child) { + wake = dwc3_apple_can_wake(child); + if (wake) + break; + } + usb_unlock_device(udev); + + return wake; +} + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The hub's own setting doesn't count. Takes the xHCI device's lock and the hubs'. + */ +static unsigned long dwc3_apple_hub_wake_ports(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_device *root, *hub, *child; + unsigned long ports = 0; + struct usb_hcd *hcd; + int port1; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return 0; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + root = hcd ? hcd->self.root_hub : NULL; + if (root) { + usb_lock_device(root); + hub = usb_hub_find_child(root, appledwc->hub_port); + if (hub) { + usb_lock_device(hub); + usb_hub_for_each_child(hub, port1, child) { + if (port1 < BITS_PER_LONG && dwc3_apple_can_wake(child)) + ports |= BIT(port1); + } + usb_unlock_device(hub); + } + usb_unlock_device(root); + } + device_unlock(&xhci->dev); + + return ports; +} + +/* + * Once after a sleep: if the hub isn't there, stop the host and start it again, with the + * hub's link down for a moment first, as unbinding and binding the driver does. This also + * catches an xHCI that didn't come back after the core was set up again. + */ +static void dwc3_apple_hub_check(struct work_struct *work) +{ + struct dwc3_apple *appledwc = container_of(to_delayed_work(work), struct dwc3_apple, + hub_check); + int ret; + + guard(mutex)(&appledwc->lock); + if (appledwc->shutting_down || appledwc->state != DWC3_APPLE_HOST || + dwc3_apple_hub_present(appledwc)) + return; + + dev_warn(appledwc->dev, "hub not back after the sleep: starting the host again\n"); + ret = dwc3_apple_exit(appledwc); + if (ret) { + dev_err(appledwc->dev, "Failed to stop the host again, err=%d\n", ret); + return; + } + msleep(500); + appledwc->dwc.current_dr_role = 0; + ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); + if (ret) + dev_err(appledwc->dev, "Failed to start the host again, err=%d\n", ret); +} + static int dwc3_apple_exit(struct dwc3_apple *appledwc) { lockdep_assert_held(&appledwc->lock); @@ -801,6 +1106,8 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) appledwc->dwc.dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; dwc3_host_exit(&appledwc->dwc); appledwc->state = DWC3_APPLE_NO_CABLE; + if (appledwc->sleep_ddr) + apple_sleep_usb_slp_ddr(false); break; } @@ -821,6 +1128,12 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role ro if (appledwc->fixed_hub) return 0; + /* Stopped for a sleep: this is what comes back after it. */ + if (appledwc->sleep_stopped) { + appledwc->sleep_state = dwc3_apple_state_for_role(role); + return 0; + } + /* * Stay off instead of starting device mode (see dwc3_apple_host_only()). Reporting success * keeps the port controller from retrying: get() then reports USB_ROLE_NONE. @@ -875,7 +1188,7 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) guard(mutex)(&appledwc->lock); - switch (appledwc->state) { + switch (appledwc->sleep_stopped ? appledwc->sleep_state : appledwc->state) { case DWC3_APPLE_HOST: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: @@ -905,11 +1218,209 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } -static bool dwc3_apple_has_fixed_hub(struct device *dev) +/* + * A core that may be unpowered gets one more power cycle, as macOS's Off and cold start, and + * the host starts over once it's back. Until then nothing may touch it, and every sleep is + * refused, as xHCI's suspend would touch it. Returns whether it's back. + */ +static bool dwc3_apple_recover(struct dwc3_apple *appledwc) +{ + int ret; + + lockdep_assert_held(&appledwc->lock); + ret = apple_atcphy_sleep_recover(appledwc->usb2_phy); + if (ret < 0) { + if (!appledwc->sleep_dead) + dev_err(appledwc->dev, + "USB host unpowered after the sleep (%d): USB off, sleeps refused until it's back or a reboot\n", + ret); + appledwc->sleep_dead = true; + return false; + } + if (appledwc->sleep_dead) + dev_info(appledwc->dev, "USB host back after a power cycle: restarting it\n"); + appledwc->sleep_dead = false; + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after the power cycle, err=%d\n", ret); + else + dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + + return true; +} + +/* + * After a sleep in place: the PHY says whether the core came back to D0, and the GUID register + * whether it kept its registers. A core set up again at the resume kept its host and devices, + * which reset-resumed. Otherwise restart the host as after a stop, and the devices enumerate + * again. A core that may be unpowered gets another power cycle first. Returns whether the + * host was kept. + */ +static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 guid = 0; + int ret; + + appledwc->sleep_in_place = false; + ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + if (ret == -ENODEV) { + dwc3_apple_recover(appledwc); + return false; + } + if (appledwc->sleep_reinit) { + appledwc->sleep_reinit = false; + dev_dbg(appledwc->dev, "host kept through the sleep, set up again\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + if (!ret || ret == -EBUSY || ret == APPLE_ATCPHY_SLEEP_POWER_LOST) { + guid = dwc3_readl(dwc, DWC3_GUID); + if (guid == DWC3_APPLE_SLEEP_GUID) { + dwc3_writel(dwc, DWC3_GUID, appledwc->sleep_guid); + dev_dbg(appledwc->dev, "host kept through the sleep\n"); + dwc3_apple_check_hub_later(appledwc); + return true; + } + ret = -EIO; + } + + dev_warn(appledwc->dev, "host lost its state in the sleep (%d, GUID %#x): restarting it\n", + ret, guid); + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after sleep, err=%d\n", ret); + else + dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + + return false; +} + +/* + * Runs before the freezer, when the sleep's real target isn't known yet, so this goes by the + * state "mem" means now. Writing "freeze" with deep selected costs a USB restart and nothing else. + * A stop that fails refuses the sleep. PM_POST_SUSPEND follows an abandoned sleep as well. + */ +static int dwc3_apple_pm_notify_locked(struct dwc3_apple *appledwc, unsigned long action, + bool in_place, bool *finish, bool *kept) +{ + struct device *dev = appledwc->dev; + int ret; + + lockdep_assert_held(&appledwc->lock); + + switch (action) { + case PM_SUSPEND_PREPARE: + /* Any sleep: xHCI's suspend would touch a core that may be unpowered. */ + if (appledwc->sleep_dead && !appledwc->shutting_down && + !dwc3_apple_recover(appledwc)) { + dev_err(dev, "refusing the sleep: the USB host is still unpowered\n"); + return notifier_from_errno(-EIO); + } + if (pm_suspend_default_s2idle() || appledwc->shutting_down) { + *finish = in_place; + *kept = true; + break; + } + /* xHCI suspends and resumes as usual, and the wakeup setting stays. */ + if (in_place) { + appledwc->sleep_guid = dwc3_readl(&appledwc->dwc, DWC3_GUID); + dwc3_writel(&appledwc->dwc, DWC3_GUID, DWC3_APPLE_SLEEP_GUID); + appledwc->sleep_in_place = true; + /* For the port controllers' USB wake. Changes after this go unseen. */ + apple_sleep_usb_set_wake_ports(dwc3_apple_hub_wake_ports(appledwc)); + break; + } + appledwc->sleep_state = appledwc->state; + ret = dwc3_apple_exit(appledwc); + if (ret) + return notifier_from_errno(ret); + /* A wakeup path would keep the domain on, and it's going away. */ + appledwc->sleep_wakeup = device_may_wakeup(dev); + device_set_wakeup_enable(dev, false); + appledwc->sleep_stopped = true; + break; + case PM_POST_SUSPEND: + if (appledwc->sleep_in_place) { + apple_sleep_usb_set_wake_ports(0); + *kept = dwc3_apple_sleep_kept(appledwc); + *finish = true; + break; + } + if (!appledwc->sleep_stopped) + break; + appledwc->sleep_stopped = false; + device_set_wakeup_enable(dev, appledwc->sleep_wakeup); + if (appledwc->shutting_down || + (appledwc->sleep_state != DWC3_APPLE_HOST && + appledwc->sleep_state != DWC3_APPLE_DEVICE)) + break; + dwc3_apple_start_after_sleep(appledwc, appledwc->sleep_state); + break; + default: + break; + } + + return NOTIFY_DONE; +} + +/* + * The PHY's sleep prepare and finish may take the SuperSpeed port down and up, under the + * controller's device lock, so they run outside the glue's lock. + */ +static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct dwc3_apple *appledwc = container_of(nb, struct dwc3_apple, pm_nb); + bool in_place = false, finish = false, kept = false; + int ret; + + /* + * The last wake's SuperSpeed restore ends before this sleep looks at the PHY. The hub + * check takes the glue's lock, and has nothing to look at until the sleep is over. + */ + if (action == PM_SUSPEND_PREPARE) { + apple_atcphy_sleep_finish_flush(appledwc->usb2_phy); + cancel_delayed_work_sync(&appledwc->hub_check); + } + if (action == PM_SUSPEND_PREPARE && !pm_suspend_default_s2idle()) { + scoped_guard(mutex, &appledwc->lock) + in_place = appledwc->fixed_hub && appledwc->state == DWC3_APPLE_HOST && + !appledwc->shutting_down && !appledwc->sleep_dead; + in_place = in_place && !apple_atcphy_sleep_prepare(appledwc->usb2_phy); + } + + scoped_guard(mutex, &appledwc->lock) + ret = dwc3_apple_pm_notify_locked(appledwc, action, in_place, &finish, &kept); + + /* + * After a wake, user space doesn't wait for SuperSpeed: it comes back in the background, + * from the PHY's work, which an unbind of this device doesn't wait for under its lock. + */ + if (finish && action == PM_POST_SUSPEND) + apple_atcphy_sleep_finish_async(appledwc->usb2_phy, kept); + else if (finish) + apple_atcphy_sleep_finish(appledwc->usb2_phy, kept); + + return ret; +} + +static void dwc3_apple_pm_notifier_remove(void *data) +{ + struct dwc3_apple *appledwc = data; + + unregister_pm_notifier(&appledwc->pm_nb); +} + +/* A fixed hub's node gives the root port it sits on in its reg, 1 if it doesn't. */ +static bool dwc3_apple_has_fixed_hub(struct device *dev, u32 *port) { struct device_node *hub __free(device_node) = of_get_available_child_by_name(dev->of_node, "hub"); + *port = 1; + if (hub) + of_property_read_u32(hub, "reg", port); + return hub; } @@ -924,9 +1435,10 @@ static int dwc3_apple_probe(struct platform_device *pdev) return -ENOMEM; appledwc->dev = &pdev->dev; - appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev); + appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev, &appledwc->hub_port); appledwc->host_only = dwc3_apple_host_only(dev); appledwc->dt_dis_u3_susphy = device_property_read_bool(dev, "snps,dis_u3_susphy_quirk"); + appledwc->sleep_ddr = device_property_read_bool(dev, "apple,sleep-ddr"); /* The PHY driver and the DARTs follow runtime PM on these SoCs. */ appledwc->idle_rpm = of_device_is_compatible(dev->of_node, "apple,t6040-dwc3") || of_device_is_compatible(dev->of_node, "apple,t8132-dwc3"); @@ -995,6 +1507,16 @@ static int dwc3_apple_probe(struct platform_device *pdev) * details. */ appledwc->state = DWC3_APPLE_PROBE_PENDING; + ret = devm_delayed_work_autocancel(dev, &appledwc->hub_check, dwc3_apple_hub_check); + if (ret) + return ret; + appledwc->pm_nb.notifier_call = dwc3_apple_pm_notify; + ret = register_pm_notifier(&appledwc->pm_nb); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dwc3_apple_pm_notifier_remove, appledwc); + if (ret) + return ret; /* * Until the first cable there is no core to keep powered: let the controller * runtime-suspend, and its DARTs with it, so that its power domain can drop. @@ -1044,6 +1566,7 @@ static void dwc3_apple_remove(struct platform_device *pdev) guard(mutex)(&appledwc->lock); + appledwc->shutting_down = true; usb_role_switch_unregister(appledwc->role_sw); /* diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 89e1c0c492dbe..652942d3f6f4f 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -79,8 +79,20 @@ static void dwc3_xhci_plat_start(struct usb_hcd *hcd) dwc3_enable_susphy(dwc, true); } +static int dwc3_xhci_plat_resume_quirk(struct usb_hcd *hcd) +{ + struct platform_device *pdev = to_platform_device(hcd->self.controller); + struct dwc3 *dwc = dev_get_drvdata(pdev->dev.parent); + + if (!usb_hcd_is_primary_hcd(hcd) || !dwc->glue_ops || !dwc->glue_ops->pre_host_resume) + return 0; + + return dwc->glue_ops->pre_host_resume(dwc); +} + static const struct xhci_plat_priv dwc3_xhci_plat_quirk = { .plat_start = dwc3_xhci_plat_start, + .resume_quirk = dwc3_xhci_plat_resume_quirk, }; static void dwc3_host_fill_xhci_irq_res(struct dwc3 *dwc, diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index dd1e3ef800e31..5ded59ac0f54d 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -636,6 +636,7 @@ static int xhci_plat_resume_common(struct device *dev, bool power_lost) struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_plat_priv *priv = hcd_to_xhci_priv(hcd); struct xhci_hcd *xhci = hcd_to_xhci(hcd); + bool lost; int ret; if (!device_may_wakeup(dev) && (xhci->quirks & XHCI_SUSPEND_RESUME_CLKS)) { @@ -650,11 +651,18 @@ static int xhci_plat_resume_common(struct device *dev, bool power_lost) } } + /* A positive result: the controller lost its state, which the quirk set up again. */ ret = xhci_priv_resume_quirk(hcd); - if (ret) + if (ret < 0) goto disable_clks; - ret = xhci_resume(xhci, power_lost || priv->power_lost, false); + lost = power_lost || priv->power_lost || ret > 0; + ret = xhci_resume(xhci, lost, false); + /* As after a restore error: a controller the quirk readied starts over if it can't. */ + if (ret && !lost && priv->resume_quirk) { + dev_warn(dev, "xHC state not restored: %d, starting over\n", ret); + ret = xhci_resume(xhci, true, false); + } if (ret) goto disable_clks; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index fed6ea4097662..c49b175e61715 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -1591,9 +1593,183 @@ static void cd321x_prepare(struct tps6598x *tps) cd321x_pm_prepare(&cd321x->pm); cd321x->state_valid = false; } - /* Gate scheduling first; neither worker may be drained under its mutex. */ + /* Gate scheduling first; no worker may be drained under its mutex. */ cancel_delayed_work_sync(&cd321x->resume_work); cancel_delayed_work_sync(&cd321x->update_work); + cancel_delayed_work_sync(&cd321x->rearm_work); + + /* As macOS does a second or so before the commit, measured on the same controller. */ + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) { + int ret; + + guard(mutex)(&tps->lock); + ret = tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S3); + if (ret) + dev_warn(tps->dev, "didn't take system power state S3: %d\n", ret); + else + dev_dbg(tps->dev, "system power state S3\n"); + } +} + +/* A try every 250 ms for 10 s */ +#define CD321X_REARM_TRIES 40 +#define CD321X_REARM_RETRY_MS 250 + +static int cd321x_debug_cmd(struct tps6598x *tps, const char *cmd, size_t len, const u8 *in) +{ + u8 out = 0; + int ret; + + ret = tps6598x_exec_cmd(tps, cmd, len, in, 1, &out); + return ret ?: (out & 0xf ? -EIO : 0); +} + +/* The firmware rejects a command for the mode it is switching to until it has switched. */ +static int cd321x_wait_dbma(struct tps6598x *tps, bool on) +{ + u32 mode; + int ret, err; + + err = read_poll_timeout(tps6598x_read32, ret, + ret || (!memcmp(&mode, modes[TPS_MODE_DBMA], 4) == on), + 1000, 100000, false, tps, TPS_REG_MODE, &mode); + return ret ?: err; +} + +/* + * The boot loader's sequence: unlock with the board's key, debug mode on, enable the debug + * VDM, debug mode off, lock. Debug mode is left and the lock set again after a failure too. + */ +static int cd321x_arm_debug_link(struct tps6598x *tps, const char *key, const char **step) +{ + static const u8 vdm[] = { 0x06, 0x46, 0x82, 0x01 }; + static const u8 no_key[4]; + u8 on = 1, off = 0; + int ret, err; + + *step = "unlock"; + ret = cd321x_debug_cmd(tps, "LOCK", sizeof(no_key), (const u8 *)key); + if (ret) + return ret; + + *step = "debug mode"; + ret = cd321x_debug_cmd(tps, "DBMa", 1, &on); + if (!ret) + ret = cd321x_wait_dbma(tps, true); + if (!ret) { + *step = "debug VDM"; + ret = cd321x_debug_cmd(tps, "DVEn", sizeof(vdm), vdm); + } + + err = cd321x_debug_cmd(tps, "DBMa", 1, &off); + if (!err) + err = cd321x_wait_dbma(tps, false); + if (err && !ret) { + *step = "leaving debug mode"; + ret = err; + } + err = cd321x_debug_cmd(tps, "LOCK", sizeof(no_key), no_key); + if (err && !ret) { + *step = "lock"; + ret = err; + } + return ret; +} + +/* + * A sleep cut this port's debug link. Right after the wake the controller can reject the + * sequence, so a failed try is repeated until the link is back. + */ +static void cd321x_rearm_debug_link(struct cd321x *cd321x) +{ + struct tps6598x *tps = &cd321x->tps; + const char *key, *step; + int ret; + + lockdep_assert_held(&tps->lock); + if (device_property_read_string(tps->dev, "apple,debug-lock-key", &key) || + strlen(key) != 4) { + cd321x->rearm_pending = false; + return; + } + + ret = cd321x_arm_debug_link(tps, key, &step); + cd321x->rearm_tries++; + if (!ret) { + if (cd321x->rearm_tries > 1) + dev_info(tps->dev, "debug link rearmed, try %u\n", cd321x->rearm_tries); + cd321x->rearm_pending = false; + return; + } + if (cd321x->rearm_tries < CD321X_REARM_TRIES) { + mod_delayed_work(system_freezable_wq, &cd321x->rearm_work, + msecs_to_jiffies(CD321X_REARM_RETRY_MS)); + return; + } + dev_warn(tps->dev, "debug link not rearmed after %u tries: %s: %d\n", + cd321x->rearm_tries, step, ret); + cd321x->rearm_pending = false; +} + +/* The cut's cold reset, and how it waits for the contract to come back, see apple_sleep_usb_down(). */ +#define CD321X_REARM_RESET_AFTER 4 +#define CD321X_REARM_RESETS 3 +#define CD321X_RESET_MS 1000 +#define CD321X_CONTRACT_MS 5000 + +static bool cd321x_has_contract(struct tps6598x *tps) +{ + u16 pwr; + + return !tps6598x_read16(tps, TPS_REG_POWER_STATUS, &pwr) && + TPS_POWER_STATUS_CONNECTION(pwr) && + TPS_POWER_STATUS_PWROPMODE(pwr) == TYPEC_PWR_MODE_PD; +} + +/* + * The debug VDM needs a PD contract. After a cut whose sleep was refused, the partner was + * found to have dropped it, both ends at default USB power, and no try could rearm the link. + * Do what the cut does when the contract doesn't come back: reset the controller again and + * wait for it. The controller then needs setting up as after the cut. + */ +static void cd321x_rearm_recover_contract(struct cd321x *cd321x) +{ + struct tps6598x *tps = &cd321x->tps; + int ms, ret; + + lockdep_assert_held(&tps->lock); + cd321x->rearm_resets++; + dev_warn(tps->dev, "debug link: no contract with the partner, reset %u\n", + cd321x->rearm_resets); + ret = tps6598x_exec_cmd_tmo(tps, "Gaid", 0, NULL, 0, NULL, 2000, 0); + if (ret) + dev_dbg(tps->dev, "reset command: %d\n", ret); + msleep(CD321X_RESET_MS); + for (ms = 0; ms < CD321X_CONTRACT_MS && !cd321x_has_contract(tps); ms += 50) + msleep(50); + if (ms < CD321X_CONTRACT_MS) + dev_info(tps->dev, "debug link: contract back %d ms after reset %u\n", + CD321X_RESET_MS + ms, cd321x->rearm_resets); + cd321x->pm.setup_pending = true; + cd321x->reset_by_cut = true; + cd321x->state_valid = false; + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + cd321x->rearm_tries = 0; +} + +static void cd321x_rearm_work(struct work_struct *work) +{ + struct cd321x *cd321x = container_of(to_delayed_work(work), struct cd321x, rearm_work); + struct tps6598x *tps = &cd321x->tps; + + guard(mutex)(&tps->lock); + if (!cd321x->rearm_pending || cd321x->pm.phase == CD321X_PM_PREPARED || + cd321x->pm.phase == CD321X_PM_REMOVED) + return; + if (cd321x->rearm_tries >= CD321X_REARM_RESET_AFTER && + cd321x->rearm_resets < CD321X_REARM_RESETS && !cd321x_has_contract(tps)) + cd321x_rearm_recover_contract(cd321x); + cd321x_rearm_debug_link(cd321x); } static void cd321x_complete(struct tps6598x *tps) @@ -1601,6 +1777,27 @@ static void cd321x_complete(struct tps6598x *tps) struct cd321x *cd321x = container_of(tps, struct cd321x, tps); guard(mutex)(&tps->lock); + /* + * The cut was a cold reset of this controller's firmware: its interrupt mask and power + * state are back at their defaults, so set them up again whether or not an event comes. + */ + if (apple_sleep_debug_link_cut_peek(tps->dev)) { + cd321x->pm.setup_pending = true; + cd321x->reset_by_cut = true; + } + if (pm_suspend_via_firmware() && device_property_read_bool(tps->dev, "apple,sleep-ssps")) + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + /* + * A link still down from an earlier sleep isn't cut again, so it stays pending. A dark + * wake sleeps again at once: the link stays down for it, and the full wake rearms it. + */ + if (apple_sleep_debug_link_was_cut(tps->dev)) + cd321x->rearm_pending = true; + if (cd321x->rearm_pending && !pm_dark_wake()) { + cd321x->rearm_tries = 0; + cd321x->rearm_resets = 0; + cd321x_rearm_debug_link(cd321x); + } if (cd321x_pm_resume(&cd321x->pm)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } @@ -1670,8 +1867,11 @@ static int cd321x_check_setup(struct cd321x *cd321x, u64 event) dev_err(tps->dev, "controller setup recovery exhausted: %d\n", ret); return ret; } - if (ret) + if (ret && cd321x->reset_by_cut) + dev_dbg(tps->dev, "set up again after the debug link's cut\n"); + else if (ret) dev_warn(tps->dev, "restored controller power state and interrupt mask\n"); + cd321x->reset_by_cut = false; /* Only a fresh snapshot may be applied after restoring the controller. */ cd321x->state_valid = false; cd321x->pm.phase = CD321X_PM_REVALIDATE; @@ -2222,6 +2422,7 @@ VISIBLE_IF_KUNIT void cd321x_stop_work(struct cd321x *cd321x) } cancel_delayed_work_sync(&cd321x->resume_work); cancel_delayed_work_sync(&cd321x->update_work); + cancel_delayed_work_sync(&cd321x->rearm_work); } EXPORT_SYMBOL_IF_KUNIT(cd321x_stop_work); @@ -2352,6 +2553,7 @@ VISIBLE_IF_KUNIT void cd321x_init_work(struct cd321x *cd321x) { INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); + INIT_DELAYED_WORK(&cd321x->rearm_work, cd321x_rearm_work); INIT_WORK(&cd321x->tbt_attach_work, cd321x_tbt_attach_work); atomic_set(&cd321x->link_event, 0); atomic_set(&cd321x->ready_event, 0); @@ -3233,6 +3435,87 @@ void tipd_remove(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_remove); +/* + * macOS tells the port controller of a USB root port with dock-remote-wake when that port + * sleeps, and the controller then watches the USB2 lines for a remote wakeup (static; what its + * firmware does with it isn't known). The J700's root port has none, but measured on a J700, + * a USB keyboard on a connector wakes it this way. The mode goes by the speed on the port: 0x09 + * for full and high speed. At the wake the controller hands the lines back before the port + * resumes. Only a firmware-driven sleep needs this, and a failure only logs. + */ +#define TIPD_USBW_HIGH_SPEED 0x09 +#define TIPD_USBW_HAND_BACK 0x80 +#define TIPD_USBW_OFF 0x00 + +static void tipd_usbw(struct tps6598x *tps, u8 mode) +{ + int ret; + + guard(mutex)(&tps->lock); + ret = tps6598x_exec_cmd(tps, "USBw", sizeof(mode), &mode, 0, NULL); + if (ret) + dev_warn(tps->dev, "USBw %#x failed: %d\n", mode, ret); + else + dev_dbg(tps->dev, "USBw %#x\n", mode); +} + +/* The fixed hub's port that the connector's USB2 pair goes to, from the graph, or 0. */ +static u32 tipd_hub_port(struct tps6598x *tps) +{ + struct fwnode_handle *con __free(fwnode_handle) = + device_get_named_child_node(tps->dev, "connector"); + struct fwnode_handle *ep __free(fwnode_handle) = + fwnode_graph_get_endpoint_by_id(con, 0, 0, 0); + struct fwnode_handle *port __free(fwnode_handle) = fwnode_graph_get_remote_port(ep); + u32 reg = 0; + + fwnode_property_read_u32(port, "reg", ®); + return reg; +} + +void tipd_usb_wake(struct tps6598x *tps, bool asleep) +{ + bool armed; + + if (!pm_suspend_via_firmware() || + !device_property_read_bool(tps->dev, "apple,usb-remote-wake")) + return; + if (asleep) { + u32 port = tipd_hub_port(tps); + bool wake = apple_sleep_usb_port_can_wake(port); + + /* + * Only for a device on this connector that can wake. Armed without one, the + * controller reported a USB2 plug at the sleep's entry and the sleep came straight + * back, on an empty port and on one left with a passive adapter (measured on a + * J700). A device plugged in during the sleep still wakes through the SMC's plug + * event. + */ + if (wake) + dev_dbg(tps->dev, "a device on hub port %u can wake: arming USB wake\n", + port); + else + dev_dbg(tps->dev, "nothing on hub port %u can wake: USB wake not armed\n", + port); + scoped_guard(mutex, &tps->lock) + tps->usbw_armed = wake; + if (wake) + tipd_usbw(tps, TIPD_USBW_HIGH_SPEED); + return; + } + /* Out of S3 first: .complete would only come once the port has resumed. */ + scoped_guard(mutex, &tps->lock) { + tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + armed = tps->usbw_armed; + tps->usbw_armed = false; + } + if (!armed) + return; + tipd_usbw(tps, TIPD_USBW_HAND_BACK); + tipd_usbw(tps, TIPD_USBW_OFF); +} +EXPORT_SYMBOL_GPL(tipd_usb_wake); + int tipd_suspend(struct tps6598x *tps) { if (tps->wakeup) { diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 30d472c3d83a4..bbe2ef971b35f 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -855,8 +855,11 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) } else { err = sn201202x_wake(sn); } - if (!err) + if (!err) { + /* Registered before xHCI, so this is before the root port resumes. */ + tipd_usb_wake(tps, false); err = tipd_resume(tps); + } /* The resume revalidation acknowledges events latched while asleep. */ if (tps->irq) enable_irq(tps->irq); @@ -906,6 +909,8 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) else dev_warn(dev, "port state unreadable, no wakeup for an attach\n"); } + /* Registered before xHCI, so its root port has suspended by now. */ + tipd_usb_wake(tps, true); reinit_completion(&sn->sleep_completion); err = spmi_command_sleep(sn->sdev); if (err) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 20b6b73436a29..a5512eea4205f 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -395,6 +395,8 @@ struct tps6598x { struct tipd_debugfs *debugfs; struct typec_altmode *partner_altmode_dp; + /* USB remote wake armed on this port for the sleep (USBw). */ + bool usbw_armed; }; /* TPS_REG_USB4_STATUS */ @@ -466,12 +468,20 @@ struct cd321x { atomic_t ready_event; atomic_t restore_event; bool display_route_active; + /* The sleep's debug link cut reset this controller: its setup is expected. */ + bool reset_by_cut; + /* The debug link is still to be rearmed after a cut, and the tries so far */ + bool rearm_pending; + unsigned int rearm_tries; + /* Resets taken since the cut for a contract the partner dropped, see cd321x_rearm_work() */ + unsigned int rearm_resets; bool dp_sink_reported; bool dp_irq_hpd; struct cd321x_status update_status; struct delayed_work update_work; struct delayed_work resume_work; + struct delayed_work rearm_work; struct cd321x_pm_state pm; struct usb_pd_identity cur_partner_identity; bool cur_partner_is_pd; @@ -522,6 +532,7 @@ struct cd321x_sleep_state { }; int cd321x_sleep_events(struct tps6598x *tps, struct cd321x_sleep_state *st); +void tipd_usb_wake(struct tps6598x *tps, bool asleep); int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, size_t in_len, const u8 *in_data, size_t out_len, u8 *out_data, diff --git a/include/linux/irqchip/irq-apple-aic.h b/include/linux/irqchip/irq-apple-aic.h new file mode 100644 index 0000000000000..3c3781322a52b --- /dev/null +++ b/include/linux/irqchip/irq-apple-aic.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_IRQCHIP_IRQ_APPLE_AIC_H +#define _LINUX_IRQCHIP_IRQ_APPLE_AIC_H + +#include + +#ifdef CONFIG_APPLE_AIC +void aic_sleep_mask_all(void); +void aic_sleep_restore(void); +u32 aic_sleep_ack(void); +#else +static inline void aic_sleep_mask_all(void) { } +static inline void aic_sleep_restore(void) { } +static inline u32 aic_sleep_ack(void) { return 0; } +#endif + +#endif diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index 604c54b0b3274..32a33410b4d2f 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -112,6 +112,17 @@ struct apple_smc { struct apple_rtkit *rtk; + /* Between the AP power notices of a firmware-driven suspend. */ + bool ap_notified; + /* Told of a coming sleep, and not yet of its end. */ + bool sleep_warned; + /* To be told at the noirq suspend, after the debug link's cut. */ + bool sleep_warn_late; + /* In a noirq phase: replies are polled for. */ + bool polled; + /* MBSW at the last wake: what the SMC had woken the AP for. */ + u64 wake_record; + struct completion init_done; enum apple_smc_boot_stage boot_stage; diff --git a/include/linux/phy/apple-atc.h b/include/linux/phy/apple-atc.h new file mode 100644 index 0000000000000..7c2e32a3c13fe --- /dev/null +++ b/include/linux/phy/apple-atc.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What the Apple Type-C PHY offers the USB controller it serves, beyond the + * generic PHY operations. + */ +#ifndef _LINUX_PHY_APPLE_ATC_H_ +#define _LINUX_PHY_APPLE_ATC_H_ + +#include +#include + +struct phy; + +/* In D0 after the sleep, but its always-on side lost power, so xHCI's saved state is gone. */ +#define APPLE_ATCPHY_SLEEP_POWER_LOST 1 + +#if IS_REACHABLE(CONFIG_PHY_APPLE_ATC) +/* + * For a firmware-driven system sleep, before any device suspends and without the + * controller's locks held. 0 means the PHY takes the controller through hibernation after + * xHCI has saved its state, then its DARTs and power domains down, and brings all of it + * back before xHCI resumes, so the host can stay up; with USB3 its PIPE is parked for it. + * Otherwise the controller has to be stopped for the sleep. + */ +int apple_atcphy_sleep_prepare(struct phy *usb2_phy); + +/* + * After the sleep: 0 if the controller is in D0 and its always-on side kept power, or + * APPLE_ATCPHY_SLEEP_POWER_LOST. -ENODEV if it may be unpowered, so that its registers must + * not be touched. Any other error: it's in D0, but the PHY isn't set up for it. + */ +int apple_atcphy_sleep_result(struct phy *usb2_phy); + +/* + * For a controller that may be unpowered after a sleep: power cycles it, as macOS's Off + * and cold start do. APPLE_ATCPHY_SLEEP_POWER_LOST once it's back, reset, or -ENODEV. + */ +int apple_atcphy_sleep_recover(struct phy *usb2_phy); + +/* + * Ends a sleep that prepare took on, with the SuperSpeed side back as the mux wants it. + * Without @kept the host was restarted; only a failure of the PHY's own then makes the + * controller stop for later sleeps. + */ +void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept); + +/* The same from the PHY's work, for after a wake; the flush waits for it. */ +void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept); +void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy); +#else +static inline int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline int apple_atcphy_sleep_result(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline int apple_atcphy_sleep_recover(struct phy *usb2_phy) +{ + return -EOPNOTSUPP; +} + +static inline void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ +} + +static inline void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept) +{ +} + +static inline void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy) +{ +} +#endif + +#endif diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index e594a74e02573..f6f27b4e0e152 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -156,6 +156,12 @@ int apple_dart_restore_tunnel_state(struct device *dev); int apple_dart_radio_check_idle(struct device *dev); /* A firmware-owned buffer's address for a host copy; Linux's own mappings refused. */ int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, phys_addr_t *phys); +/* + * A DART clocked by its host's port: after a real sleep it stays gated until the host has + * set the port up and calls apple_dart_host_resume(). + */ +int apple_dart_set_host_resumes(struct device *dev); +int apple_dart_host_resume(struct device *dev); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } @@ -167,6 +173,68 @@ static inline int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool { return -EOPNOTSUPP; } +static inline int apple_dart_set_host_resumes(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_host_resume(struct device *dev) { return -EOPNOTSUPP; } +#endif + +/* For a consumer whose firmware gates itself around a firmware-driven suspend. */ +#if IS_REACHABLE(CONFIG_APPLE_DART) +bool apple_dart_sleep_listed(struct device *dev); +int apple_dart_sleep_prepare(struct device *dev); +void apple_dart_sleep_restore(struct device *dev); +/* A display DART's streams around a display power cycle, see the DART driver. */ +int apple_dart_display_streams(struct device *dev, u32 streams, bool on, bool flush); +/* A GAPF slice released before its coprocessor's CPU domain comes up after a wake. */ +int apple_dart_gapf_ready(struct device *dev); +/* A display DART whose domain goes off at runtime while others still map into it. */ +void apple_dart_power_dark(struct device *dev, bool dark); +/* A DART's state kept across its consumer's own power cycle, outside a system sleep. */ +int apple_dart_state_save(struct device *dev); +int apple_dart_state_restore(struct device *dev); +void apple_dart_sleep_restore_lost(struct device *dev, bool lost); +#else +static inline bool apple_dart_sleep_listed(struct device *dev) +{ + return false; +} + +static inline int apple_dart_sleep_prepare(struct device *dev) +{ + return -ENODEV; +} + +static inline void apple_dart_sleep_restore(struct device *dev) +{ +} + +static inline int apple_dart_display_streams(struct device *dev, u32 streams, bool on, + bool flush) +{ + return -ENODEV; +} + +static inline int apple_dart_gapf_ready(struct device *dev) +{ + return 0; +} + +static inline void apple_dart_power_dark(struct device *dev, bool dark) +{ +} + +static inline int apple_dart_state_save(struct device *dev) +{ + return -ENODEV; +} + +static inline int apple_dart_state_restore(struct device *dev) +{ + return -ENODEV; +} + +static inline void apple_dart_sleep_restore_lost(struct device *dev, bool lost) +{ +} #endif /* diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index d7fc9dd4134f1..dd6ae8fef53db 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -100,8 +100,15 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); +/* For a caller with interrupts off, where apple_mbox_send()'s own deadline can't pass. */ +bool apple_mbox_send_ready(struct apple_mbox *mbox); int apple_mbox_restore(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* Narrow no-post-callback-MMIO receive; see provider's supported transport guard. */ +bool apple_mbox_poll_one_supported(struct apple_mbox *mbox); +int apple_mbox_poll_one(struct apple_mbox *mbox); +/* Bit 0 of the receive control word, for firmware that powers itself down. */ +void apple_mbox_set_recv_enabled(struct apple_mbox *mbox, bool enabled); /* * Leave received messages in the hardware FIFO, without an interrupt, until * apple_mbox_release_rx(). For consumers that must not wake the system. @@ -123,6 +130,14 @@ int apple_mbox_ascwrap_v6_get_lifecycle( int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); +/* + * For a CPU that a system sleep's last step stops with interrupts off: its control word, for + * polling its status, and the stop's register writes alone. Once the sleep is over, + * apple_mbox_ascwrap_v6_cpu_stopped() settles the books without touching the registers. + */ +void __iomem *apple_mbox_ascwrap_v6_cpu_control(struct apple_mbox *mbox); +void apple_mbox_ascwrap_v6_stop_cpu_atomic(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_cpu_stopped(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, u32 *control, u32 *status); diff --git a/include/linux/soc/apple/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h index 3b2971b8b1db4..3a496a2040989 100644 --- a/include/linux/soc/apple/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -5,16 +5,30 @@ #include #include +struct module; struct pci_dev; #if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) /* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); +/* A function that a sleep left unusable asks for both to be restarted from cold. */ +void apple_piodma_radio_recover(void); +/* What the host calls before it removes Wi-Fi for that. A failure keeps it bound. */ +void __apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev), struct module *owner); +#define apple_piodma_radio_set_stop(stop) __apple_piodma_radio_set_stop(stop, THIS_MODULE) #else static inline int apple_piodma_radio_check(struct pci_dev *pdev) { return -ENODEV; } + +static inline void apple_piodma_radio_recover(void) +{ +} + +static inline void apple_piodma_radio_set_stop(int (*stop)(struct pci_dev *pdev)) +{ +} #endif #endif /* _LINUX_SOC_APPLE_PCI_PIODMA_H */ diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 66e97565edc1e..77fed7a9e7edd 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -24,6 +24,11 @@ int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); +void apple_pmp_report_sleep(struct device *pmp, bool at_commit); +void apple_pmp_report_wake(struct device *pmp, bool lost_power); +int apple_pmp_report_entry_request(const struct device_node *entry, bool enable); +int apple_pmp_report_entry_commit_owned(const struct device_node *entry, bool owned); +int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry); /* * Set or clear one bit of the PMP power request word of the report bound to @report_np, under the * report's lock, with the acknowledge wait (see pmp-report.c). @@ -40,6 +45,31 @@ void apple_pmp_agx_activity_end(void); void apple_pmp_agx_activity_release(void); void apple_pmp_agx_activity_resume(void); #else +static inline int apple_pmp_report_entry_request(const struct device_node *entry, + bool enable) +{ + return 0; +} + +static inline int apple_pmp_report_entry_commit_owned(const struct device_node *entry, + bool owned) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_entry_withdraw_polled(const struct device_node *entry) +{ + return 0; +} + +static inline void apple_pmp_report_sleep(struct device *pmp, bool at_commit) +{ +} + +static inline void apple_pmp_report_wake(struct device *pmp, bool lost_power) +{ +} + static inline int apple_pmp_report_ane_available(const struct device_node *np) { return -EOPNOTSUPP; diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 8b329bf5d4dd5..8663ca195febe 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -5,6 +5,7 @@ #include #include struct device; +struct dev_pm_ops; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); @@ -25,8 +26,27 @@ static inline int apple_pmp_t6030_gpu_ready(void) } #endif /* Registration is private to the PMP Rust driver and its C bridge. */ +bool apple_pmp_keep_bound(void); int apple_pmp_register(struct device *dev, const void *data); void apple_pmp_unregister(const void *data); int apple_pmp_send_power_command(const void *data, u8 command, u16 device_id, u32 enabled); +extern const struct dev_pm_ops apple_pmp_pm_ops; +int apple_pmp_sleep(const void *data); +int apple_pmp_wake(const void *data); + +/* + * What the sleep's last step needs to idle and stop the PMP with interrupts off: its RTKit and + * CPU control, and where the SRAM it keeps goes (no range: sram NULL). Filled in suspend_late. + */ +struct apple_rtkit; +struct apple_pmp_commit_stop { + struct apple_rtkit *rtk; + void __iomem *control; + void __iomem *sram; + u32 *words; + size_t count; +}; + +int apple_pmp_sleep_prepare(const void *data, struct apple_pmp_commit_stop *stop); #endif diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index ac646240f8587..44dcb386efd76 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -215,6 +215,9 @@ void apple_rtkit_set_handshake_timeout(struct apple_rtkit *rtk, */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* apple_rtkit_boot(), calling @before_ap between the IOP's report and the AP going on. */ +int apple_rtkit_boot_with(struct apple_rtkit *rtk, void (*before_ap)(void *data), void *data); + /* * Mark an instance created with apple_rtkit_init() as running without a * handshake, see apple_rtkit_init_adopted(). @@ -259,6 +262,92 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk); */ int apple_rtkit_idle(struct apple_rtkit *rtk); +/* + * Turn the mailbox off for a co-processor that has acknowledged idle, before its + * run bit and power state go. Only apple_rtkit_warm_restart() follows this. + */ +void apple_rtkit_mailbox_off(struct apple_rtkit *rtk); + +/* + * The same with interrupts off and one CPU up, for a co-processor that doesn't come + * back from idle without losing power, so that its idle has to follow every refusal. + * @us is about how long the two acknowledgements took. Only + * apple_rtkit_warm_restart() follows this, once the sleep has reached the hardware. + */ +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us); + +/* + * Only the mailbox's receive bit, for the moment before that co-processor's run bit + * goes. Safe with interrupts off, and apple_rtkit_warm_restart() sets it again. + */ +void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk); + +/* + * Prepare for a firmware being restarted with its buffers kept. + */ +int apple_rtkit_warm_restart(struct apple_rtkit *rtk); + +/* + * For a firmware whose last endpoint map is answered, on a warm restart, with + * only the endpoints in @last_mask rather than an echo of all of them. + */ +void apple_rtkit_set_warm_epmap_mask(struct apple_rtkit *rtk, u32 last_mask); + +/* + * Tell a co-processor that keeps running across a system sleep about the AP's + * power state, without changing its own: idle before the sleep and on after. + */ +int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on); + +/* The same, for a noirq phase: device interrupts are off, so it polls. */ +int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on); + +/* + * Stop a co-processor for a system sleep that may remove its power: idle, wait + * for the status that shows it has finished, then clear the run bit. On an error + * the run bit is still set and the firmware still runs. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_stop(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Bring back a co-processor taken down for a system sleep with + * apple_rtkit_sleep_stop() or apple_rtkit_idle(), from a process context with its + * mailbox's supplier resumed. + * + * Where the sleep reached the hardware (pm_resume_via_firmware()) the firmware + * restarts and keeps its buffers and endpoints: this restarts the receive path, sets + * the run bit, echoes the endpoint map and takes the IOP state it reports unasked, + * then sets the AP state. Where it didn't, one that idled gets -EAGAIN: an idled + * firmware doesn't wake with its power kept, so it stays down, and comes back the + * same way after the next sleep that does reach the hardware. One whose idle + * failed still runs, and gets 0 once its AP state is on again. + * + * @cpu_control: the ASC's CPU control register. + */ +int apple_rtkit_sleep_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * A start as after a real sleep, for a co-processor its driver power-cycled without one: + * the warm restart, the run bit set, then RTKit's handshake. + */ +int apple_rtkit_power_cycled_return(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * The first half of apple_rtkit_sleep_return() after a real sleep: the restart and the run + * bit, with no wait. apple_rtkit_boot() then waits for the firmware to report on, so other + * firmware can be started in between. -EOPNOTSUPP after a sleep that kept the power. + */ +int apple_rtkit_sleep_return_start(struct apple_rtkit *rtk, void __iomem *cpu_control); + +/* + * Echo each OSLOG channel-0 counter the firmware sends, as macOS does. On by default on + * T8140, where a firmware with one outstanding takes no AP power notice. + */ +void apple_rtkit_set_oslog_echo(struct apple_rtkit *rtk, bool echo); + /* * Checks if RTKit is running and ready to handle messages. */ @@ -301,6 +390,21 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * A port that only answers the co-processor, polled, with no interrupts. Map answers + * its HELLO and endpoint map, and on brings the AP side of the port up. Handshake is + * both. Idle takes the AP side down. Restart prepares for a co-processor that was + * stopped. + */ +int apple_rtkit_port_map(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_on(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_handshake(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_idle(struct apple_rtkit *rtk, unsigned int timeout_ms); +int apple_rtkit_port_restart(struct apple_rtkit *rtk); + +/* apple_rtkit_idle(), calling @after_ap between the AP's idle and the IOP's. */ +int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), void *data); + /* * Stop taking interrupts for messages from the co-processor, which then * stay queued in the mailbox until apple_rtkit_resume_rx(). diff --git a/include/linux/soc/apple/sart.h b/include/linux/soc/apple/sart.h index f5d0db5022831..53186e759e55d 100644 --- a/include/linux/soc/apple/sart.h +++ b/include/linux/soc/apple/sart.h @@ -64,4 +64,10 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, size_t size); +/* + * After a firmware-driven sleep, writes the entries in use before it back again and reads + * each back. Returns how many didn't hold, or -EOPNOTSUPP. + */ +int apple_sart_replay(struct apple_sart *sart); + #endif /* _LINUX_SOC_APPLE_SART_H_ */ diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h new file mode 100644 index 0000000000000..91be63b7aa18f --- /dev/null +++ b/include/linux/soc/apple/sleep.h @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * What drivers tell the code that takes an Apple SoC into a firmware-driven + * suspend. + */ +#ifndef _LINUX_SOC_APPLE_SLEEP_H_ +#define _LINUX_SOC_APPLE_SLEEP_H_ + +#include +#include +#include +#include + +struct device; +struct device_node; + +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) +/* + * A coprocessor that can't be stopped until a sleep is certain, and can't be + * restarted without one. While one is registered and not ready, a suspend is + * refused. Once ready, the last step before the commit runs @last_stop, with + * interrupts off and every refusal behind it, then clears its run bit. With + * @pcie_ans, that also lowers the PCIe and ANS domains, which nothing else may do + * under it. @last_stop can't refuse: a nonzero return means it failed, and the sleep + * goes on. + */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data); + +/* + * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power + * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. + */ +int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), + void *data); +void apple_sleep_coproc_unregister(struct device *dev); + +/* + * For a registered coprocessor: the last of its stop that may still refuse the sleep, run + * with interrupts off before any coprocessor's @last_stop. A negative return refuses, and the + * coprocessor must still be able to carry on. + */ +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)); + +/* + * The enclave's irreversible sleep request. @stop runs with interrupts off, + * after every refusal and before other irreversible coprocessor stops. Nonzero + * may refuse ONLY if no request was sent. Once sent, protocol failure is fatal; + * success must continue through physical sleep/wake, even for a newly pending wake. + * Unsupported/no-provider paths refuse before sending, never at device suspend. + */ +int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data); +void apple_sleep_enclave_unregister(struct device *dev); + +/* Its driver has quiesced it for that last step, or has taken it back. */ +void apple_sleep_coproc_set_ready(struct device *dev, bool ready); + +/* Whether the last step stopped it: after a refused sleep, false means it still runs. */ +bool apple_sleep_coproc_stopped(struct device *dev); + +/* + * Whether this sleep's commit stops the registered coprocessors. Then one that can't be + * registered refuses the sleep: stopped any earlier, a later refusal would strand it. + */ +bool apple_sleep_commit_stops_coprocs(void); + +/* + * The USB host sleeps in place for this sleep, as macOS's does: its always-on domain stays + * up through the commit. Its PHY sets it once the controller is in its hibernation state, + * and clears it at the wake. + */ +void apple_sleep_usb_set_asleep(bool asleep); + +/* + * The USB host is up, as macOS's USB_SLP_DDR vote has it: from the host's start to its stop, + * awake and asleep. It sets bit 0 of the wake block's word 0 for every sleep in between. + */ +void apple_sleep_usb_slp_ddr(bool up); + +/* + * The fixed hub's ports with a device at or below them that can wake the system, bit n for + * port n. The host that sleeps in place sets them before the freeze and clears them after. + */ +void apple_sleep_usb_set_wake_ports(unsigned long ports); + +/* + * For a driver that takes its own power domain down for a sleep and up again, as macOS's USB + * host does: @pd is the domain's power controller node. Lowering waits until it reads off. + * 0, -ETIMEDOUT, -ENODEV without the sleep driver, or -EINVAL for another node, which leave + * the domain alone and which apple_sleep_domain_check() reports beforehand. + */ +int apple_sleep_domain_set(struct device_node *pd, bool on); +int apple_sleep_domain_check(struct device_node *pd); + +/* For the PMGR's driver at the wake: a driver took @pd down and brings it up itself. */ +bool apple_sleep_domain_is_down(struct device_node *pd); + +/* For the port controller wired to the fixed hub's @port, as it arms USB wake. */ +bool apple_sleep_usb_port_can_wake(unsigned int port); + +/* The last sleep ended the debug link on @dev, whose driver may rearm it. */ +bool apple_sleep_debug_link_was_cut(struct device *dev); + +/* The same, without taking it: the port controller's own setup after its cold reset. */ +bool apple_sleep_debug_link_cut_peek(struct device *dev); + +/* + * End the debug link now, if this sleep hasn't: true if this call did. For the SMC's driver, + * ahead of its AP idle notice. It takes over a second, and doesn't sleep. + */ +bool apple_sleep_debug_link_cut(void); + +/* This sleep will end the debug link: for the SMC's driver, which warns the SMC after it. */ +bool apple_sleep_debug_link_up(void); + +/* + * From the SMC's driver, in its noirq resume: the SMC woke the AP for the lid alone, and the + * lid reads closed. If nothing else woke the SoC, the PM core goes back to sleep once every + * device has resumed, as macOS does from a dark wake. + */ +void apple_sleep_lid_closed_wake(void); + +/* + * From the SMC's driver, in its noirq resume: the SMC woke the AP for the port alone. If + * this sleep cut the debug link and nothing else woke the SoC, it's a dark wake. + */ +void apple_sleep_debug_link_cut_wake(void); + +/* The radio's host has its port back after a real sleep, so the next commit may stop it. */ +void apple_sleep_radio_port_up(void); + +/* From RTKit: this coprocessor's firmware crashed. */ +void apple_sleep_coproc_crashed(struct device *dev); + +/* After a deep sleep, put back the pins it reset. ANS's driver calls it before the run bit. */ +void apple_sleep_restore_pins(void); + +/* + * After a refused sleep, for a firmware that idled on a domain that can't gate alone: the + * sleep engine's PMGR reset of its own domains, its DARTs kept. The caller starts it after. + */ +int apple_sleep_engine_reset(struct device *dev); +#else +static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, + bool pcie_ans, int (*last_stop)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_coproc_register_first(struct device *dev, + int (*last_stop)(void *data), void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_coproc_unregister(struct device *dev) +{ +} + +static inline int apple_sleep_coproc_set_last_check(struct device *dev, + int (*last_check)(void *data)) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_enclave_unregister(struct device *dev) +{ +} + +static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ +} + +static inline bool apple_sleep_coproc_stopped(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_commit_stops_coprocs(void) +{ + return false; +} + +static inline void apple_sleep_usb_set_asleep(bool asleep) +{ +} + +static inline void apple_sleep_usb_slp_ddr(bool up) +{ +} + +static inline void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ +} + +static inline int apple_sleep_domain_set(struct device_node *pd, bool on) +{ + return -ENODEV; +} + +static inline int apple_sleep_domain_check(struct device_node *pd) +{ + return -ENODEV; +} + +static inline bool apple_sleep_domain_is_down(struct device_node *pd) +{ + return false; +} + +static inline bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + return false; +} + +static inline bool apple_sleep_debug_link_was_cut(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut_peek(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_debug_link_cut(void) +{ + return false; +} + +static inline bool apple_sleep_debug_link_up(void) +{ + return false; +} + +static inline void apple_sleep_lid_closed_wake(void) +{ +} + +static inline void apple_sleep_debug_link_cut_wake(void) +{ +} + +static inline void apple_sleep_radio_port_up(void) +{ +} + +static inline void apple_sleep_coproc_crashed(struct device *dev) +{ +} + +static inline void apple_sleep_restore_pins(void) +{ +} + +static inline int apple_sleep_engine_reset(struct device *dev) +{ + return -EOPNOTSUPP; +} +#endif + +#endif diff --git a/include/linux/suspend.h b/include/linux/suspend.h index b02876f1ae38a..365437ce2cc02 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -159,6 +159,7 @@ extern unsigned int pm_suspend_global_flags; #define PM_SUSPEND_FLAG_FW_SUSPEND BIT(0) #define PM_SUSPEND_FLAG_FW_RESUME BIT(1) #define PM_SUSPEND_FLAG_NO_PLATFORM BIT(2) +#define PM_SUSPEND_FLAG_DARK_WAKE BIT(3) static inline void pm_suspend_clear_flags(void) { @@ -180,6 +181,11 @@ static inline void pm_set_suspend_no_platform(void) pm_suspend_global_flags |= PM_SUSPEND_FLAG_NO_PLATFORM; } +static inline void pm_set_dark_wake(void) +{ + pm_suspend_global_flags |= PM_SUSPEND_FLAG_DARK_WAKE; +} + /** * pm_suspend_via_firmware - Check if platform firmware will suspend the system. * @@ -237,6 +243,27 @@ static inline bool pm_suspend_no_platform(void) return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_NO_PLATFORM); } +/** + * pm_dark_wake - Check if the system woke up for nothing user space has to see. + * + * To be called during system-wide power management transitions from sleep + * states. + * + * Return 'true' if the platform found that the event that woke up the system + * needs nothing from user space (for example, a laptop lid closing while the + * system slept). Every device then resumes as usual, and the system goes back + * to the same sleep state with user space still frozen. A device should not do + * what the user would notice, such as lighting a display, during such a wake. + * + * Only a wake from the sleep state can be dark. A device that had to leave out + * part of its resume finishes it in a later resume that is not dark, or in + * ->complete() if the next suspend fails before suspending it. + */ +static inline bool pm_dark_wake(void) +{ + return !!(pm_suspend_global_flags & PM_SUSPEND_FLAG_DARK_WAKE); +} + /* Suspend-to-idle state machnine. */ enum s2idle_states { S2IDLE_STATE_NONE, /* Not suspended/suspending. */ @@ -287,6 +314,8 @@ static inline void pm_set_resume_via_firmware(void) {} static inline bool pm_suspend_via_firmware(void) { return false; } static inline bool pm_resume_via_firmware(void) { return false; } static inline bool pm_suspend_no_platform(void) { return false; } +static inline void pm_set_dark_wake(void) {} +static inline bool pm_dark_wake(void) { return false; } static inline bool pm_suspend_default_s2idle(void) { return false; } static inline void suspend_set_ops(const struct platform_suspend_ops *ops) {} diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 604b9dda94317..b8bf424370bc7 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -112,6 +112,7 @@ struct usb_hcd { #define HCD_FLAG_DEAD 6 /* controller has died? */ #define HCD_FLAG_INTF_AUTHORIZED 7 /* authorize interfaces? */ #define HCD_FLAG_DEFER_RH_REGISTER 8 /* Defer roothub registration */ +#define HCD_FLAG_PHY_SUSPENDED 9 /* root hub PHYs released by a suspend */ /* The flags can be tested using these macros; they are likely to * be slightly faster than test_bit(). diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 57c44268698f7..34253036bbbd9 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -532,6 +532,9 @@ int suspend_devices_and_enter(suspend_state_t state) do { error = suspend_enter(state, &wakeup); } while (!error && !wakeup && platform_suspend_again(state)); + /* Only a wake from the sleep state can be dark. */ + if (error) + pm_suspend_global_flags &= ~PM_SUSPEND_FLAG_DARK_WAKE; Resume_devices: suspend_test_start(); @@ -616,6 +619,15 @@ static int enter_state(suspend_state_t state) trace_suspend_resume(TPS("suspend_enter"), state, false); pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); error = suspend_devices_and_enter(state); + /* + * Every device has resumed, but nothing woke the system that user space + * needs to see: go back to sleep with user space still frozen. + */ + while (!error && pm_dark_wake()) { + pr_info("dark wake: suspending again (%s)\n", mem_sleep_labels[state]); + pm_suspend_clear_flags(); + error = suspend_devices_and_enter(state); + } Finish: events_check_enabled = false; diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f763d5541472..389c67d5740ed 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -102,6 +102,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "suspend.c" #include "srcu.c" #include "sync.c" #include "task.c" diff --git a/rust/helpers/suspend.c b/rust/helpers/suspend.c new file mode 100644 index 0000000000000..a616907ecc0b5 --- /dev/null +++ b/rust/helpers/suspend.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper bool rust_helper_pm_suspend_via_firmware(void) +{ + return pm_suspend_via_firmware(); +} + +__rust_helper bool rust_helper_pm_resume_via_firmware(void) +{ + return pm_resume_via_firmware(); +} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 1f845937899db..7bce2fe29e841 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -80,6 +80,7 @@ unsafe impl driver::RegistrationOps for Adapter { (*pdrv.get()).driver.of_match_table = of_table; (*pdrv.get()).driver.acpi_match_table = acpi_table; (*pdrv.get()).driver.suppress_bind_attrs = T::suppress_bind_attrs(); + (*pdrv.get()).driver.pm = T::pm_ops().map_or(core::ptr::null(), core::ptr::from_ref); } // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. @@ -230,6 +231,11 @@ pub trait Driver { false } + /// The driver's power management callbacks, if it has any. + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + None + } + /// Platform driver probe. /// /// Called when a new platform device is added or discovered. diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index c4f86930e3c65..e0c4be586801d 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -91,11 +91,34 @@ impl Mailbox { unsafe { device::Device::from_raw((*self.mbox).dev) } } + /// The C mailbox, for code that drives it with interrupts off. + pub fn as_raw(&self) -> *mut bindings::apple_mbox { + self.mbox + } + /// Sends the specified message pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { // SAFETY: Calling the c function, `mbox` is a valid pointer to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) } + + /// Sets bit 0 of the receive control word, for firmware that powers itself down. + pub fn set_recv_enabled(&self, enabled: bool) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_set_recv_enabled(self.mbox, enabled) }; + } + + /// Stops taking receive interrupts, so that no handler reads the mailbox. + pub fn hold_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_hold_rx(self.mbox) }; + } + + /// Undoes [`Self::hold_rx`]. + pub fn release_rx(&self) { + // SAFETY: Calling the c function, `mbox` is a valid pointer + unsafe { bindings::apple_mbox_release_rx(self.mbox) }; + } } impl Drop for Mailbox { diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 1524127cd475a..c34f187f4fb10 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -373,6 +373,26 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) } + /// Asks the coprocessor to go idle, AP then IOP state, and waits for both acknowledgments. + pub fn idle(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_idle(self.rtk) }) + } + + /// Turns the mailbox off for a coprocessor that has acknowledged idle, before its run bit and + /// power state go. Only [`Self::warm_restart`] follows this. + pub fn mailbox_off(self: Pin<&mut Self>) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_mailbox_off(self.rtk) }; + } + + /// Prepares for HELLO from firmware restarting warm, which keeps its buffers and rings. The + /// caller then sets the run bit and calls [`Self::boot`]. + pub fn warm_restart(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_warm_restart(self.rtk) }) + } + /// Starts a non-system endpoint. pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { // SAFETY: `rtk` is valid per the type invariant. @@ -399,6 +419,12 @@ impl RtKit { unsafe { bindings::apple_rtkit_is_crashed(self.rtk) } } + /// Echoes the OSLOG counters the firmware sends, or drops them. + pub fn set_oslog_echo(&self, echo: bool) { + // SAFETY: `rtk` is valid per the type invariant. + unsafe { bindings::apple_rtkit_set_oslog_echo(self.rtk, echo) }; + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/scripts/ci/apple-dwc3-atc-compile.sh b/scripts/ci/apple-dwc3-atc-compile.sh new file mode 100755 index 0000000000000..0d769c907f213 --- /dev/null +++ b/scripts/ci/apple-dwc3-atc-compile.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0 +# Compile the real Apple DWC3 consumer with ATC's unreachable-provider stubs. +set -euo pipefail + +if [[ $# != 1 || -z ${RUNNER_TEMP:-} ]]; then + echo "usage: RUNNER_TEMP= $0 " >&2 + exit 2 +fi +profile=$1 +case "$profile" in + builtin-no-atc) dwc3=y; atc=n ;; + modular-no-atc) dwc3=m; atc=n ;; + builtin-modular-atc) dwc3=y; atc=m ;; + *) echo "unknown DWC3/ATC profile: $profile" >&2; exit 2 ;; +esac + +source_dir=$(git rev-parse --show-toplevel) +cd "$source_dir" +work="$RUNNER_TEMP/dwc3-atc-$profile" +# Refuse reused output: a stale object must never satisfy this compile gate. +mkdir "$work" +output="$work/output" +artifacts="$work/artifacts" +mkdir "$output" "$artifacts" +fragment="$artifacts/requested.config" +compile_exit=NOT_RUN +stage=configuration + +finish() +{ + local rc=$? + trap - EXIT + { + printf 'last_stage=%s\ncompile_exit=%s\nscript_exit=%s\n' "$stage" "$compile_exit" "$rc" + if [[ -f $output/.config ]]; then + cp "$output/.config" "$artifacts/resolved.config" + sha256sum "$artifacts/resolved.config" + fi + } | tee -a "$artifacts/receipt.txt" + exit "$rc" +} +trap finish EXIT + +{ + printf 'profile=%s\nsource_commit=%s\nsource_tree=%s\n' "$profile" \ + "$(git rev-parse HEAD)" "$(git rev-parse HEAD^{tree})" + printf 'github_sha=%s\npr_head=%s\npr_base=%s\n' "${GITHUB_SHA:-unset}" \ + "${PR_HEAD_SHA:-unset}" "${PR_BASE_SHA:-unset}" + printf 'requested_USB_DWC3=%s\nrequested_USB_DWC3_APPLE=%s\nrequested_PHY_APPLE_ATC=%s\n' \ + "$dwc3" "$dwc3" "$atc" + clang --version + ld.lld --version +} | tee "$artifacts/receipt.txt" + +cat > "$fragment" <> "$fragment" +else + printf 'CONFIG_PHY_APPLE_ATC=%s\n' "$atc" >> "$fragment" +fi +sha256sum "$fragment" | tee -a "$artifacts/receipt.txt" + +export KCONFIG_CONFIG="$output/.config" +kmake=(make -C "$source_dir" O="$output" ARCH=arm64 LLVM=1) +"${kmake[@]}" defconfig 2>&1 | tee "$artifacts/defconfig.log" +scripts/kconfig/merge_config.sh -m -O "$output" "$output/.config" "$fragment" \ + 2>&1 | tee "$artifacts/merge-config.log" +"${kmake[@]}" olddefconfig 2>&1 | tee "$artifacts/olddefconfig.log" + +assert_config() +{ + local symbol=$1 expected=$2 actual + actual=$(scripts/config --file "$output/.config" --state "$symbol") + printf 'resolved_%s=%s\n' "$symbol" "$actual" | tee -a "$artifacts/receipt.txt" + if [[ $actual != "$expected" ]]; then + echo "$symbol resolved to $actual; expected $expected" >&2 + exit 1 + fi +} +# Assert the legal configuration after Kconfig resolution, before invoking CC. +assert_config USB_DWC3 "$dwc3" +assert_config USB_DWC3_APPLE "$dwc3" +assert_config PHY_APPLE_ATC "$atc" +for symbol in MODULES ARCH_APPLE OF PM USB_SUPPORT USB USB_GADGET USB_XHCI_HCD \ + TYPEC GENERIC_PHY USB_DWC3_DUAL_ROLE; do + assert_config "$symbol" y +done +cp "$output/.config" "$artifacts/resolved.config" +sha256sum "$artifacts/resolved.config" | tee -a "$artifacts/receipt.txt" + +target=drivers/usb/dwc3/dwc3-apple.o +command=("${kmake[@]}" -j2 V=1 "$target") +printf '%q ' "${command[@]}" | tee "$artifacts/compile-command.txt" +printf '\n' | tee -a "$artifacts/compile-command.txt" +stage=compile +set +e +"${command[@]}" 2>&1 | tee "$artifacts/compile.log" +pipeline_status=("${PIPESTATUS[@]}") +set -e +compile_exit=${pipeline_status[0]} +if [[ $compile_exit != 0 ]]; then + exit "$compile_exit" +fi +if [[ ${pipeline_status[1]} != 0 ]]; then + exit "${pipeline_status[1]}" +fi +stage=object-verification +test -s "$output/$target" +sha256sum "$output/$target" | tee -a "$artifacts/receipt.txt" +cp "$output/drivers/usb/dwc3/.dwc3-apple.o.cmd" "$artifacts/kbuild-command.txt" +stage=complete diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs index 4d3442bbfeafe..34f9c75b17d49 100644 --- a/sound/soc/apple/aop_audio.rs +++ b/sound/soc/apple/aop_audio.rs @@ -1092,6 +1092,36 @@ kernel::of_device_table!( ] ); +/// A system sleep with a firmware behind it. The firmware keeps running, so what it runs for +/// the streams has to stop first. Other variants' firmware isn't traced around a sleep. +/// +/// # Safety +/// +/// Called by the PM core with a live device of this driver. +unsafe extern "C" fn aop_audio_suspend(dev: *mut bindings::device) -> i32 { + // SAFETY: the helper only reads a flag. + if !unsafe { bindings::pm_suspend_via_firmware() } { + return 0; + } + // SAFETY: the PM core passes a live device, and keeps unbind away for this call. + let dev = unsafe { device::Device::::from_raw(dev) }; + let driver = dev.get_drvdata::(); + if driver.is_null() { + return 0; + } + // SAFETY: this driver stored it at probe, and it stays until unbind. + if let AopAudio::T8140(driver) = unsafe { &*driver } { + driver.sleep_stop(); + } + 0 +} + +static AOP_AUDIO_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { + suspend: Some(aop_audio_suspend), + // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. + ..unsafe { core::mem::zeroed() } +}; + /// The bound driver, as the variant set it up. enum AopAudio { /// Held for its `Drop`, which frees the card. @@ -1105,6 +1135,10 @@ impl platform::Driver for AopAudio { const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn pm_ops() -> Option<&'static bindings::dev_pm_ops> { + Some(&AOP_AUDIO_PM_OPS) + } + fn probe<'bound>( pdev: &'bound platform::Device>, info: Option<&'bound Self::IdInfo>, diff --git a/sound/soc/apple/aop_t8140.rs b/sound/soc/apple/aop_t8140.rs index aa97deda3b5c3..751cde0c4eb8e 100644 --- a/sound/soc/apple/aop_t8140.rs +++ b/sound/soc/apple/aop_t8140.rs @@ -62,6 +62,10 @@ use kernel::{ new_spinlock, platform, prelude::*, + regulator::{ + self, + Regulator, // + }, soc::apple::aop::{ from_fourcc, EPICService, @@ -492,6 +496,10 @@ struct SndSocT8140AopData { spkr_want_run: AtomicFlag, /// A microphone-array stream has been triggered and not yet stopped. hpai_want_run: AtomicFlag, + /// The array's rail (ADT function-input_power_enable), held while its + /// service runs. + #[pin] + hpai_supply: Mutex>>, /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. spkr_volume: Atomic, /// REG_SPKR_FULL_SCALE: the volume is below SPKR_CEILING_Q31, not the @@ -958,7 +966,11 @@ impl SndSocT8140AopData { if !self.hpai_want_run.load(Relaxed) || self.hpai.running.load(Relaxed) { return Ok(()); } - self.service_run(&self.hpai) + // As macOS does: the rail, then the stream. A failed start drops it again. + let supply = Regulator::::get(&self.dev, c"mic")?; + self.service_run(&self.hpai)?; + *self.hpai_supply.lock() = Some(supply); + Ok(()) } /// Called from sleeping close paths: a queued start must finish before @@ -972,7 +984,9 @@ impl SndSocT8140AopData { fn hpai_shutdown(&self) { self.flush_hpai_start(); - self.service_shutdown(&self.hpai, &self.hpai_domains()) + self.service_shutdown(&self.hpai, &self.hpai_domains()); + // As macOS does: the stream's stop, then the rail. + *self.hpai_supply.lock() = None; } // ---- internal speakers: the spkr service on LEAP TX0 (leap-ns ADMAC @@ -1037,6 +1051,35 @@ impl SndSocT8140AopData { Ok(()) } + /// `pw0 ` if running; the leaves stay with the stream. + fn service_stop(&self, svc: &Service) { + if svc.running.load(Relaxed) { + if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { + dev_err!(self.dev, "{}: pw0 failed: {:?}\n", svc.name, e); + } + svc.running.store(false, Relaxed); + } + } + + /// A system sleep: nothing the firmware runs may outlast the streams, whose DMA stops with + /// the sleep. Each one's next prepare or start asks for it again. + fn sleep_stop(&self) { + if self.powered.xchg(false, Relaxed) { + if let Err(e) = self.lpai_run_put() { + dev_err!(self.dev, "unable to return lpai to idle: {:?}\n", e); + } + } + self.flush_hpai_start(); + self.service_stop(&self.hpai); + *self.hpai_supply.lock() = None; + if let Err(e) = self.spkr_prepare() { + dev_err!(self.dev, "spkr: stop failed: {:?}\n", e); + } + self.service_stop(&self.tap); + self.service_stop(&self.cout); + self.service_stop(&self.cin); + } + /// Called from sleeping prepare/shutdown paths while ALSA serializes /// this stream: a queued start must finish before the mute and pw0. fn flush_spkr_start(&self) { @@ -1571,6 +1614,9 @@ fn pcm_open_dmaengine( if hw.formats == 0 { return EINVAL.to_errno(); } + // A system sleep stops the services, and only a prepare starts them all again. Offer + // neither resume nor pause: a transfer paused by the suspend would block the next start. + hw.info &= !(bindings::SNDRV_PCM_INFO_PAUSE | bindings::SNDRV_PCM_INFO_RESUME); // SAFETY: the substream's runtime exists for the op's duration; the // dmaengine open attaches `chan` to it on success. unsafe { @@ -2609,6 +2655,8 @@ impl AsocContext { pub(crate) struct SndSocT8140AopDriver { card: *mut bindings::snd_card, + /// The card's two microphone PCMs, null until created. + pcms: [*mut bindings::snd_pcm; 2], data: Arc, asoc: KBox, asoc_registered: bool, @@ -2664,6 +2712,7 @@ impl SndSocT8140AopDriver { // whatever was set up. let mut this = SndSocT8140AopDriver { card: ptr::null_mut(), + pcms: [ptr::null_mut(); 2], asoc: AsocContext::new(data.clone())?, data: data.clone(), asoc_registered: false, @@ -2777,6 +2826,7 @@ impl SndSocT8140AopDriver { (*pcm).private_free = Some(pcm_free_private); (*pcm).info_flags = 0; } + self.pcms[device as usize] = pcm; // SAFETY: as above. copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; if device == 0 { @@ -2797,6 +2847,21 @@ impl SndSocT8140AopDriver { Ok(()) } + /// A system sleep: stop the card's own streams, then whatever the firmware runs for them + /// and for the sound card's. Userspace prepares its streams again afterwards, since none + /// of them can resume. + pub(crate) fn sleep_stop(&self) { + for pcm in self.pcms.iter().copied().filter(|pcm| !pcm.is_null()) { + // SAFETY: the PCMs live as long as the card, and the driver core keeps + // unbind away from a system sleep. + let ret = unsafe { bindings::snd_pcm_suspend_all(pcm) }; + if ret < 0 { + dev_err!(self.data.dev, "unable to suspend a PCM: {}\n", ret); + } + } + self.data.sleep_stop(); + } + /// Undo probe: stop the reports, unregister the ASoC component and free /// the card (which closes every stream, releasing its channel and power /// references), then return lpai to idle. Runs from unbind, and again @@ -2889,6 +2954,9 @@ impl SndSocT8140AopDriver { // SAFETY: the pointer was checked above and stays valid with the child. let svc = unsafe { *service }; + // A capture takes the array's rail, which needs the SMC's GPIOs bound. + Regulator::::get(&dev, c"mic")?; + // The source ring: 512 KiB through our own IOMMU stream (dart-aop 9), // owned by the AOP driver because the firmware binds it once per boot. // SAFETY: no DMA is in flight on this device yet. @@ -2940,6 +3008,7 @@ impl SndSocT8140AopDriver { pd_rx0, spkr_want_run: AtomicFlag::new(false), hpai_want_run: AtomicFlag::new(false), + hpai_supply <- new_mutex!(None), spkr_volume: Atomic::new(SPKR_VOLUME_MAX), spkr_full_scale: AtomicFlag::new(false), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 18d73b50af861..9dfb329f46201 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -1467,10 +1468,41 @@ static const struct of_device_id apple_mca_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_mca_of_match); +/* + * hw_params is the only place a stream's cluster and serdes setup is written. A + * sleep that takes the registers would leave an open stream silent, and nothing + * here restores them yet. + */ +static int __maybe_unused apple_mca_suspend_late(struct device *dev) +{ + struct mca_data *mca = dev_get_drvdata(dev); + int i; + + if (!pm_suspend_via_firmware()) + return 0; + + for (i = 0; i < mca->nclusters; i++) { + struct mca_cluster *cl = &mca->clusters[i]; + + if (cl->pd_link || cl->clocks_in_use[SNDRV_PCM_STREAM_PLAYBACK] || + cl->clocks_in_use[SNDRV_PCM_STREAM_CAPTURE]) { + dev_err(dev, "cluster %d is in use, refusing the sleep\n", i); + return -EBUSY; + } + } + + return 0; +} + +static const struct dev_pm_ops apple_mca_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_mca_suspend_late, NULL) +}; + static struct platform_driver apple_mca_driver = { .driver = { .name = "apple-mca", .of_match_table = apple_mca_of_match, + .pm = pm_sleep_ptr(&apple_mca_pm_ops), }, .probe = apple_mca_probe, .remove = apple_mca_remove, diff --git a/tools/asahi/aop-bootargs-tests.py b/tools/asahi/aop-bootargs-tests.py index d2e24ecb07eb2..fd2920a7a3353 100644 --- a/tools/asahi/aop-bootargs-tests.py +++ b/tools/asahi/aop-bootargs-tests.py @@ -90,9 +90,29 @@ def method(source): def configs(source): + """struct AopHwConfig and the HW_CFG_* constants, each with its doc comment. + + Other items between them (the sleep's PM callbacks) use kernel bindings the harness + doesn't have. + """ start = source.index('struct AopHwConfig {') end = source.index('kernel::of_device_table!', start) - return source[start:end] + out, pending, item, keep, depth = [], [], [], None, 0 + for line in source[start:end].splitlines(keepends=True): + if keep is None: + if not line.strip() or line.startswith(('//', '#[')): + pending.append(line) + continue + keep = line.startswith(('struct AopHwConfig', 'const HW_CFG_')) + item, pending, depth = pending + [line], [], 0 + else: + item.append(line) + depth += line.count('{') - line.count('}') + if depth == 0 and line.rstrip().endswith((';', '}')): + if keep: + out += item + keep = None + return ''.join(out + pending) def compile_run(rustc, p, binary, code): diff --git a/tools/asahi/apple-dart-dapf-tests.py b/tools/asahi/apple-dart-dapf-tests.py new file mode 100644 index 0000000000000..9a516d2feead8 --- /dev/null +++ b/tools/asahi/apple-dart-dapf-tests.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the actual DAPF parser's optional-description and rejection contract. + +The host dependencies model DT metadata, allocation and mapping counts only. +No filter register or firmware is accessed; this cannot qualify hardware replay. +""" +import argparse +from pathlib import Path +import re +import resource +import subprocess + + +def function(text): + match = re.search(r'^static int apple_dart_dapf_parse\(', text, re.M) + if not match: + raise RuntimeError('production DAPF parser not found') + start = text.index('{', match.start()) + end, depth = start + 1, 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[match.start():end] + + +PREFIX = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define IORESOURCE_MEM 0 +#define GFP_KERNEL 0 +#define IS_ERR(p) ((uintptr_t)(p) >= (uintptr_t)-4095) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +struct device_node { int unused; }; +struct device { struct device_node *of_node; }; +struct platform_device { struct device dev; }; +struct resource { size_t size; }; +struct apple_dart { unsigned dapf_count; void *dapf; u64 *dapf_ranges; u32 *dapf_r20; }; +static bool has_ranges, has_r20, force_replay, map_failure; +static int count, maps, allocations; +static struct resource region; +static struct resource *platform_get_resource_byname(struct platform_device *p, int type, const char *name) +{ return ®ion; } +static size_t resource_size(struct resource *r) { return r->size; } +static bool of_property_present(struct device_node *n, const char *name) +{ return !strcmp(name,"apple,dapf-ranges") ? has_ranges : has_r20; } +static bool of_property_read_bool(struct device_node *n, const char *name) +{ assert(!strcmp(name,"apple,restore-on-power-up")); return force_replay; } +static int of_property_count_u64_elems(struct device_node *n, const char *name) +{ assert(!strcmp(name,"apple,dapf-ranges")); return has_ranges ? count : -EINVAL; } +static int dev_err_probe(struct device *d, int error, const char *format, ...) { return error; } +static void *devm_platform_ioremap_resource_byname(struct platform_device *p, const char *name) +{ ++maps; return map_failure ? (void *)(intptr_t)-EIO : ®ion; } +static void *devm_kcalloc(struct device *d, size_t n, size_t size, int flags) +{ ++allocations; return calloc(n ? n : 1,size); } +static int of_property_read_u64_array(struct device_node *n, const char *name, u64 *values, int nr) +{ for (int i=0;i\n' + 'static inline bool pm_suspend_via_firmware(void) { return false; }\n' + 'static inline bool pm_resume_via_firmware(void) { return false; }\n') + # The bridge's sleep-commit stop: MMIO, RTKit and the sleep driver, reached only in a suspend. + write(tmp, 'linux/io.h', '#include "mock.h"\n' + 'static inline u32 readl_relaxed(const volatile void __iomem *a) { assert(!"no system sleep here"); return 0; }\n' + 'static inline void iounmap(volatile void __iomem *a) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/of_address.h', '#include "mock.h"\n' + 'static inline void __iomem *of_iomap(struct device_node *n, int i) { assert(!"no system sleep here"); return NULL; }\n') + write(tmp, 'linux/soc/apple/rtkit.h', '#include "mock.h"\n#include \n' + 'struct apple_rtkit;\n' + 'static inline int apple_rtkit_sleep_idle_polled(struct apple_rtkit *r, void __iomem *c,\n' + '\t\t\t\t\t\tunsigned int *us) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_rtkit_mailbox_recv_off(struct apple_rtkit *r) { assert(!"no system sleep here"); }\n') + write(tmp, 'linux/soc/apple/sleep.h', '#include "mock.h"\n#include \n' + 'static inline int apple_sleep_coproc_register(struct device *d, void __iomem *a, bool p,\n' + '\t\t\t\t\t int (*f)(void *), void *x) { assert(!"no system sleep here"); return -EIO; }\n' + 'static inline void apple_sleep_coproc_set_ready(struct device *d, bool r) { assert(!"no system sleep here"); }\n' + 'static inline bool apple_sleep_coproc_stopped(struct device *d) { assert(!"no system sleep here"); return false; }\n' + 'static inline bool apple_sleep_commit_stops_coprocs(void) { assert(!"no system sleep here"); return false; }\n') + write(tmp, 'linux/soc/apple/pmp-display.h', '#include \nbool apple_pmp_display_retained_board(void);\n') write(tmp, 'mock.h', '''#pragma once #include #include #include #include +#define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_err_probe(d, e, ...) (e) +#define dev_info(...) ((void)0) +#include +#define __iomem #include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; @@ -121,6 +153,7 @@ def main(): /* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} +static inline int of_property_match_string(const struct device_node*n,const char*p,const char*s){assert(!"no system sleep here");return -EINVAL;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" @@ -129,6 +162,12 @@ def main(): int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} +bool apple_pmp_display_retained_board(void){return false;} +void apple_pmp_report_sleep(struct device*d,bool at_commit){assert(!"no system sleep here");} +void apple_pmp_report_wake(struct device*d,bool real){assert(!"no system sleep here");} +int apple_pmp_sleep(const void*data){assert(!"no system sleep here");return -EIO;} +int apple_pmp_sleep_prepare(const void*data,struct apple_pmp_commit_stop*stop){assert(!"no system sleep here");return -EIO;} +int apple_pmp_wake(const void*data){assert(!"no system sleep here");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 13d1262f601ed..e72bebc02f3ab 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -126,6 +126,8 @@ def function(source, name): static void mt_awdl_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_data_complete(struct mt7932*m,const u8*p,size_t n){} static void mt_packet_receive(struct mt7932*m,const u8*p,size_t n){} +static void mt_hif_event(struct mt7932*m,const struct mt7932_event*e){} +static void mt_wow_reason_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_rf_fail_locked(struct mt7932*m,int e){m->rf_ready=false;m->link_failed=true;m->cal_state.error=e;m->cal_state.active=false;m->connect_error=e;complete_all(&m->assoc_done);complete_all(&m->cal_response);} CHANNEL LINK @@ -254,6 +256,9 @@ def main(): headers.append(header) cal=(root/DRIVER/"calibration.h").read_text() headers.append(cal[cal.index("struct mt7932_cal_completion {"):cal.index("struct mt7932_cal_segment {")]) + # The receive path's other events, by their production IDs: the firmware's HIF and WoW reports. + headers.append(re.search(r"^#define MT7932_CID_HIF_CTRL\b.*$", rx, re.M).group(0)) + headers.append(re.search(r"^#define MT7932_EID_WOW_REASON\b.*$", (root/DRIVER/"wow.h").read_text(), re.M).group(0)) parts["HEADERS"]="\n".join(headers) candidates={"production":parts} if args.old_source: diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py index 4f4d183021905..fe4a55aa16d79 100644 --- a/tools/asahi/mtp-pm-tests.py +++ b/tools/asahi/mtp-pm-tests.py @@ -46,12 +46,18 @@ def function(source, name): static bool schedule_delayed_work(struct delayed_work *w, unsigned long d) { w->queued++; w->pending = true; return true; } struct hid_device { struct magicmouse_sc *data; int id; }; struct dchid_iface { struct hid_device *hid; }; -struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; +struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended, host_asleep; }; struct device { struct dockchannel_hid *data; }; static struct dockchannel_hid *transport; static unsigned suspends[16], resumes[16], key_releases; static int suspend_error_at = -1, resume_error_at = -1; static void *dev_get_drvdata(struct device *d) { return d->data; } +#define HOST_STATE_SLEEP 1 +#define HOST_STATE_AWAKE 2 +#define dev_warn(...) ((void)0) +/* Not a firmware-driven suspend: the host state is never set here. */ +static bool pm_suspend_via_firmware(void) { return false; } +static int dchid_host_state(struct dockchannel_hid *d, int state) { (void)d; (void)state; assert(0); return 0; } static void *hid_get_drvdata(struct hid_device *d) { return d->data; } static void magicmouse_force_key(struct magicmouse_sc *m, bool down) { assert(!down); key_releases++; diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index 1d0a5302af4ce..8cfe659498c49 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -80,7 +80,8 @@ def function(source, name): enum dwc3_apple_state {DWC3_APPLE_PROBE_PENDING,DWC3_APPLE_NO_CABLE,DWC3_APPLE_HOST,DWC3_APPLE_DEVICE,DWC3_APPLE_SUSPENDED}; #define DWC3_GCTL_PRTCAP_HOST 1 struct dwc3 {int num_usb3_ports,num_usb2_ports;int usb3_generic_phy[2],usb2_generic_phy[2],usb3_phy,usb2_phy;int current_dr_role;bool dis_u3_susphy_quirk;}; -struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending,dt_dis_u3_susphy;unsigned usb3_power_off_pending,usb2_power_off_pending;int usb2_phy;}; +struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending,dt_dis_u3_susphy,sleep_ddr;unsigned usb3_power_off_pending,usb2_power_off_pending;int usb2_phy;}; +static void apple_sleep_usb_slp_ddr(bool on){(void)on;} static int core_error,host_error,gadget_error,reset_error,role_error,start_role; static unsigned rpm_puts; /* A started core holds the glue runtime PM reference until core_stop drops it. */ diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 469cacc4b3082..062597ca80b56 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -25,6 +25,9 @@ 'control |= rtc->hw->control_event_bit;'), 'stowe-offset': ('static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = {', 'static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = {\n\t.offset_in_seconds = true,'), + 'baku-limit': ('.max_delay = U32_MAX,', '.max_delay = APPLE_PMU_RTC_MAX_DELAY,'), + 'set-no-round': ('(counter + BIT(APPLE_PMU_RTC_HZ_SHIFT - 1))', '(counter)'), + 'set-negative': ('if (offset < 0 || offset', 'if (offset'), } PRELUDE = r''' @@ -41,12 +44,14 @@ typedef uint64_t u64; typedef int64_t s64; typedef uint32_t __le32; +typedef uint64_t __le64; #define BIT(n) (1U << (n)) #define U32_MAX UINT32_MAX #define RTC_TIMESTAMP_END_9999 253402300799LL #define cpu_to_le32(x) (x) #define le32_to_cpu(x) (x) #define le64_to_cpu(x) (x) +#define cpu_to_le64(x) (x) #define IS_ERR(x) ((intptr_t)(x) < 0) #define PTR_ERR(x) ((int)(intptr_t)(x)) #define kfree free @@ -63,13 +68,15 @@ struct rtc_wkalrm { struct rtc_time time; bool enabled, pending; }; struct rtc_device { int events; }; struct device { void *data; bool wake; int wake_events; }; -struct nvmem_cell { u8 data[8]; size_t bytes; int error; }; +struct nvmem_cell { u8 data[8]; size_t bytes; int error, write_error; }; struct regmap { u8 bytes[65536]; unsigned ctrl, counter, compare, mask, event, bit, control_event; int ops, fail; }; static int disabled, pm_suspend_target_state = PM_SUSPEND_TO_IDLE; +static bool firmware_suspend; +static bool pm_suspend_via_firmware(void) { return firmware_suspend; } static inline __attribute__((unused)) s64 sign_extend64(u64 v, int bit) { return (s64)(v << (63 - bit)) >> (63 - bit); } static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } static void mutex_lock(struct mutex *m) { (void)m; } @@ -90,6 +97,10 @@ void *p = malloc(c->bytes); assert(p); memcpy(p, c->data, c->bytes); *bytes = c->bytes; return p; } +static int nvmem_cell_write(struct nvmem_cell *c, void *buf, size_t bytes) { + if (c->write_error) return c->write_error; + assert(bytes == c->bytes); memcpy(c->data, buf, bytes); return bytes; +} static bool io_error(struct regmap *m) { return ++m->ops == m->fail; } static int regmap_read(struct regmap *m, unsigned reg, unsigned *v) { if (io_error(m)) return -EIO; @@ -226,16 +237,59 @@ assert(!apple_pmu_rtc_validate_alarm(&rtc, 101)); assert(apple_pmu_rtc_validate_alarm(&rtc, 100) == -ETIME); assert(!apple_pmu_rtc_validate_alarm(&rtc, 3700)); - assert(apple_pmu_rtc_validate_alarm(&rtc, 3701) == -ERANGE); alarm.time.seconds = 1100; assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ETIME); alarm.time.seconds = 4701; - assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ERANGE); + if (a) { + /* One hour, where the alarm only ends a suspend-to-idle. */ + assert(apple_pmu_rtc_validate_alarm(&rtc, 3701) == -ERANGE); + assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ERANGE); + } else { + /* Baku: as far as the 32-bit comparator reaches. */ + assert(!apple_pmu_rtc_validate_alarm(&rtc, 3701)); + assert(!apple_pmu_rtc_validate_alarm(&rtc, U32_MAX)); + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && comparator() == 3701); + alarm.time.seconds = 86400 * 7 + 1000; + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && comparator() == 86400 * 7); + alarm.time.seconds = 1000 + (s64)U32_MAX + 1; + assert(apple_pmu_rtc_set_alarm(&dev, &alarm) == -ERANGE); + assert(!apple_pmu_rtc_alarm_enable(&dev, 0)); + } /* Reject malformed offsets and propagate NVMEM errors. */ offset.bytes = a ? 8 : 6; assert(apple_pmu_rtc_offset(&rtc, &ticks) == -EINVAL); offset.error = -EIO; assert(apple_pmu_rtc_read_time(&dev, &time) == -EIO); + if (!a) { + /* Baku sets its whole-second offset, rounding the counter to the second. */ + reset(v); + time.seconds = 5000; + set_counter((100ULL << 16) + 32767); + assert(!apple_pmu_rtc_set_time(&dev, &time)); + assert(get_unaligned_le64(offset.data) == 4900); + assert(!apple_pmu_rtc_read_time(&dev, &time) && time.seconds == 5000); + time.seconds = 5000; + set_counter((100ULL << 16) + 32768); + assert(!apple_pmu_rtc_set_time(&dev, &time)); + assert(get_unaligned_le64(offset.data) == 4899); + /* Alarms follow the new offset. */ + alarm.time.seconds = 5030; + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && comparator() == 131); + assert(!apple_pmu_rtc_alarm_enable(&dev, 0)); + /* Before the counter's own time, and past what the offset may hold. */ + time.seconds = 99; + assert(apple_pmu_rtc_set_time(&dev, &time) == -ERANGE); + time.seconds = RTC_TIMESTAMP_END_9999; + assert(apple_pmu_rtc_set_time(&dev, &time) == -ERANGE); + assert(get_unaligned_le64(offset.data) == 4899); + time.seconds = 5000; + offset.write_error = -EIO; + assert(apple_pmu_rtc_set_time(&dev, &time) == -EIO); + offset.write_error = 0; + map.ops = 0; map.fail = 1; + assert(apple_pmu_rtc_set_time(&dev, &time) == -EIO); + assert(get_unaligned_le64(offset.data) == 4899); + } reset(v); set_offset(-32768); if (a) { @@ -275,11 +329,15 @@ reset(v); assert(!apple_pmu_rtc_prepare(&dev)); pm_suspend_target_state = 2; assert(apple_pmu_rtc_prepare(&dev) == -EBUSY); + /* A firmware-driven suspend programs the wake itself. */ + firmware_suspend = true; + assert(!apple_pmu_rtc_prepare(&dev)); + firmware_suspend = false; pm_suspend_target_state = PM_SUSPEND_TO_IDLE; } int main(void) { run_variant(BAKU); run_variant(ABBEY); run_variant(STOWE); run_variant(MAVERICK); - puts("PMU RTC: Baku, Abbey, Stowe and Maverick arithmetic, W1C transactions and fault injection passed"); + puts("PMU RTC: Baku, Abbey, Stowe and Maverick arithmetic, W1C transactions, Baku clock setting and fault injection passed"); return 0; } ''' diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index e8e7166399a0e..9df97f65c1591 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -37,6 +37,7 @@ def function(source, name): #define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) #define APPLE_RTKIT_MAX_ENDPOINTS 256 #define WRITE_ONCE(x,v) ((x)=(v)) +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) #define spin_lock_irqsave(l,f) ((void)(l), (f)=0) #define spin_unlock_irqrestore(l,f) ((void)(l), (void)(f)) struct apple_rtkit_shmem { size_t size; }; @@ -49,6 +50,9 @@ def function(source, name): struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer, pmp_aux_buffer; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; + bool oslog_echo; + u64 oslog_owed; + int boot_result; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4], started[4]; char *syslog_msg_buffer; @@ -69,6 +73,7 @@ def function(source, name): static void set_bit(unsigned n, unsigned long *b) { b[n/64] |= 1UL << (n%64); } static void bitmap_zero(unsigned long *b, unsigned n) { memset(b,0,n/8); } static void reinit_completion(int *c) { *c=0; } +static void complete_all(int *c) { *c=1; } static void apple_rtkit_rx(void) {} static void apple_rtkit_detach_rx(struct apple_rtkit *r) { ++detachments; r->shutting_down=true; } static void flush_workqueue(void *w) { (void)w; ++flushes; } diff --git a/tools/asahi/rtkit-retained-tests.py b/tools/asahi/rtkit-retained-tests.py index b79a9cd542bd2..f4e81d99f681d 100755 --- a/tools/asahi/rtkit-retained-tests.py +++ b/tools/asahi/rtkit-retained-tests.py @@ -66,18 +66,20 @@ def constants(source): { (void)rtk; sent++; sent_type = type; sent_data = data; return 0; } -/* A signal is pending for every interruptible wait. */ +/* A fatal signal is pending for every killable wait. */ #define ERESTARTSYS 512 #define READ_ONCE(x) (x) +enum { PM_SUSPEND_ON, PM_SUSPEND_TO_IDLE, PM_SUSPEND_STANDBY, PM_SUSPEND_MEM }; +static int pm_suspend_target_state; struct completion { int unused; }; -static unsigned waited_uninterruptible, waited_interruptible; +static unsigned waited_uninterruptible, waited_killable; static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long t) { (void)c; waited_uninterruptible++; return t; } -static long wait_for_completion_interruptible_timeout(struct completion *c, unsigned long t) +static long wait_for_completion_killable_timeout(struct completion *c, unsigned long t) { - (void)c; (void)t; waited_interruptible++; return -ERESTARTSYS; + (void)c; (void)t; waited_killable++; return -ERESTARTSYS; } ''' @@ -156,14 +158,23 @@ def constants(source): assert(sent == 0 && r.iop_pwr_ack_completion == 1); assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_QUIESCED); - /* A retained wake waits through a signal; other waits stay interruptible. */ + /* + * A retained wake waits through a signal, and so does a wait in a system + * sleep; any other wait ends only for a fatal one, not the freezer's. + */ struct completion c; r = fresh(); assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); - assert(waited_uninterruptible == 1 && waited_interruptible == 0); + assert(waited_uninterruptible == 1 && waited_killable == 0); r.retained_wake = false; assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); - assert(waited_uninterruptible == 1 && waited_interruptible == 1); + assert(waited_uninterruptible == 1 && waited_killable == 1); + pm_suspend_target_state = PM_SUSPEND_MEM; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); + assert(waited_uninterruptible == 2 && waited_killable == 1); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); + assert(waited_uninterruptible == 2 && waited_killable == 2); puts("PASS retained endpoint map replies, IOP acknowledgments and waits"); return 0; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 6900b7fe82f5f..0792d35750fc8 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -73,6 +73,8 @@ def mutations(flag, names, base, expect=KILL): 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-m4-pmu-tests.py': [([], PASS)] + mutations('--mutation', ('m1-layout', 'cycles-anywhere', 'expose-8-9', 'flags-in-event'), []), + 'apple-dart-dapf-tests.py': [(out(), PASS)] + + mutations('--mutation', ('missing-guard', 'ignore-replay', 'ignore-orphan'), out()), 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'asahi-stats-raw-tests.py': [([], PASS)], @@ -161,12 +163,21 @@ def mutations(flag, names, base, expect=KILL): + [(out('--output') + ['--old-usb'], KILL), (out('--output') + ['--old-regulatory'], KILL)], 'pciec-policy-tests.py': [(out(), PASS)], 'pmgr-bus-tests.py': [([], PASS)], + 'sep-suspend-errors-tests.py': [(out(), PASS)] + + mutations('--mutation', ('unsupported-as-success', 'read-as-deadline', 'no-capability-check', + 'no-transport-check', 'lost-accept', 'nonphysical-wake', + 'wake-before-arm', 'bulk-error-as-success', 'omit-pending-wake', + 'omit-before-send', 'leak-prepared', 'sent-error-return', + 'accept-unanswered-gate', 'ignore-result', 'ignore-gate-timeout', + 'finish-without-physical', 'poll-after-callback', 'poll-unsupported', + 'stale-observation', 'drain-two', 'post-send-wake-refusal'), out()), 'pmp-agx-vote-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'pmp-j613-hwmon-tests.py': [([], PASS)], 'pmu-rtc-tests.py': [(out(), PASS)] + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', 'offset-sign', 'event-bit', 'wrong-control', - 'maverick-event', 'stowe-offset'), out()), + 'maverick-event', 'stowe-offset', 'baku-limit', + 'set-no-round', 'set-negative'), out()), 'perf-mmap-alias-lifecycle-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-crashlog-tests.py': [(out(), PASS)] diff --git a/tools/asahi/sep-suspend-errors-tests.py b/tools/asahi/sep-suspend-errors-tests.py new file mode 100644 index 0000000000000..6c36f56642b63 --- /dev/null +++ b/tools/asahi/sep-suspend-errors-tests.py @@ -0,0 +1,733 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual SEP commit/error paths with host-only dependency stubs. + +Extract current C stop/finish/prepare/noirq, provider poll-one and registration, +Rust wrappers/enter/leave/exit/kick, and the kernel polling macros verbatim. +C fatal paths are observed wholly within C, never by unwinding a Rust frame. +Virtual time/MMIO only test AP control decisions, not firmware timing or safety. +No target access or kernel build occurs. +""" +import argparse +from pathlib import Path +import re +import resource +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def function(source, name): + match = re.search(r'^[ \t]*(?:(?:pub(?:\([^)]*\))? )?fn|(?:static )?(?:int|bool|void)(?: __maybe_unused)?) ' + re.escape(name) + r'\([^;]*?\)\s*(?:->[^\{]+)?\{', + source, re.M) + if not match: + raise AssertionError(f'missing function: {name}') + start = source.index('{', match.start()) + end, depth = start + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +def macro(source, name): + match = re.search(r'^#define ' + re.escape(name) + r'\([^\n]*\n(?:.*\\\n)*.*', + source, re.M) + if not match: + raise AssertionError(f'missing macro: {name}') + return match.group() + + +def replace_once(source, before, after): + # A stale mutation is a broken harness, never a killed behavioral control. + if source.count(before) != 1: + raise AssertionError(f'mutation requires one exact match: {before!r}') + return source.replace(before, after, 1) + + +# Mutations apply to an extracted real function, not to a mock decision model. +# All must compile and fail a C assertion or a Rust test assertion in remote CI. +MUTATIONS = { + 'unsupported-as-success': ('leave_for_commit', 'Err(Error::from_errno(rc))', 'Ok(())'), + 'read-as-deadline': ('sep_pm_wait_gated', 'if (ret)\n\t\treturn ret;\n\n\treturn !!poll_ret;', + 'if (ret)\n\t\treturn 1;\n\n\treturn !!poll_ret;'), + 'no-capability-check': ('sep_pm_commit_prepare', + ' || !IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) ||\n\t !is_kernel_in_hyp_mode()', ''), + 'no-transport-check': ('sep_pm_commit_prepare', + 'if (!apple_mbox_poll_one_supported(mbox))', 'if (false)'), + 'lost-accept': ('sleep_enter', 'pm::SleepGate::Lost => return Err(EIO),', + 'pm::SleepGate::Lost => return Ok(()),'), + 'nonphysical-wake': ('sleep_exit', 'let real = unsafe { bindings::pm_resume_via_firmware() };', + 'let real = true;'), + 'wake-before-arm': ('sleep_kick', + 'self.control.lock().wake_reset();\n self.sleep_release_rx();', + 'self.sleep_release_rx();\n self.control.lock().wake_reset();'), + 'bulk-error-as-success': ('sep_pm_commit_stop', 'if (ret < 0)\n\t\tgoto out;', + 'if (ret < 0) { ret = 0; goto out; }'), + 'omit-pending-wake': ('sep_pm_commit_stop', 'if (pm_wakeup_pending())', 'if (false)'), + 'omit-before-send': ('sep_pm_commit_stop', 'if (xchg(&sleep_refuse_before_send, false))', + 'if (false)'), + 'leak-prepared': ('sep_pm_commit_stop', 'if (prepared)\n\t\t\tapple_dart_sleep_restore(c->dart);', + 'if (false)\n\t\t\tapple_dart_sleep_restore(c->dart);'), + 'sent-error-return': ('sep_pm_commit_stop', + 'panic("SEP SLEEP failed after send: ret %d reply_valid %d result %#x state %#x",\n\t\t ret, c->answered, c->reply_result, c->state);', + 'return ret;'), + 'accept-unanswered-gate': ('sep_pm_commit_stop', 'if (ret >= 0 && !c->answered)', + 'if (ret >= 0 && !c->answered && !(c->state & SEP_PS_DEV_DISABLE))'), + 'ignore-result': ('sep_pm_commit_stop', 'if (c->answered && c->reply_result)', + 'if (false && c->answered && c->reply_result)'), + 'ignore-gate-timeout': ('sep_pm_commit_stop', 'if (c->gate_ret)', 'if (false && c->gate_ret)'), + 'finish-without-physical': ('sep_pm_commit_finish', 'if (c->sent && !pm_resume_via_firmware())', + 'if (false)'), + 'poll-after-callback': ('apple_mbox_poll_one', 'mbox->rx(mbox, msg, mbox->cookie);', + 'mbox->rx(mbox, msg, mbox->cookie);\n\t\tcontrol = readl_relaxed(mbox->regs + mbox->hw->i2a_control);'), + 'poll-unsupported': ('apple_mbox_poll_one', 'if (!apple_mbox_poll_one_supported(mbox))', + 'if (false)'), + 'stale-observation': ('sep_pm_commit_prepare', 'c->answered = false;', '/* stale answered */'), + 'drain-two': ('apple_mbox_poll_one', 'ret = 1;', + 'msg = apple_mbox_read_msg(mbox);\n\t\tmbox->rx(mbox, msg, mbox->cookie);\n\t\tret = 2;'), + 'post-send-wake-refusal': ('sep_pm_commit_stop', 'c->sent = true;', + 'c->sent = true;\n\tif (pm_wakeup_pending()) return -EBUSY;'), +} + +C_PREFIX = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint32_t u32; typedef uint64_t u64; +typedef int64_t s64; typedef long long ktime_t; +#define BIT(n) (1u << (n)) +#define SEP_PS_DEV_DISABLE BIT(10) +#define USEC_PER_MSEC 1000 +#define NSEC_PER_USEC 1000 +#define READ_ONCE(x) (x) +#define IS_ENABLED(x) (config_enabled) +#define CONFIG_APPLE_SOC_SLEEP 1 +#define __maybe_unused +#define __iomem +#define unlikely(x) (x) +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_info(...) ((void)0) +#define might_sleep_if(x) ((void)0) +#define barrier() ((void)0) +#define cpu_relax() ((void)0) +#define guard(kind) (void) +#define xchg(p, v) ({ __typeof__(*(p)) old = *(p); *(p) = (v); old; }) +struct device_node { struct device_node *parent; }; +struct device { struct device_node *of_node; }; +struct resource { unsigned long start; }; +struct regmap { int unused; }; +struct apple_mbox_msg { u64 msg0; u32 msg1; }; +struct apple_mbox_hw { + bool is_ascwrap_v8, counted_fifo, has_irq_controls; unsigned i2a_control; +}; +struct apple_mbox { + void *regs; struct apple_mbox_hw *hw; bool removing; int rx_lock; + void (*rx)(struct apple_mbox *, struct apple_mbox_msg, void *); void *cookie; +}; +static struct device_node parent, node = { &parent }; +static struct device device = { &node }, dart; +static struct regmap regmap; +static struct apple_mbox_hw hw; +static struct apple_mbox mbox; +static int apple_sleep_coprocs_lock; +static void *apple_sleep; +static struct { + struct device *dev; int (*stop)(void *); void *data; +} apple_sleep_enclave; +static bool config_enabled, hyp, sep_pm_suspending, dart_present, dart_ready, map_ok; +static bool firmware, real_wake, pending_wake, late_wake, sleep_refuse_before_send, tx_ready; +static int domain_error, property_error, address_error, read_error, read_error_after; +static int gate_after, reads, put_refs, maps, unmaps, lookups, prepares, restores; +static int prepare_error, send_error, bulk_error, one_error, sends, closes; +static int mailbox_reads, message_reads, forwarded, lock_depth; +static int resource_bytes, close_gate, early_gate; +static bool message_in_lock, matched_read; +static struct apple_mbox_msg queue[4]; +static unsigned queued, consumed; +static ktime_t now; +static bool (*sep_pm_told)(void); +static bool yes(void) { return true; } +static bool no(void) { return false; } +static bool pm_suspend_via_firmware(void) { return firmware; } +static bool pm_resume_via_firmware(void) { return real_wake; } +static bool is_kernel_in_hyp_mode(void) { return hyp; } +static bool pm_wakeup_pending(void) { return pending_wake; } +static int sep_pm_domain(void *p, struct regmap **m, u32 *o) { + lookups++; *m = ®map; *o = 0; return domain_error; +} +static int regmap_read(struct regmap *m, u32 o, u32 *v) { + ++reads; if (read_error && reads >= read_error_after) return read_error; + *v = gate_after && reads >= gate_after ? SEP_PS_DEV_DISABLE : 0; return 0; +} +static int regmap_write(struct regmap *m, u32 o, u32 v) { return 0; } +static ktime_t ktime_get(void) { return now; } +static ktime_t ktime_add_us(ktime_t t, u64 us) { return t + us; } +static int ktime_compare(ktime_t a, ktime_t b) { return (a > b) - (a < b); } +static void usleep_range(unsigned long a, unsigned long b) { now += b; } +static void udelay(unsigned long us) { now += us; } +static struct device_node *of_parse_phandle(struct device_node *n, const char *p, int i) { + lookups++; return domain_error ? NULL : &node; +} +static int of_property_read_u32(struct device_node *n, const char *p, u32 *v) { + *v = 0; return property_error; +} +static int of_address_to_resource(struct device_node *n, int i, struct resource *r) { + r->start = 0; return address_error; +} +static void of_node_put(struct device_node *n) {} +static unsigned resource_size(struct resource *r) { return resource_bytes; } +static struct device *sep_pm_dart(struct device *d) { return dart_present ? &dart : NULL; } +static void *dev_get_drvdata(struct device *d) { return dart_ready ? d : NULL; } +static void *ioremap(unsigned long start, unsigned size) { ++maps; return map_ok ? ®map : NULL; } +static void iounmap(void *p) { ++unmaps; } +static void put_device(struct device *d) { ++put_refs; } +static int apple_dart_sleep_prepare(struct device *d) { ++prepares; return prepare_error; } +static void apple_dart_sleep_restore(struct device *d) { ++restores; } +static void lock_rx(void) { + assert(lock_depth == 0); lock_depth++; message_in_lock = false; +} +static void unlock_rx(void) { assert(lock_depth == 1); lock_depth--; message_in_lock = false; } +#define spin_lock_irqsave(lock, flags) do { (flags) = 0; lock_rx(); } while (0) +#define spin_unlock_irqrestore(lock, flags) do { (void)(flags); unlock_rx(); } while (0) +static void foreign_rx(struct apple_mbox *m, struct apple_mbox_msg msg, void *cookie) { + assert(cookie == &device); ++forwarded; +} +static u32 readl_relaxed(void *p) { + // A second status/ACK/read after delivering a message is forbidden. + assert(!message_in_lock && !matched_read); ++mailbox_reads; + return queued == consumed; +} +static bool apple_mbox_rx_empty(struct apple_mbox *m, u32 control) { return control != 0; } +static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *m) { + assert(!message_in_lock && !matched_read); assert(consumed < queued); + message_in_lock = true; ++message_reads; + struct apple_mbox_msg msg = queue[consumed++]; + if ((msg.msg0 & 0xffffff) == 0x1234) matched_read = true; + return msg; +} +static int apple_mbox_poll(struct apple_mbox *m) { return bulk_error; } +static bool apple_mbox_send_ready(struct apple_mbox *m) { return tx_ready; } +static int apple_mbox_send(struct apple_mbox *m, struct apple_mbox_msg msg, bool atomic) { + assert(atomic); assert(prepares == 1); + // Production apple_mbox_send has only pre-write negative exits. + if (send_error) return send_error; + ++sends; + if (late_wake) pending_wake = true; + if (one_error == -ENODEV) m->removing = true; + if (one_error == -EOPNOTSUPP) m->hw->is_ascwrap_v8 = true; + return 0; +} +static void apple_mbox_set_recv_enabled(struct apple_mbox *m, bool enabled) { + assert(!enabled); assert(matched_read); ++closes; +} +static u32 readl(void *p) { + // PMGR is the sole permitted query after the matched reply/receive close. + return early_gate || (closes && close_gate) ? SEP_PS_DEV_DISABLE : 0; +} +static jmp_buf fatal_env; +static bool fatal_armed; +static char fatal_message[256]; +static _Noreturn void panic_observe(const char *fmt, ...) { + va_list ap; va_start(ap, fmt); vsnprintf(fatal_message, sizeof(fatal_message), fmt, ap); va_end(ap); + assert(fatal_armed && "unexpected C panic (must never unwind across Rust)"); + longjmp(fatal_env, 1); +} +#define panic panic_observe +''' + +# Appended after the actual functions/struct: only dependency state and drivers. +C_CONTROLS = r''' +void fixture(void) { + memset(&sep_pm_commit, 0, sizeof(sep_pm_commit)); + memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); + memset(&hw, 0, sizeof(hw)); + mbox = (struct apple_mbox){ .regs = &hw, .hw = &hw, .rx = foreign_rx, .cookie = &device }; + config_enabled = hyp = sep_pm_suspending = dart_present = dart_ready = map_ok = true; + firmware = tx_ready = true; apple_sleep = &device; + real_wake = pending_wake = late_wake = sleep_refuse_before_send = false; + domain_error = property_error = address_error = read_error = gate_after = reads = 0; + put_refs = maps = unmaps = lookups = prepares = restores = sends = closes = 0; + prepare_error = send_error = bulk_error = one_error = 0; + mailbox_reads = message_reads = forwarded = lock_depth = 0; + resource_bytes = 4; close_gate = 1; early_gate = 0; read_error_after = 1; + queued = consumed = 0; message_in_lock = matched_read = false; now = 0; + fatal_armed = false; fatal_message[0] = 0; sep_pm_told = yes; +} +void parameter(int which, int value) { + switch (which) { + case 0: config_enabled = value; break; case 1: hyp = value; break; + case 2: sep_pm_suspending = value; break; case 3: dart_present = value; break; + case 4: map_ok = value; break; case 5: domain_error = value; break; + case 6: read_error = value; break; case 7: apple_sleep = value ? &device : NULL; break; + case 8: gate_after = value; break; case 9: sep_pm_commit.registered = value; break; + case 10: read_error_after = value; break; case 11: dart_ready = value; break; + case 12: real_wake = value; break; case 13: hw.is_ascwrap_v8 = value; break; + case 14: hw.counted_fifo = value; break; case 15: hw.has_irq_controls = value; break; + case 16: property_error = value; break; case 17: address_error = value; break; + case 18: resource_bytes = value; break; + case 19: apple_sleep_enclave.dev = value ? &device : NULL; break; + case 20: sep_pm_commit.sent = value; break; case 21: firmware = value; break; + case 22: sep_pm_told = value ? yes : NULL; break; + } +} +int counter(int which) { + switch (which) { + case 0: return put_refs; case 1: return maps; case 2: return unmaps; + case 3: return sep_pm_commit.registered; case 4: return lookups; + case 5: return reads; case 6: return sends; case 7: return mailbox_reads; + default: assert(0); return -1; + } +} +void *device_pointer(void) { return &device; } +void *mbox_pointer(void) { return &mbox; } +int noirq(void) { return sep_pm_suspend_noirq(&device); } + +#ifdef CONTROL_MAIN +static int stage(void) { + return sep_pm_commit_prepare(&device, &mbox, 0x55, 0, 0x1234, 0xffffff); +} +static void reply(unsigned result) { + assert(queued < 4); queue[queued++] = (struct apple_mbox_msg){0x1234 | ((u64)result << 24), 0}; +} +static void balanced(void) { + assert(lock_depth == 0); assert(mbox.rx == foreign_rx && mbox.cookie == &device); +} +static void expect_stop_fatal(const char *reason) { + // This setjmp frame, production callback and panic observer are all C. + fatal_armed = true; + if (setjmp(fatal_env) == 0) { + (void)sep_pm_commit_stop(&sep_pm_commit); + assert(0 && "sent failure must panic, never return"); + } + fatal_armed = false; + assert(strstr(fatal_message, reason)); balanced(); + assert(sep_pm_commit.sent && sends == 1 && restores == 0); + assert(sep_pm_commit.registered && put_refs == 0 && unmaps == 0); +} +static void test_prepare_and_cleanup(void) { + // Unsupported RAM guards do no mailbox MMIO, mapping or sending. + for (int p = 0; p < 3; ++p) { + fixture(); parameter(p, 0); assert(stage() == -EOPNOTSUPP); + assert(!maps && !lookups && !sends && !mailbox_reads); + } + for (int p = 13; p <= 15; ++p) { + fixture(); parameter(p, 1); assert(stage() == -EOPNOTSUPP); + assert(!maps && !lookups && !sends && !mailbox_reads); + } + // Offset/resource/property/ref/registration failures retain useful negatives. + const int cases[][5] = { + {5,-ENOENT,-ENOENT,0,0}, {16,-EIO,-EIO,0,0}, {17,-EINVAL,-EINVAL,0,0}, + {18,3,-EINVAL,0,0}, {3,0,-ENODEV,0,0}, {11,0,-ENODEV,1,0}, + {4,0,-ENOMEM,1,0}, {7,0,-EOPNOTSUPP,1,1}, {19,1,-EBUSY,1,1}, + {9,1,-EBUSY,0,0}, + }; + for (unsigned i = 0; i < sizeof(cases)/sizeof(cases[0]); ++i) { + fixture(); parameter(cases[i][0], cases[i][1]); + assert(stage() == cases[i][2]); assert(put_refs == cases[i][3]); assert(unmaps == cases[i][4]); + assert(!sends && !mailbox_reads); + } + fixture(); sep_pm_commit.sent = sep_pm_commit.answered = sep_pm_commit.gate_valid = true; + sep_pm_commit.reply_result = 0; sep_pm_commit.answer_us = 90; sep_pm_commit.state = 42; + assert(stage() == 0); assert(!sep_pm_commit.sent && !sep_pm_commit.answered); + assert(!sep_pm_commit.gate_valid && sep_pm_commit.gate_ret == -EINVAL); + assert(sep_pm_commit.reply_result == 0xff && !sep_pm_commit.answer_us && !sep_pm_commit.state); + assert(!sends && !mailbox_reads); assert(sep_pm_commit_finish(&dart) == -ENOENT); + assert(sep_pm_commit.registered); assert(sep_pm_commit_finish(&device) == 0); + assert(put_refs == 1 && unmaps == 1 && !apple_sleep_enclave.dev); + assert(sep_pm_commit_finish(&device) == -ENOENT); +} +static void test_unsent(void) { + for (int scenario = 0; scenario < 7; ++scenario) { + fixture(); assert(stage() == 0); reply(0); int expected; + switch (scenario) { + case 0: bulk_error = -EIO; expected = -EIO; break; + case 1: bulk_error = -ETIMEDOUT; expected = -ETIMEDOUT; break; + case 2: tx_ready = false; expected = -ETIMEDOUT; break; + case 3: prepare_error = -EIO; expected = -EIO; break; + case 4: pending_wake = true; expected = -EBUSY; break; + case 5: sleep_refuse_before_send = true; expected = -ECANCELED; break; + default: send_error = -ENODEV; expected = -ENODEV; break; + } + assert(sep_pm_commit_stop(&sep_pm_commit) == expected); balanced(); + assert(!sends && !sep_pm_commit.sent && !closes && !message_reads); + assert(!sep_pm_commit.answered && !sep_pm_commit.gate_valid); + assert(restores == (scenario >= 4)); assert(prepares == (scenario >= 3)); + if (scenario == 5) assert(!sleep_refuse_before_send); + assert(sep_pm_commit_finish(&device) == 0); + assert(put_refs == 1 && unmaps == 1 && !apple_sleep_enclave.dev); + } +} +static void test_sent(void) { + fixture(); assert(stage() == 0); late_wake = true; bulk_error = 3; + queue[queued++] = (struct apple_mbox_msg){0x7777, 0}; reply(0); + assert(sep_pm_commit_stop(&sep_pm_commit) == 0); balanced(); + assert(sends == 1 && closes == 1 && forwarded == 1 && message_reads == 2); + assert(pending_wake); // A wake after the send decision cannot cancel this stop. + assert(sep_pm_commit.answered && !sep_pm_commit.reply_result && sep_pm_commit.gate_valid); + assert(!sep_pm_commit.gate_ret && !restores); real_wake = true; + assert(sep_pm_commit_finish(&device) == 1); assert(put_refs == 1 && unmaps == 1); + + fixture(); assert(stage() == 0); one_error = -ENODEV; expect_stop_fatal("ret -19 "); assert(!closes); + fixture(); assert(stage() == 0); one_error = -EOPNOTSUPP; + expect_stop_fatal("ret -95 "); assert(!closes); + fixture(); assert(stage() == 0); reply(7); expect_stop_fatal("ret -121 "); + assert(sep_pm_commit.answered && sep_pm_commit.reply_result == 7 && !closes); + fixture(); assert(stage() == 0); early_gate = 1; + expect_stop_fatal("ret -112 "); assert(!sep_pm_commit.answered && !closes && !mailbox_reads); + fixture(); assert(stage() == 0); expect_stop_fatal("ret -110 "); + assert(!sep_pm_commit.answered && !closes); + fixture(); assert(stage() == 0); reply(0); close_gate = 0; + expect_stop_fatal("did not gate after send: -110"); + assert(closes == 1 && sep_pm_commit.gate_valid && sep_pm_commit.gate_ret == -ETIMEDOUT); + + fixture(); assert(stage() == 0); reply(0); assert(sep_pm_commit_stop(&sep_pm_commit) == 0); + fatal_armed = true; + if (setjmp(fatal_env) == 0) { + (void)sep_pm_commit_finish(&device); + assert(0 && "sent finish requires physical wake"); + } + fatal_armed = false; assert(strstr(fatal_message, "without a physical sleep/wake")); + assert(sep_pm_commit.registered && !put_refs && !unmaps && apple_sleep_enclave.dev); +} +static void test_provider(void) { + for (int p = 13; p <= 15; ++p) { + fixture(); parameter(p, 1); assert(!apple_mbox_poll_one_supported(&mbox)); + assert(apple_mbox_poll_one(&mbox) == -EOPNOTSUPP); + assert(!mailbox_reads && !message_reads && !forwarded); balanced(); + } + fixture(); mbox.removing = true; assert(apple_mbox_poll_one(&mbox) == -ENODEV); + assert(!mailbox_reads); balanced(); + fixture(); mbox.rx = NULL; assert(apple_mbox_poll_one(&mbox) == 0); assert(!mailbox_reads); + fixture(); assert(apple_mbox_poll_one(&mbox) == 0); + assert(mailbox_reads == 1 && !message_reads && !forwarded); balanced(); + fixture(); queue[queued++] = (struct apple_mbox_msg){0x7777, 0}; + queue[queued++] = (struct apple_mbox_msg){0x8888, 0}; + assert(apple_mbox_poll_one(&mbox) == 1); + assert(consumed == 1 && forwarded == 1 && mailbox_reads == 1 && message_reads == 1); balanced(); + assert(apple_mbox_poll_one(&mbox) == 1); + assert(consumed == 2 && forwarded == 2 && mailbox_reads == 2 && message_reads == 2); balanced(); +} +static void test_poll_contract(void) { + fixture(); assert(sep_pm_wait_gated(&device, 5) == 1); int n = reads; assert(n > 1); + fixture(); gate_after = n; assert(sep_pm_wait_gated(&device, 5) == 0 && reads == n); + fixture(); gate_after = n + 1; assert(sep_pm_wait_gated(&device, 5) == 1 && reads == n); + for (int e = 0; e < 2; ++e) { + int err = e ? -ETIMEDOUT : -EIO; + fixture(); read_error = err; assert(sep_pm_wait_gated(&device, 5) == err); + fixture(); read_error = err; read_error_after = n; + assert(sep_pm_wait_gated(&device, 5) == err && reads == n); + fixture(); read_error = err; assert(noirq() == err); + } + fixture(); domain_error = -ENOENT; assert(sep_pm_wait_gated(&device, 5) == -ENOENT); + assert(noirq() == 0); + fixture(); domain_error = -EIO; assert(noirq() == -EIO); + fixture(); assert(noirq() == -ETIMEDOUT); + fixture(); gate_after = 1; assert(noirq() == 0); + fixture(); firmware = false; assert(noirq() == 0 && !lookups); + fixture(); sep_pm_told = NULL; assert(noirq() == 0 && !lookups); + fixture(); sep_pm_told = no; assert(noirq() == 0 && !lookups); +} +int main(void) { + test_prepare_and_cleanup(); test_unsent(); test_provider(); test_sent(); test_poll_contract(); + puts("SEP C commit controls passed"); return 0; +} +#endif +''' + +RUST_PREFIX = r''' +#![allow(dead_code, unused_imports, unused_variables)] +use std::cell::{Cell, RefCell, RefMut}; +use std::ffi::{c_int, c_void}; +use std::sync::atomic::{AtomicU32, Ordering, Ordering::Relaxed}; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] struct Error(i32); +impl Error { fn from_errno(n: i32) -> Self { assert!(n < 0); Self(n) } } +type Result = std::result::Result; +const EBUSY: Error = Error(-16); const ENODEV: Error = Error(-19); +const ENOMEM: Error = Error(-12); const EIO: Error = Error(-5); +const ENOENT: Error = Error(-2); const EINVAL: Error = Error(-22); +const ETIMEDOUT: Error = Error(-110); const EOPNOTSUPP: Error = Error(-95); +macro_rules! dev_err { ($($a:tt)*) => {{}}; } +macro_rules! dev_warn { ($($a:tt)*) => {{}}; } +macro_rules! dev_dbg { ($($a:tt)*) => {{}}; } +macro_rules! dev_info { ($($a:tt)*) => {{}}; } +extern "C" { + fn fixture(); fn parameter(which: i32, value: i32); fn counter(which: i32) -> i32; + fn device_pointer() -> *mut c_void; fn mbox_pointer() -> *mut c_void; +} +thread_local! { + static REAL: Cell = const { Cell::new(false) }; + static EVENTS: RefCell> = const { RefCell::new(Vec::new()) }; + static RING: Cell> = const { Cell::new(Ok(())) }; +} +fn event(n: u8) { EVENTS.with(|v| v.borrow_mut().push(n)); } +mod bindings { pub unsafe fn pm_resume_via_firmware() -> bool { super::REAL.with(|v| v.get()) } } +mod kernel { + pub mod device { + pub struct Device; + impl Device { pub fn as_raw(&self) -> *mut std::ffi::c_void { unsafe { crate::device_pointer() } } } + } + pub mod bindings { #[allow(non_camel_case_types)] pub enum apple_mbox {} } + pub mod soc { pub mod apple { pub mod mailbox { + #[derive(Clone, Copy)] pub struct Message { pub msg0: u64, pub msg1: u32 } + } } } + pub mod time { + pub struct Delta; impl Delta { pub fn from_millis(_: i64) -> Self { Self } } + pub mod delay { pub fn fsleep(_: super::Delta) {} } + } +} +mod time { pub fn msecs_to_jiffies(ms: u32) -> u64 { ms.into() } } +mod proto { + pub const EP_CONTROL: u32 = 0; pub const CONTROL_TIMEOUT_MS: u32 = 5000; + pub const TAG_SLEEP: u32 = 0; pub const SLEEP_REPLY: u64 = 0x1234; + pub const SLEEP_REPLY_MASK: u64 = 0xffffff; + pub const WAKE_KICK: crate::kernel::soc::apple::mailbox::Message = + crate::kernel::soc::apple::mailbox::Message { msg0: 0x99, msg1: 0 }; + pub fn op_sleep() {} + pub fn encode_control(_: &(), _: u32) -> crate::kernel::soc::apple::mailbox::Message { + crate::kernel::soc::apple::mailbox::Message { msg0: 0x55, msg1: 0 } + } +} +mod control { + pub fn wait_idle bool, D: FnMut(u32)>(_: u32, _: u32, mut f: F, _: D) -> bool { !f() } +} +struct Lock(RefCell); +impl Lock { fn lock(&self) -> RefMut<'_, T> { self.0.borrow_mut() } } +struct Flag(Cell); +impl Flag { fn xchg(&self, v: bool, _: Ordering) -> bool { self.0.replace(v) } } +struct Control { busy: bool, wake_ok: bool } +impl Control { + fn requests_in_flight(&self) -> bool { self.busy } + fn wake_reset(&mut self) { event(1); } + fn wake_complete(&self) -> bool { self.wake_ok } +} +enum CondVarTimeoutResult { Woken { jiffies: u64 }, Signal { jiffies: u64 }, Timeout } +struct Wq; +impl Wq { + fn wait_interruptible_timeout(&self, _: &mut RefMut<'_, Control>, _: u64) -> CondVarTimeoutResult { + CondVarTimeoutResult::Timeout + } +} +struct Mbox; +impl Mbox { + fn as_raw(&self) -> *mut kernel::bindings::apple_mbox { unsafe { mbox_pointer().cast() } } + fn set_recv_enabled(&self, enabled: bool) { assert!(enabled); event(3); } + fn hold_rx(&self) { event(5); } + fn release_rx(&self) { event(2); } +} +const PHASE_READY: u32 = 1; +struct SepData { + dev: kernel::device::Device, phase: AtomicU32, control: Lock, mbox: Lock>, + gate: Cell, sleep_rx_held: Flag, endpoint: Cell, control_wq: Wq, +} +impl SepData { + fn new() -> Self { + Self { dev: kernel::device::Device, phase: AtomicU32::new(PHASE_READY), + control: Lock(RefCell::new(Control { busy: false, wake_ok: true })), + mbox: Lock(RefCell::new(Some(Mbox))), gate: Cell::new(pm::SleepGate::Open), + sleep_rx_held: Flag(Cell::new(false)), endpoint: Cell::new(true), control_wq: Wq } + } + fn endpoint_present(&self, _: u32) -> bool { self.endpoint.get() } + fn sleep_gate(&self) -> pm::SleepGate { self.gate.get() } + fn set_sleep_gate(&self, g: pm::SleepGate) { self.gate.set(g); } + fn ring(_: &Mbox, msg: kernel::soc::apple::mailbox::Message) -> Result<()> { + assert_eq!(msg.msg0, proto::WAKE_KICK.msg0); event(4); RING.with(|v|v.get()) + } +} +''' + +RUST_TESTS = r''' +fn reset() { + unsafe { fixture() }; REAL.with(|v|v.set(false)); + EVENTS.with(|v|v.borrow_mut().clear()); RING.with(|v|v.set(Ok(()))); +} +fn param(n: i32, v: i32) { unsafe { parameter(n, v) } } +fn count(n: i32) -> i32 { unsafe { counter(n) } } +fn events() -> Vec { EVENTS.with(|v|v.borrow().clone()) } +fn clear_events() { EVENTS.with(|v|v.borrow_mut().clear()); } +fn real_wake() { REAL.with(|v|v.set(true)); param(12,1); } +#[test] +fn staging_errors_preserve_errno_and_cleanup() { + for (p,v,e,puts,unmaps) in [ + (4,0,ENOMEM,1,0),(3,0,ENODEV,0,0),(5,-2,ENOENT,0,0), + (9,1,EBUSY,0,0),(11,0,ENODEV,1,0),(16,-5,EIO,0,0), + (17,-22,EINVAL,0,0),(18,3,EINVAL,0,0),(19,1,EBUSY,1,1), + (7,0,EOPNOTSUPP,1,1), + ] { + reset(); param(p,v); let s=SepData::new(); + assert_eq!(s.sleep_enter(),Err(e)); assert_eq!(s.gate.get(),pm::SleepGate::Open); + assert!(!s.sleep_rx_held.0.get()); assert_eq!(events(),Vec::::new()); + assert_eq!(count(0),puts); assert_eq!(count(2),unmaps); assert_eq!(count(6),0); + } +} +#[test] +fn unsupported_is_refused_without_early_sleep() { + for p in [0,1,2,13,14,15] { + reset(); param(p,if p < 3 {0} else {1}); let s=SepData::new(); + assert_eq!(s.sleep_enter(),Err(EOPNOTSUPP)); + assert_eq!(s.gate.get(),pm::SleepGate::Open); assert!(!s.sleep_rx_held.0.get()); + assert_eq!(count(6),0); assert_eq!(count(7),0); assert_eq!(count(1),0); + assert_eq!(count(4),0); assert!(events().is_empty()); + } +} +#[test] +fn staging_holds_rx_and_unsent_unwind_only_releases() { + reset(); let s=SepData::new(); assert_eq!(s.sleep_enter(),Ok(())); + assert_eq!(s.gate.get(),pm::SleepGate::Draining); assert!(s.sleep_rx_held.0.get()); + assert_eq!(count(3),1); assert_eq!(count(6),0); assert_eq!(events(),vec![5]); + s.sleep_exit(); assert_eq!(s.gate.get(),pm::SleepGate::Open); + assert!(!s.sleep_rx_held.0.get()); assert_eq!(events(),vec![5,2]); + assert_eq!(count(0),1); assert_eq!(count(2),1); assert_eq!(count(3),0); + assert_eq!(pm::commit_finish(&s.dev),None); + s.sleep_exit(); assert_eq!(events(),vec![5,2]); +} +#[test] +fn admission_and_missing_mailbox_refuse_before_staging() { + for mode in 0..6 { + reset(); let s=SepData::new(); + match mode { + 0 => s.phase.store(0,Relaxed), 1 => s.endpoint.set(false), + 2 => s.control.lock().busy=true, 3 => *s.mbox.lock()=None, + 4 => s.gate.set(pm::SleepGate::Draining), _ => s.gate.set(pm::SleepGate::Told), + } + assert_eq!(s.sleep_enter(),Err(if mode==3 {ENODEV} else {EBUSY})); + assert_eq!(count(1),0); assert_eq!(count(6),0); assert!(!s.sleep_rx_held.0.get()); + assert!(events().is_empty()); + if mode < 4 { assert_eq!(s.gate.get(),pm::SleepGate::Open); } + } +} +#[test] +fn only_physical_wake_runs_the_actual_kick_in_arm_order() { + reset(); let s=SepData::new(); assert_eq!(s.sleep_enter(),Ok(())); + // Current-attempt sent guard is exercised only in C, with no Rust frames. + param(20,1); real_wake(); clear_events(); s.sleep_exit(); + assert_eq!(events(),vec![1,2,3,4]); assert_eq!(s.gate.get(),pm::SleepGate::Open); + assert!(!s.sleep_rx_held.0.get()); assert_eq!(count(3),0); + reset(); let s=SepData::new(); s.gate.set(pm::SleepGate::Told); + s.sleep_rx_held.0.set(true); s.sleep_exit(); + assert_eq!(s.gate.get(),pm::SleepGate::Lost); assert!(s.sleep_rx_held.0.get()); + assert!(events().is_empty()); assert_eq!(count(5),0); assert_eq!(count(0),0); +} +#[test] +fn failed_real_wake_stays_lost_and_refuses_new_firmware_sleep() { + for mode in 0..4 { + reset(); let s=SepData::new(); s.gate.set(pm::SleepGate::Told); + s.sleep_rx_held.0.set(true); real_wake(); + match mode { + 0 => param(6,-5), 1 => s.control.lock().wake_ok=false, + 2 => RING.with(|v|v.set(Err(EIO))), _ => *s.mbox.lock()=None, + } + s.sleep_exit(); assert_eq!(s.gate.get(),pm::SleepGate::Lost); + assert!(s.sleep_rx_held.0.get()); + if mode==0 { assert!(events().is_empty()); } // clear error precedes RX release + clear_events(); assert_eq!(s.sleep_enter(),Err(EIO)); assert!(events().is_empty()); + assert_eq!(count(6),0); + REAL.with(|v|v.set(false)); param(12,0); let reads=count(5); let refs=count(0); s.sleep_exit(); + assert_eq!(count(5),reads); assert_eq!(count(0),refs); + assert_eq!(s.gate.get(),pm::SleepGate::Lost); assert!(s.sleep_rx_held.0.get()); + assert!(events().is_empty()); + } + reset(); let s=SepData::new(); s.gate.set(pm::SleepGate::Lost); + s.sleep_rx_held.0.set(true); assert_eq!(s.sleep_enter(),Err(EIO)); + s.sleep_exit(); assert!(events().is_empty()); assert!(s.sleep_rx_held.0.get()); + assert_eq!(count(5),0); assert_eq!(count(0),0); +} +#[test] +fn lost_and_gate_permissions_are_preserved() { + for gate in [pm::SleepGate::Open,pm::SleepGate::Draining,pm::SleepGate::Told,pm::SleepGate::Lost] { + assert_eq!(pm::SleepGate::from_u32(gate as u32),gate); + assert_eq!(gate.may_request(),gate==pm::SleepGate::Open); + assert_eq!(gate.may_reply(),matches!(gate,pm::SleepGate::Open|pm::SleepGate::Draining)); + assert_eq!(gate.needs_wake(),matches!(gate,pm::SleepGate::Told|pm::SleepGate::Lost)); + } +} +''' + + +def sources(source, contracts, mutation): + """Assemble source only. Compilation/execution belongs exclusively to main.""" + pm = (source / 'drivers/soc/apple/pm.rs').read_text() + sep = (source / 'drivers/soc/apple/sep.rs').read_text() + shim = (source / 'drivers/soc/apple/pm_shim.c').read_text() + mailbox = (source / 'drivers/soc/apple/mailbox.c').read_text() + sleep = (source / 'drivers/soc/apple/sleep.c').read_text() + iopoll = (contracts / 'include/linux/iopoll.h').read_text() + names = { + 'shim': ['sep_pm_wait_gated', 'sep_pm_clear_dev_disable', 'sep_pm_dart_restore', + 'sep_pm_commit_rx', 'sep_pm_commit_take_rx', 'sep_pm_commit_stop', + 'sep_pm_commit_prepare', 'sep_pm_commit_finish', 'sep_pm_suspend_noirq'], + 'mailbox': ['apple_mbox_poll_one_supported', 'apple_mbox_poll_one'], + 'sleep': ['apple_sleep_commit_stops_coprocs', 'apple_sleep_enclave_register', + 'apple_sleep_enclave_unregister'], + 'pm': ['leave_for_commit', 'commit_finish', 'clear_dev_disable', 'dart_restore'], + 'sep': ['sleep_enter', 'sleep_leave_for_commit', 'sleep_hold_rx', 'sleep_release_rx', + 'sleep_exit', 'sleep_kick'], + } + texts = dict(shim=shim, mailbox=mailbox, sleep=sleep, pm=pm, sep=sep) + bodies = {n: function(texts[k], n) for k, group in names.items() for n in group} + if mutation: + name, before, after = MUTATIONS[mutation] + bodies[name] = replace_once(bodies[name], before, after) + macros = ['poll_timeout_us', 'read_poll_timeout', 'poll_timeout_us_atomic', + 'read_poll_timeout_atomic', 'readx_poll_timeout_atomic', 'readl_poll_timeout_atomic'] + c = C_PREFIX + '\n' + '\n'.join(macro(iopoll, n) for n in macros) + c += '\n' + '\n'.join(re.search(r'^#define ' + n + r'\s+.*$', shim, re.M).group() + for n in ['SEP_PM_POLL_US', 'SEP_PM_TIMEOUT_US']) + start = shim.index('static struct sep_pm_commit {') + c += '\n' + shim[start:shim.index('} sep_pm_commit;', start) + len('} sep_pm_commit;')] + # Use the actual provider helper, with only removal/capability dependency state + # injected by send for the post-send negative-poll scenarios. + c += '\n' + '\n'.join(bodies[n] for group in ['mailbox', 'sleep', 'shim'] for n in names[group]) + c += '\n' + C_CONTROLS + gate_start = pm.index('#[derive(Clone, Copy, Debug, PartialEq, Eq)]') + gate_end = pm.index('/// Stages `request`', gate_start) + rust = RUST_PREFIX + '\nmod pm {\nuse crate::*;\n' + rust += '''extern "C" { + fn sep_pm_commit_prepare(dev:*mut c_void, mbox:*mut c_void, a:u64, b:u32, c:u64, d:u64)->c_int; + fn sep_pm_commit_finish(dev:*mut c_void)->c_int; + fn sep_pm_clear_dev_disable(dev:*mut c_void)->c_int; + fn sep_pm_dart_restore(dev:*mut c_void); + } +''' + rust += pm[gate_start:gate_end] + rust += '\n'.join(bodies[n] for n in names['pm']) + '\n}\nimpl SepData {\n' + rust += '\n'.join(bodies[n] for n in names['sep']) + '\n}\n' + RUST_TESTS + return c, rust + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--source', type=Path, default=ROOT) + parser.add_argument('--contracts', type=Path) + parser.add_argument('--mutation', choices=MUTATIONS) + args = parser.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + args.out.mkdir(parents=True, exist_ok=True) + c, rust = sources(args.source, args.contracts or args.source, args.mutation) + cpath, rpath = args.out / 'control.c', args.out / 'control.rs' + obj, cbinary, binary = args.out / 'control.o', args.out / 'control-c', args.out / 'control' + cpath.write_text(c) + rpath.write_text(rust) + cc = ['cc', '-std=gnu11', '-Wall', '-Wextra', '-Wno-unused-parameter'] + # Finish all compilation before executing controls: setup failures cannot kill mutants. + subprocess.run(cc + ['-c', str(cpath), '-o', str(obj)], check=True) + subprocess.run(cc + ['-DCONTROL_MAIN', str(cpath), '-o', str(cbinary)], check=True) + subprocess.run(['rustc', '--edition=2021', '--test', str(rpath), '-C', + f'link-arg={obj.resolve()}', '-o', str(binary)], check=True) + for command in [[str(cbinary.resolve())], [str(binary.resolve()), '--test-threads=1']]: + result = subprocess.run(command, check=False) + if result.returncode: + # Keep the assertion output, without a Python traceback masking a real kill. + raise SystemExit(result.returncode) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 840d961ff8cbb..4c6b2952484f8 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -159,6 +159,9 @@ struct apple_pmp_report { /* Requests held until the PMP runs; T6030 does not defer them. */ u64 shadow, known; bool defer; + /* Firmware-driven sleep: the request word at its start, the DVFS words kept. */ + u64 sleep_request; + u64 *dvfs_shadow; unsigned int polls; struct delayed_work ready_work; struct apple_pmp_ane ane; @@ -628,6 +631,11 @@ static void *devm_kzalloc(struct device *d, size_t n, int g) allocations[allocation_count++] = v; return v; } + +static void *devm_kcalloc(struct device *d, size_t n, size_t size, int g) +{ + return devm_kzalloc(d, n * size, g); +} #define kzalloc_obj(v) calloc(1, sizeof(v)) #define kfree free static char *kstrdup(const char *s, int g) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index 69f832bff9cb3..468edbf9446a1 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -126,6 +126,8 @@ static bool passive_present, passive_lockable, passive_data; static const char *machine; static const void *pmp_data; static struct device *pmp_device; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; diff --git a/tools/aurora-wake/bt-wake-cycle.sh b/tools/aurora-wake/bt-wake-cycle.sh index 3c10162256e97..2eb419b345823 100755 --- a/tools/aurora-wake/bt-wake-cycle.sh +++ b/tools/aurora-wake/bt-wake-cycle.sh @@ -17,9 +17,21 @@ set -u +# The PMU RTC, which can wake the system: rtc0 on J700, rtc1 on others. +pmu_rtc() { + local name + for name in /sys/class/rtc/rtc*/name; do + [[ $(cat "$name" 2> /dev/null) == apple-pmu-rtc* ]] || continue + name=${name%/name} + echo "${name##*/}" + return + done + echo rtc0 +} + mode= secs=120 -rtc=rtc1 +rtc=$(pmu_rtc) addr= out=/var/tmp/bt-wake/cycle-$(date +%Y%m%d-%H%M%S) bdf=0000:01:00.1 diff --git a/tools/aurora-wake/bt-wake-power.sh b/tools/aurora-wake/bt-wake-power.sh index af5d49dadb216..be6d1eb06c8f6 100755 --- a/tools/aurora-wake/bt-wake-power.sh +++ b/tools/aurora-wake/bt-wake-power.sh @@ -9,6 +9,17 @@ # - the battery energy used, when running on battery (energy_now). # Usage: bt-wake-power.sh [-s secs] [-n pairs] [-o outdir] set -u +# The PMU RTC, which can wake the system: rtc0 on J700, rtc1 on others. +pmu_rtc() { + local name + for name in /sys/class/rtc/rtc*/name; do + [[ $(cat "$name" 2> /dev/null) == apple-pmu-rtc* ]] || continue + name=${name%/name} + echo "${name##*/}" + return + done + echo rtc0 +} secs=300 pairs=2 out=/var/tmp/bt-wake/power-$(date +%Y%m%d-%H%M%S) @@ -19,6 +30,7 @@ done dev=/sys/bus/pci/devices/0000:01:00.1 bat=/sys/class/power_supply/macsmc-battery irq=$(awk -F: '/mt7932-bt/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) +rtc=$(pmu_rtc) mkdir -p "$out" orig=$(cat "$dev/power/wakeup") trap 'echo "$orig" > "$dev/power/wakeup"' EXIT @@ -42,7 +54,7 @@ for p in $(seq 1 "$pairs"); do e0=$(cat "$bat/energy_now") st=$(cat "$bat/status") t0=$(date +%s.%N) - rtcwake -d rtc1 -m freeze -s "$secs" > /dev/null 2>&1 + rtcwake -d "$rtc" -m freeze -s "$secs" > /dev/null 2>&1 t1=$(date +%s.%N) b1=$(irq_sum "$irq"); a1=$(irq_sum all); e1=$(cat "$bat/energy_now") dmesg > "$out/dmesg-$p-$mode.txt" diff --git a/tools/aurora-wake/s2idle-rtc-regression.sh b/tools/aurora-wake/s2idle-rtc-regression.sh index 89a131aca0b59..74697560040db 100755 --- a/tools/aurora-wake/s2idle-rtc-regression.sh +++ b/tools/aurora-wake/s2idle-rtc-regression.sh @@ -3,7 +3,7 @@ # # Unattended suspend-to-idle regression test for Apple silicon (J700 and others). # -# Suspends with an RTC alarm (rtc1, apple-pmu-rtc, by default) several times and +# Suspends with an RTC alarm (apple-pmu-rtc by default) several times and # checks after every resume that: # - the time actually spent suspended (CLOCK_BOOTTIME minus CLOCK_MONOTONIC) # lasted until the alarm: no spurious wakeup, e.g. from a running xHC, @@ -24,9 +24,21 @@ set -u +# The PMU RTC, which can wake the system: rtc0 on J700, rtc1 on others. +pmu_rtc() { + local name + for name in /sys/class/rtc/rtc*/name; do + [[ $(cat "$name" 2> /dev/null) == apple-pmu-rtc* ]] || continue + name=${name%/name} + echo "${name##*/}" + return + done + echo rtc0 +} + cycles=3 secs=20 -rtc=rtc1 +rtc=$(pmu_rtc) out= ignore= dry_run=0 diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py index 14c837ceeb5f3..95c02e87ddc02 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -19,7 +19,8 @@ def oslog(root): #define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctzll(mask)) #define APPLE_RTKIT_EP_OSLOG 8 struct buffer { unsigned sentinel; unsigned size; }; -struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack; }; +struct apple_rtkit { void *dev; struct buffer oslog_buffer; bool oslog_inherited, sync_syslog_ack, oslog_echo; u64 oslog_owed; }; +#define cmpxchg(p,o,n) __sync_val_compare_and_swap((p),(o),(n)) static unsigned warnings, notifications, requests, acks, errors; static u64 observed; static bool neo, m5; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 6c140662b6967..f7eafd88b68ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -146,6 +146,15 @@ def atc_restore_helpers(source): return atc_function(source, 'atcphy_setup_pipehandler') +def atc_common_state_helpers(source): + """What atcphy_restore_after_pd_off() calls to drop the common block's DisplayPort and + tunnel state after a power-down. Older sources do that inline: ''. + """ + if 'atcphy_forget_common_state(' not in atc_function(source, 'atcphy_restore_after_pd_off'): + return '' + return atc_function(source, 'atcphy_forget_common_state') + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 4c9f0ac4faad6..67d07ef2c2a63 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -86,12 +86,21 @@ struct dwc3_apple { void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending, shutting_down; + /* Stopped for a deep sleep, and the state to bring back. */ + bool sleep_stopped; + enum dwc3_apple_state sleep_state; + /* Votes sleep DDR while the host is up; the fixtures leave it off. */ + bool sleep_ddr; bool rpm_held, host_only; /* T8132 only: runtime PM before the first core probe; never set here. */ bool pre_core_rpm; bool dt_dis_u3_susphy; unsigned long usb2_power_off_pending, usb3_power_off_pending; }; +static void __attribute__((unused)) apple_sleep_usb_slp_ddr(bool on) +{ + (void)on; +} static unsigned int polls, fail_poll, reset_calls, reset_writes, core_writes; static unsigned int host_exits, clocks_disabled, callback_calls; static unsigned int gadget_exits, event_cleanups, legacy_suspends, @@ -329,6 +338,11 @@ static void dwc3_apple_setup_cio(struct dwc3_apple *a) (void)a; setup_cio++; } +/* The hibernation block's register setup; teardown order doesn't depend on it. */ +static void __attribute__((unused)) dwc3_apple_setup_hibernation(struct dwc3_apple *a) +{ + (void)a; +} static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, enum phy_mode *m) { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index f459d57f5e475..4dcbaff1d59fd 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -792,6 +792,8 @@ def poweroff_replug(root): { (void)d; (void)generation; return 0; } static void dcp_external_crtc_released(struct apple_dcp *d) { (void)d; } static bool dcp_uses_soft_dpms(struct apple_dcp *d) { return d->soft_dpms; } +/* The T8140 display power cycle has its own test on hardware; here it only has to be reached. */ +static void dcp_dpms_power_off(struct apple_dcp *d) { (void)d; } static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return !d->typec_output; } static void __attribute__((unused)) iomfb_blank_h17p(struct apple_dcp *d) { (void)d; assert(!"unexpected internal-panel blank in external-link test"); } @@ -1068,6 +1070,9 @@ def fn(n, name): for name in ('atcphy_mode_configuration', 'atcphy_hw', 'atcphy_dp_link_rate_configuration'): decls.append(block(files['atc'], files['atc'].index('struct ' + name + ' {')) + ';') decls.append(block(files['state'], files['state'].index('enum dwc3_apple_state {')) + ';') + role = 'static inline enum dwc3_apple_state dwc3_apple_state_for_role(' + if role in files['state']: + decls.append(block(files['state'], files['state'].index(role))) start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') end = files['atc'].index('\n};', start) + 3 decls.append(files['atc'][start:end]) @@ -1087,7 +1092,7 @@ def fn(n, name): 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', - 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_restore_after_pd_off', + 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_forget_common_state', 'atcphy_restore_after_pd_off', 'atcphy_usb2_init', 'atcphy_usb3_power_off', 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), @@ -1195,6 +1200,7 @@ def block(marker): int iomfb_method_profile,firmware_compat; const struct resource *firmware_clock; u32 firmware_display_windows,firmware_scratch,firmware_request; + bool s2r,s2r_ext; }; struct apple_dcp { struct device *dev; struct apple_dcp_hw_data hw; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index a56a3a3ccfbff..b2e1d718404da 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -4,7 +4,7 @@ from pathlib import Path import re from common import (block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE, - atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, + atc_function, ATC_SPLIT, ATC_T8103_LIVE, atc_restore_helpers, atc_common_state_helpers, atc_cio_helpers) def parking(root): @@ -36,6 +36,8 @@ def function(source, name): /* T8132-only paths are covered by drivers/apple/atc-t8132-usb3.py. */ bool t8132_ss, t8132_common_owned, t8132_failed, cio_tunables; struct { void *usb3_dp; } tunables; + /* A sleep took the USB host's domains down; never set here. */ + bool sleep_dead; const struct atcphy_hw *hw; struct { unsigned char *core, *lpdptx, *pipehandler, *usb2phy; } regs; @@ -181,6 +183,7 @@ def function(source, name): parts.append(function(atc, name)) parts.append(dp) parts.append(atc_restore_helpers(atc)) + parts.append(atc_common_state_helpers(atc)) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_set_orientation', diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 92ccae9ed052f..64889dc06683c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -428,6 +428,8 @@ def ownership(root, usb_unreachable=False): helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n' helpers+=''.join(atc_function(s,name) for name in ('atcphy_park_retry_delay','atcphy_park_retry','atcphy_park_due','atcphy_mux_change')) + # The mux callback's body, split out so the wake can set the mode it had. + helpers+=''.join(atc_function(s,name) for name in ('atcphy_mux_done','atcphy_mux_update_to')) helpers+=ns['function'](s,'atcphy_mux_update')+'\n' # Lane fixtures provide the target mapping and locked body; use the real callback here. # The mux callback powers an idle-power-off PHY first; this PHY is not one. @@ -495,7 +497,7 @@ def function(text, name): mocks = r''' struct cd321x { struct tps6598x tps; - struct work update_work,resume_work; + struct work update_work,resume_work,rearm_work; int link_event,ready_event,restore_event; struct cd321x_pm_state pm; void *mux,*tbt_switch,*port_altmode_dp,*port_altmode_tbt; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index b293087d4917c..0ad3420e6fdb1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -95,7 +95,8 @@ def unit_for(source): struct dwc3 { bool sys_wakeup, dis_u3_susphy_quirk; struct platform_device *xhci; }; enum { DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_HOST, DWC3_APPLE_DEVICE }; /* pre_core_rpm is T8132 only: runtime PM before the first core probe; never set here. */ -struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy; }; +struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; bool pre_core_rpm, dt_dis_u3_susphy, shutting_down, sleep_ddr; }; +static void __attribute__((unused)) apple_sleep_usb_slp_ddr(bool on) { (void)on; } static struct dwc3_apple apple_context; #define to_dwc3_apple(d) container_of(d,struct dwc3_apple,dwc) static bool inside_host_remove; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 631237b2ac7b3..e63e5b4799a07 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -56,7 +56,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): actual=actual.replace('static struct apple_dcp_typec_port *sole_port(void) { return &port; }', 'static struct apple_dcp_typec_port *sole_port(void) { return &port; }\nstatic struct apple_dcp_typec_route * __attribute__((unused)) sole_route(void) { return &route; }',1) actual=actual.replace('bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;', 'struct typec_altmode *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;',1) - actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel;',1) + actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel,sleep_parked;',1) extra=r''' /* actual DCP leaves */ #define WRITE_ONCE(x,v) ((x)=(v)) diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c index 806148f64d1bf..8154098ee4157 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -19,6 +19,8 @@ struct device { void *of_node; }; static struct device owner; static struct device *pmp_device = &owner; static const void *pmp_data = &owner; +/* Set only while the PMP is stopped for a system sleep, which this test never does. */ +static bool pmp_asleep; static bool t6030, requested; static int ready, check, power_result[2], report_result[3], powers, reports; static char calls[16]; From 2a57a36f5f28c2666544c5140f8bbcd72445e4cc Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Tue, 10 Nov 2026 03:14:17 -0500 Subject: [PATCH 02/29] soc: apple: sleep: look for the debug link's contract from 50 ms after its cut The cut waited a fixed second after the port controller's cold reset before its first look at the contract, and in every logged sleep the contract was already back at that first look. That second was most of a cut sleep's entry. Look from 50 ms on instead. The contract reads back as before for a moment after the reset, so count it as back only once it was seen gone (no contract, or no answer while the controller restarts), or after the old second if it never seemed to go. Log both times. Signed-off-by: Diljit Singh --- drivers/soc/apple/sleep.c | 40 ++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 73befaeb87d1b..c51a71ce852be 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -163,7 +164,9 @@ struct aura_s2r_mailbox { #define HPM_CMD_SSPS 0x53505353 #define HPM_CMD_COLD_RESET 0x64696147 #define HPM_SSPS_S3 3 -#define HPM_RESET_MS 1000 +/* The first look after the reset, and how long a contract that never seemed to go is trusted to have gone. */ +#define HPM_RESET_FIRST_MS 50 +#define HPM_RESET_FLOOR_MS 1000 #define HPM_PAGE_POWER_STATUS 0x3f #define HPM_POWER_CONNECTED BIT(0) #define HPM_POWER_TYPEC_MODE GENMASK(3, 2) @@ -1811,26 +1814,38 @@ static int sleep_hpm_sleep(struct apple_sleep *s) * Left to finish across the commit, that sometimes failed: neither side had a contract or * saw the other, and no VDM got through until a replug. Returns the ms it took. */ -static int sleep_hpm_wait_contract(struct apple_sleep *s) +static int sleep_hpm_wait_contract(struct apple_sleep *s, int *gone_ms) { - bool wake = true; + ktime_t start = ktime_get(); + bool wake = true, up; u32 val = 0; int ms, ret; - for (ms = 0; ms < HPM_CONTRACT_MS; ms += 10) { + /* + * It reads back as before for a moment after the reset. Count it as back once it was + * seen gone (no contract, or no answer while the controller restarts), or after the + * floor if it never seemed to go. + */ + *gone_ms = -1; + mdelay(HPM_RESET_FIRST_MS); + for (;;) { + ms = ktime_ms_delta(ktime_get(), start); + if (ms >= HPM_CONTRACT_MS) + return -ETIMEDOUT; ret = wake ? sleep_hpm_cmd(s, SPMI_OP_WAKEUP, 0, NULL, NULL) : 0; if (!ret) ret = sleep_hpm_select(s, HPM_PAGE_POWER_STATUS); if (!ret) ret = sleep_hpm_cmd(s, SPMI_OP_EXT_READ(4), HPM_REG_DATA, NULL, &val); - if (!ret && (val & HPM_POWER_CONNECTED) && - FIELD_GET(HPM_POWER_TYPEC_MODE, val) == HPM_POWER_TYPEC_PD) + up = !ret && (val & HPM_POWER_CONNECTED) && + FIELD_GET(HPM_POWER_TYPEC_MODE, val) == HPM_POWER_TYPEC_PD; + if (!up && *gone_ms < 0) + *gone_ms = ms; + if (up && (*gone_ms >= 0 || ms >= HPM_RESET_FLOOR_MS)) return ms; wake = ret; mdelay(10); } - - return -ETIMEDOUT; } /* A live debug link holds the debug switch domain on. */ @@ -1851,7 +1866,7 @@ static bool apple_sleep_usb_down(struct apple_sleep *s) { u32 cmd = HPM_CMD_COLD_RESET; u32 enable; - int i, ret, ms; + int i, ret, ms, gone; if (s->usb_down_ran) return false; @@ -1873,11 +1888,10 @@ static bool apple_sleep_usb_down(struct apple_sleep *s) if (ret) break; WRITE_ONCE(sleep_debug_link_cut, true); - mdelay(HPM_RESET_MS); - ms = sleep_hpm_wait_contract(s); + ms = sleep_hpm_wait_contract(s, &gone); if (ms >= 0) { - dev_info(s->dev, "debug link: contract back %d ms after reset %d\n", - HPM_RESET_MS + ms, i); + dev_info(s->dev, "debug link: contract back %d ms after reset %d (gone at %d ms)\n", + ms, i, gone); break; } dev_warn(s->dev, "debug link: no contract with the partner after reset %d\n", i); From 38e9c6dede5482030da3e6d21ad241ba63376830 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Tue, 10 Nov 2026 03:14:18 -0500 Subject: [PATCH 03/29] mfd: macsmc: shorten the waits a cut sleep adds before its commit A sleep that cuts the debug link waits for the SMC's outbox to be quiet for 500 ms after the cut, then warns it late and leaves macOS's 770 ms before slpa. No message came in the quiet window in any logged sleep, and the sleeps that warn at .prepare, with no cable, commit 150-500 ms after the warning and run to their alarm (154 of 154). Use 100 ms of quiet and a 200 ms gap. Signed-off-by: Diljit Singh --- drivers/mfd/macsmc.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 1d0e4d58b68a6..fce9dc9527e66 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -683,11 +683,14 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); /* The gaps macOS leaves before the AP idle notice and after it. */ #define SMC_SLEEP_ARM_MS 6 #define SMC_SLEEP_IDLE_MS 30 -/* The shortest gap macOS left between slpw and slpa, in two traced sleeps. */ -#define SMC_SLEEP_WARN_MS 770 +/* + * Between a late warning (a sleep that cut the debug link) and slpa. macOS's shortest gap + * was 770 ms; the warning at .prepare leaves 150-500 ms, and those sleeps run to the alarm. + */ +#define SMC_SLEEP_WARN_MS 200 -/* After the debug link's cut. Its port and charger events keep coming for over a second. */ -#define SMC_OUTBOX_QUIET_MS 500 +/* After the debug link's cut. No message came in it in any logged sleep. */ +#define SMC_OUTBOX_QUIET_MS 100 #define SMC_OUTBOX_SETTLE_MS 5000 static int apple_smc_ap_power(struct apple_smc *smc, bool on) From 3224a675fa74371e28436474b4dad5e0f4c5638c Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Tue, 10 Nov 2026 17:15:04 -0500 Subject: [PATCH 04/29] soc: apple: sleep: give up on a debug link reset after 1.5 s In every logged cut sleep the partner's contract was back 660-704 ms after the port controller's cold reset, or not at all: 42 first resets never got it back within 6 s, and the next reset did, at about 0.7 s. Each of those waited the full 5 s before trying again. Wait 1.5 s per reset instead. Signed-off-by: Diljit Singh --- drivers/soc/apple/sleep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index c51a71ce852be..21d4b3889413b 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -171,7 +171,8 @@ struct aura_s2r_mailbox { #define HPM_POWER_CONNECTED BIT(0) #define HPM_POWER_TYPEC_MODE GENMASK(3, 2) #define HPM_POWER_TYPEC_PD 3 -#define HPM_CONTRACT_MS 5000 +/* A contract that comes back does by 0.7 s; one that doesn't is back after another reset. */ +#define HPM_CONTRACT_MS 1500 #define HPM_RESET_TRIES 3 /* A PMU register write on the sleep's SPMI bus */ From fbe6f6640ad51a8342cbb0dc3cf233f43d7ca548 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 00:23:16 -0500 Subject: [PATCH 05/29] dmaengine: apple-sio: stop the T8140 firmware at the sleep's commit A T8140 whose boot tables aren't the T6041 ones left the SIO unbound, so nothing idled it for a sleep. macOS idles it in the sleep's last step, 38 ms before the commit and just before the PMP: AP power idle, IOP power idle, then the mailbox's receive interrupt closed. Run the firmware without DMA service in that case, as T8140 did before the T6041 tables, and leave its idle and stop to the sleep's last step. A sleep refused before that step finds the firmware still running. Signed-off-by: Diljit Singh --- drivers/dma/apple-sio.c | 66 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 3b9d804f206d1..330bb7fa6b753 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -167,6 +167,10 @@ struct sio_hw_data { * uses its DMA here, and its service parameters haven't been traced. */ bool firmware_only; + /* Without the boot tables for DMA, still run the firmware so a sleep can stop it. */ + bool firmware_without_tables; + /* macOS idles this firmware in the sleep's last step: leave it running until then. */ + bool stop_at_commit; /* DMA arrays owned by the driver, if not protocol 9's */ const struct sio_dma_arrays *arrays; int (*check_boot_tables)(struct device *dev); @@ -182,6 +186,9 @@ struct sio_data { struct apple_rtkit *rtk; /* The firmware is stopped for a system sleep. */ bool asleep; + bool firmware_only; + bool commit_registered; + bool commit_used; u32 protocol; void *shmem; dma_addr_t shmem_iova; @@ -547,7 +554,9 @@ static int sio_t6041_check_boot_tables(struct device *dev) u32 size[ARRAY_SIZE(sio_t6041_params)] = {}; if (sio_t6041_check_node(dev->of_node, addr, size)) { - dev_err(dev, "boot tables are not T6041 SIO tables; leaving the SIO stopped\n"); + /* T8140 still runs its firmware, and says so once it does. */ + if (!of_device_is_compatible(dev->of_node, "apple,t8140-sio")) + dev_err(dev, "boot tables are not T6041 SIO tables; leaving the SIO stopped\n"); return -ENODEV; } return 0; @@ -652,6 +661,8 @@ static const struct sio_hw_data sio_hw_t6041 = { * (ADT dp-audioN dma-channels): [2, 0, 0x800, 0x800, 0x800, 0]. */ static const struct sio_hw_data sio_hw_t8140 = { + .firmware_without_tables = true, + .stop_at_commit = true, .protocol = 10, .fifo_threshold = 0x800, .cfg_word4 = 0x800, @@ -1925,6 +1936,19 @@ static int sio_send_dt_params(struct sio_data *sio) return dev_err_probe(sio->dev, err, "failed to read '%s'\n", propname); } +/* Interrupts off, one CPU: the idle pair and the closed mailbox. The caller clears the run bit. */ +static int sio_commit_stop(void *data) +{ + struct sio_data *sio = data; + unsigned int us; + int ret; + + ret = apple_rtkit_sleep_idle_polled(sio->rtk, sio->base + REG_CPU_CONTROL, &us); + apple_rtkit_mailbox_recv_off(sio->rtk); + + return ret ? 1 : 0; +} + static int sio_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -1932,6 +1956,7 @@ static int sio_probe(struct platform_device *pdev) const struct sio_hw_data *hw; struct sio_data *sio; struct dma_device *dma; + bool firmware_only; int nchannels; int err, i; @@ -1940,10 +1965,13 @@ static int sio_probe(struct platform_device *pdev) return -ENODEV; hw = match->data; /* Refuse incompatible firmware tables before DMA or hardware setup. */ + firmware_only = hw->firmware_only; if (hw->check_boot_tables) { err = hw->check_boot_tables(&pdev->dev); - if (err) + if (err && !hw->firmware_without_tables) return err; + if (err) + firmware_only = true; } err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); @@ -1963,6 +1991,7 @@ static int sio_probe(struct platform_device *pdev) sio->dev = &pdev->dev; sio->nchannels = nchannels; sio->hw = hw; + sio->firmware_only = firmware_only; sio->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sio->base)) @@ -2023,7 +2052,7 @@ static int sio_probe(struct platform_device *pdev) if (err) return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); - if (hw->firmware_only) { + if (sio->firmware_only) { dev_info(&pdev->dev, "firmware running, no DMA service\n"); goto rpm_put; } @@ -2062,6 +2091,9 @@ static int sio_probe(struct platform_device *pdev) } rpm_put: + if (!err && hw->stop_at_commit) + sio->commit_registered = !apple_sleep_coproc_register(&pdev->dev, sio->base, false, + sio_commit_stop, sio); pm_runtime_put(&pdev->dev); return err; @@ -2071,7 +2103,9 @@ static void sio_remove(struct platform_device *pdev) { struct sio_data *sio = platform_get_drvdata(pdev); - if (sio->hw->firmware_only) + if (sio->commit_registered) + apple_sleep_coproc_unregister(&pdev->dev); + if (sio->firmware_only) return; of_dma_controller_free(pdev->dev.of_node); @@ -2119,6 +2153,19 @@ static __maybe_unused int sio_suspend_late(struct device *dev) } } + if (sio->commit_registered && apple_sleep_commit_stops_coprocs()) { + apple_sleep_coproc_set_ready(dev, true); + sio->commit_used = true; + sio->asleep = true; + return 0; + } + + /* Idled any earlier, a refusal after it would leave it stopped. */ + if (sio->hw->stop_at_commit && apple_sleep_commit_stops_coprocs()) { + dev_err(dev, "no stop at the sleep's commit: refusing the sleep\n"); + return -EBUSY; + } + ret = apple_rtkit_sleep_stop(sio->rtk, sio->base + REG_CPU_CONTROL); if (ret) { dev_err(dev, "firmware did not stop for sleep: %d\n", ret); @@ -2138,6 +2185,17 @@ static __maybe_unused int sio_resume_early(struct device *dev) if (!sio->asleep) return 0; + /* Left for the commit: if the sleep was refused before it, the firmware never idled. */ + if (sio->commit_used) { + bool stopped = apple_sleep_coproc_stopped(dev); + + sio->commit_used = false; + apple_sleep_coproc_set_ready(dev, false); + if (!pm_resume_via_firmware() && !stopped) { + sio->asleep = false; + return 0; + } + } /* * A refused sleep: an idled firmware doesn't wake with its power kept, and SIO_CPU shares * SIO's switch, so it can't be power-cycled alone. The sleep engine's own PMGR reset of it From 30c3dafdc1604836a20827929e4cfcb118606d3e Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 00:23:16 -0500 Subject: [PATCH 06/29] soc: apple: sleep: refuse a commit over an unbound running coprocessor An unbound coprocessor that must stop only warned, because the gate moved on before its run-bit test. Nothing will stop it, so refuse the sleep if its domain is fully on and its CPU runs. Signed-off-by: Diljit Singh --- drivers/soc/apple/sleep.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 21d4b3889413b..7308c86e04cb7 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -1155,6 +1155,12 @@ static bool sleep_ps_on(struct apple_sleep *s, u32 offset) return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)); } +/* Below full power the hardware has idled it, and its registers may not answer. */ +static bool sleep_ps_full(struct apple_sleep *s, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(s->ps + offset)) == FIELD_MAX(PS_ACTUAL); +} + /* @pd's word in "ps": a power controller node, child of the PMGR that "ps" maps. */ static int sleep_ps_offset(struct apple_sleep *s, struct device_node *pd) { @@ -1511,6 +1517,14 @@ static bool apple_sleep_gate_failed(struct apple_sleep *s) if (!g->bound) { dev_err(s->dev, "%s isn't bound\n", g->name); bad++; + /* Nothing will stop it, and a commit over a running CPU resets the SoC. */ + if (g->must_stop && g->asc && g->ps && sleep_ps_full(s, g->ps) && + (readl(g->asc + ASC_CPU_CONTROL) & ASC_CPU_CONTROL_RUN)) { + dev_err(s->dev, "%s runs unbound: control %#x status %#x\n", g->name, + readl(g->asc + ASC_CPU_CONTROL), + readl(g->asc + ASC_CPU_STATUS)); + running++; + } continue; } /* A CPU whose domain is off isn't running, and its registers aren't there. */ @@ -3484,7 +3498,7 @@ static const struct apple_sleep_word t8140_stray_words[] = { static const struct apple_sleep_gate_coproc t8140_gate_coprocs[] = { { "/soc/dcp@412e00000", true, T8140_PS_DISP_CPU }, { "/soc/dcp@416e00000", true, T8140_PS_DISPEXT0_CPU }, - { "/soc/pmp@300500000", true }, { "/soc/dma-controller@385e00000", true }, + { "/soc/pmp@300500000", true }, { "/soc/dma-controller@385e00000", true, T8140_PS_SIO_CPU }, { "/soc/nvme@3cdcc0000" }, { "/soc/mtp@314600000" }, { "/soc/aop@310c00000" }, { "/soc/smc@30c600000" }, { "/soc/sep@282600000" }, { "/soc/gpu@482600000" }, From adcdcbbcf81941ed65b0183e7fff66e896f6d5db Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 00:23:16 -0500 Subject: [PATCH 07/29] soc: apple: sleep: clear latched power words before the commit The DRAM-kept wake's M2A_1B writes 44 power words and gives each switch 171 us to report closed. Old SOCD records show it stopping on ane_sys and on isp_cpu_core1, then WD2 resetting the SoC. macOS leaves these domains off with BUSY clear. The PMGR driver pulses a PS reset to clear a latch when a domain goes off, but not in a system sleep, which is when genpd takes ane_sys down. Clear a latch on each polled word that is off, just before the commit. If one stays latched, take the deep path for that sleep, which cuts their power: a refused sleep would be a failed suspend. Signed-off-by: Diljit Singh --- drivers/soc/apple/sleep.c | 78 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 7308c86e04cb7..cca6b3a785b53 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -79,6 +79,7 @@ struct aura_s2r_mailbox { /* PS word */ #define PS_AUTO_ENABLE BIT(28) +#define PS_UNLATCH BIT(12) #define PS_DEV_DISABLE BIT(10) #define PS_FLAGS GENMASK(9, 8) #define PS_ACTUAL GENMASK(7, 4) @@ -418,6 +419,8 @@ struct apple_sleep_soc { struct apple_sleep_words ps_aon_wake_order; /* Polled off just before the complex words */ struct apple_sleep_words ps_still_on; + /* The wake's timed power-gate polls: off with BUSY clear at the commit, on macOS */ + struct apple_sleep_words ps_engine_polled; /* Lowered at the last step if still on, with no user left */ const struct apple_sleep_word *stray; unsigned int nstray; @@ -568,6 +571,14 @@ struct apple_sleep { bool deep_ran; /* This sleep's wake came from the SMC's sources alone. */ bool smc_woke_alone; + /* A power word the DRAM-kept wake polls stayed latched: this sleep went deep. */ + bool deep_once; + /* Each latched word the commit cleared, as it found it */ + struct { + u32 offset; + u32 val; + } ps_unlatched[48]; + unsigned int ps_unlatched_count; }; static struct apple_sleep *apple_sleep; @@ -629,6 +640,9 @@ static void __iomem *sleep_slp_ddr_word; */ static bool sleep_slp_ddr_crashed; +/* This sleep only: the DRAM-kept wake would stall on something this commit leaves wrong. */ +static bool sleep_slp_ddr_deep_once; + static void sleep_slp_ddr_write(void) { u32 val; @@ -637,11 +651,18 @@ static void sleep_slp_ddr_write(void) if (!sleep_slp_ddr_word) return; val = readl(sleep_slp_ddr_word) & ~WAKE_SLP_DDR_USB; - if (sleep_slp_ddr_usb && !sleep_slp_ddr_crashed) + if (sleep_slp_ddr_usb && !sleep_slp_ddr_crashed && !sleep_slp_ddr_deep_once) val |= WAKE_SLP_DDR_USB; writel(val, sleep_slp_ddr_word); } +static void sleep_slp_ddr_set_deep_once(bool deep) +{ + guard(spinlock_irqsave)(&sleep_slp_ddr_lock); + sleep_slp_ddr_deep_once = deep; + sleep_slp_ddr_write(); +} + void apple_sleep_usb_slp_ddr(bool up) { guard(spinlock_irqsave)(&sleep_slp_ddr_lock); @@ -2414,6 +2435,38 @@ static void sleep_pmu_write(struct apple_sleep *s, u16 reg, u8 val) writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); } +/* + * The DRAM-kept wake writes each of these words and gives its switch 171 us to report closed. + * A domain that went off with BUSY latched holds that poll until WD2 resets the SoC. The PMGR + * driver doesn't clear the latch in a system sleep, so do it here. Returns whether one stayed + * latched; the deep path then cuts their power instead. + */ +static bool apple_sleep_ps_unlatch(struct apple_sleep *s) +{ + const struct apple_sleep_words *words = &s->soc->ps_engine_polled; + bool stuck = false; + unsigned int i; + + s->ps_unlatched_count = 0; + for (i = 0; i < words->count && i < ARRAY_SIZE(s->ps_unlatched); i++) { + void __iomem *reg = s->ps + words->offset[i]; + u32 val = readl(reg), off; + + if (!(val & PS_BUSY) || (val & (PS_ACTUAL | PS_TARGET))) + continue; + s->ps_unlatched[s->ps_unlatched_count].offset = words->offset[i]; + s->ps_unlatched[s->ps_unlatched_count++].val = val; + off = val & ~(PS_AUTO_ENABLE | PS_FLAGS | PS_BUSY | PS_UNLATCH | PS_DEV_DISABLE); + writel(off | PS_UNLATCH, reg); + readl_poll_timeout_atomic(reg, val, !(val & PS_BUSY), 1, PS_OFF_TIMEOUT_US); + writel(off, reg); + if (readl(reg) & PS_BUSY) + stuck = true; + } + + return stuck; +} + static void apple_sleep_match(struct apple_sleep *s) { const struct apple_sleep_soc *soc = s->soc; @@ -2650,6 +2703,12 @@ static int apple_sleep_commit(unsigned long unused) apple_sleep_arm_wd2(s); apple_sleep_wait_still_on(s); + /* Everything the wake polls is down by now, the words just lowered included. */ + if (apple_sleep_ps_unlatch(s)) { + s->deep_once = true; + sleep_slp_ddr_set_deep_once(true); + } + /* The SoC comes down before memory does, as on macOS. */ if (sleep_perf_wait(s->clk_perf + CLK_PERF_SOC, SOC_SLEEP_STATE)) { writel(SOC_SLEEP_STATE, s->clk_perf + CLK_PERF_SOC); @@ -2827,6 +2886,7 @@ static int apple_sleep_enter(suspend_state_t state) apple_sleep_coprocs_stop(s, false); /* Done by now unless no SMC driver is bound. */ apple_sleep_usb_down(s); + s->deep_once = false; s->cntkctl = read_sysreg(cntkctl_el1); pm_set_resume_via_firmware(); @@ -2834,6 +2894,8 @@ static int apple_sleep_enter(suspend_state_t state) if (ret) panic("Apple committed sleep returned without physical wake: %d", ret); apple_sleep_return(s); + if (s->deep_once) + sleep_slp_ddr_set_deep_once(false); apple_sleep_match_undo(s); apple_sleep_stray_restore(s); @@ -2950,6 +3012,11 @@ static void apple_sleep_report(void) dev_info(s->dev, "back from sleep, boot object resumes %llu -> %llu\n", s->resumes_before, s->resumes_after); + for (i = 0; i < s->ps_unlatched_count; i++) + dev_warn(s->dev, "power word %#x was off with BUSY latched at the commit: %#x\n", + s->ps_unlatched[i].offset, s->ps_unlatched[i].val); + if (s->deep_once) + dev_info(s->dev, "took the deep path: a power word stayed latched\n"); if (s->deep_ran) dev_info(s->dev, "the deep path ran\n"); for (i = 0; i < s->soc->nwake_names; i++) { @@ -3476,6 +3543,14 @@ static const u32 t8140_ps_aon_macos_no_auto[] = { 0x4078, 0x4080, }; +/* M2A_1B's power-gate polls, less the SEP, PMP and SIO switch, which macOS doesn't have off. */ +static const u32 t8140_ps_engine_polled[] = { + 0xc008, 0xc010, 0xc000, 0x290, 0x270, 0x8038, 0x8030, 0x8048, 0x8040, 0x8028, 0x248, + 0x8010, 0x8008, 0x8020, 0x8018, 0x8000, 0x240, 0x4038, 0x4030, 0x4028, 0x4040, 0x4060, + 0x4058, 0x4050, 0x4048, 0x4018, 0x4010, 0x4008, 0x4000, 0x268, 0x1f0, 0x298, 0x238, + 0x3b8, 0x3b0, 0x328, 0x288, 0x308, 0x3a0, 0x2a0, 0x278, +}; + static const u32 t8140_ps_still_on[] = { T8140_PS_ATC0_COMMON, T8140_PS_DISP_SYS, T8140_PS_DISPEXT0_SYS, T8140_PS_ANS, 0x270 /* AVD_SYS */, 0x240 /* VENC0_SYS */, 0x248 /* VENC1_SYS */, 0x1f0 /* GFX */, @@ -3555,6 +3630,7 @@ static const struct apple_sleep_soc t8140_sleep_soc = { .ps_wake_order = SLEEP_WORDS(t8140_ps_wake_order), .ps_aon_wake_order = SLEEP_WORDS(t8140_ps_aon_wake_order), .ps_still_on = SLEEP_WORDS(t8140_ps_still_on), + .ps_engine_polled = SLEEP_WORDS(t8140_ps_engine_polled), .stray = t8140_stray_words, .nstray = ARRAY_SIZE(t8140_stray_words), .gate_off = t8140_gate_off, From d82906e252ce9da92d26537507e29817281167be Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 02:20:34 -0500 Subject: [PATCH 08/29] pmdomain: apple: lower T8140 domains as macOS does A domain under automatic power management went down in one write that cleared the auto bit and the target together. macOS clears the auto bit first, sees the domain back at full power, then writes the target and clears the history bits (traced for AVD, VENC1 and JPEG). ane_sys went off with BUSY latched in about one sleep in 80, and the sleep engine's wake then stalled on it until the watchdog reset the SoC. Lower T8140 domains the macOS way, and clear a latch when a domain goes off even in a system sleep, while the domain is off and long before the commit. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmgr-pwrstate.c | 30 ++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 403161b5224d4..454e43e9faca2 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -66,6 +66,8 @@ struct apple_pmgr_ps { bool force_reset; bool externally_clocked; bool system_sleep; + /* Lowered as macOS lowers it, and unlatched even in a system sleep. */ + bool macos_lower; bool firmware_owned; /* apple,sleep-retained: kept on while the system sleeps */ bool sleep_retained; @@ -220,11 +222,27 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a genpd->name, cur); } + /* + * As macOS lowers a domain: automatic power management off first, back at full + * power, and only then the target. Taking both at once from an auto-gated state + * can leave BUSY latched, and a latch stalls the sleep engine's wake. + */ + if (ps->macos_lower && pstate != APPLE_PMGR_PS_ACTIVE && (reg & APPLE_PMGR_AUTO_ENABLE)) { + regmap_write(ps->regmap, ps->offset, + reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY)); + regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) == + APPLE_PMGR_PS_ACTIVE, 1, timeout); + } + /* Do not replay the hardware-owned BUSY bit in this whole-word write. */ reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY | APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); + /* Writing the history bits clears them. */ + if (ps->macos_lower && pstate != APPLE_PMGR_PS_ACTIVE) + reg |= APPLE_PMGR_FLAGS; dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); @@ -291,9 +309,16 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a * path (not in a system sleep transition, see apple_pmgr_ps_clear_latch()). */ if (!ret && pstate != APPLE_PMGR_PS_ACTIVE && (cur & APPLE_PMGR_BUSY) && - !FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) && !ps->system_sleep && - !ps->wait_parent && !ps->strict_state && !ps->externally_clocked) + !FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) && (!ps->system_sleep || ps->macos_lower) && + !ps->wait_parent && !ps->strict_state && !ps->externally_clocked) { + /* macOS never needs this: say so when it does. */ + if (ps->macos_lower) + dev_info_ratelimited(ps->dev, + "PS %s: off with BUSY%s: %#x; cleared by a PS reset\n", + genpd->name, ps->system_sleep ? " in a system sleep" : "", + cur); apple_pmgr_ps_unlatch_off(ps, &cur); + } report: if (ret < 0) @@ -765,6 +790,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ps); ps->reset_time_us = t8132 ? APPLE_PMGR_T8132_RESET_TIME : APPLE_PMGR_RESET_TIME; ps->needs_reset_idle_wait = t8132; + ps->macos_lower = of_device_is_compatible(node, "apple,t8140-pmgr-pwrstate"); ps->no_auto_enable = t8132 && !strict_state; ps->strict_state = strict_state; From 042933d17503360aff0af9da6a5c6ece95e72ea2 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 02:19:03 -0500 Subject: [PATCH 09/29] accel/ane: take the H17 engine down in macOS's order and check BUSY On a failing sleep the commit map read ane_sys off with BUSY latched and the wake's engine stopped in the ANE's power-on. Linux's power-down waits only for the actual state to reach zero, so it can't see a BUSY latch, and it skips a step macOS takes. macOS's H17 path (DisableSubblocksClocksAndPower and DisableCPUClocksAndPower, ane-type 0x200, 25G83) writes each power word whole as 0x300 and waits for (word & 0x8ff) == 0, which covers target, actual and BUSY. Between the PWGATE request and the CPU word it clears the word at PWGATE + 0x30 and waits for bit 0 to read zero. Do the same. A word that stays BUSY now fails the power-down, which refuses the sleep instead of carrying the latch into the commit. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index a9224c63326a1..c38680685a7a7 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -511,21 +511,39 @@ static int ane_fw_boot(struct ane_fw_backend *b); static int ane_fw_startup(struct ane_fw_backend *b); static int ane_fw_rvbar_ready(struct ane_fw_backend *b); -/* Reverse of the proven power-up: sub-block, PWGATE, CPU, then ANE_SYS. */ +/* A power word goes off whole, and off means target, actual and BUSY clear. */ +#define ANE_FW_PS_OFF 0x300 +#define ANE_FW_PS_OFF_MASK 0x8ff + +static int ane_fw_ps_off(struct ane_fw_backend *b, u32 offset) +{ + int ret; + + ane_wr32(b, ANE_FW_PS_OFF, b->pmgr + offset); + ret = ane_poll32(b, b->pmgr + offset, ANE_FW_PS_OFF_MASK, 0, 1, 100000); + if (ret) + dev_err(&b->pdev->dev, "H17 ANE: power word %#x stuck at %#x\n", offset, + ane_rd32(b, b->pmgr + offset)); + return ret; +} + +/* The order macOS uses for this engine: sub-block, PWGATE, its ack, CPU, then ANE_SYS. */ static int ane_fw_power_down(struct ane_fw_backend *b) { int ret; - ane_wr32(b, ane_rd32(b, b->pmgr + 0xc008) & ~0xf, b->pmgr + 0xc008); - ret = ane_poll32(b, b->pmgr + 0xc008, 0xf0, 0, 1, 100000); + ret = ane_fw_ps_off(b, 0xc008); if (ret) return ret; ane_wr32(b, 3, b->gate + 0xc); ret = ane_poll32(b, b->gate + 0xc, 3, 3, 1, 100000); if (ret) return ret; - ane_wr32(b, ane_rd32(b, b->pmgr + 0xc000) & ~0xf, b->pmgr + 0xc000); - ret = ane_poll32(b, b->pmgr + 0xc000, 0xf0, 0, 1, 100000); + ane_wr32(b, 0, b->gate + 0x30); + ret = ane_poll32(b, b->gate + 0x30, 1, 0, 1, 100000); + if (ret) + return ret; + ret = ane_fw_ps_off(b, 0xc000); if (ret) return ret; /* No engine or DART access from here until ANE_SYS is back on. */ From 60581160fb05cc6f9473db4f806bd31ae4a86825 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 02:19:03 -0500 Subject: [PATCH 10/29] accel/ane: stop the H17 firmware before its power goes The H17 sleep path cuts the engine's power with the firmware running. macOS never does that. In ANE_deInit, on this chip, it sends command 1 (12 bytes, with a reply), writes 0xf7fbdff9 to SCRATCH0, posts POWER_DOWN (0x0b, 8 bytes) on the IO ring, and waits up to a second for the firmware to answer 0x08042006 in SCRATCH0. It then waits up to 200 ms for CPU_STATUS & 3, and goes on without it if that doesn't come. It takes the firmware out of Exclave mode (0x31) first, and sends SET_SNE_PMU_BASE2 (0x29) at every start. SUSPEND (0x24), the command this driver tried, is only chosen when a per-chip flag is set, and the H17 configuration doesn't set it. That is the likely reason 0x24 was never taken, along with waiting on the ring instead of on SCRATCH0. Before the power words macOS also deactivates the DARTs, whose power-down path waits for every port to read idle at DART + 0x230. Check the same registers here. The deactivation itself goes through SPTM and isn't visible in the kernelcache, so the DARTs are left programmed. macOS doesn't clear the CPU's run bit here, and neither does this. Any failure after the first command quarantines the engine and refuses this sleep. ane.clean_stop=0 restores the old path. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 121 +++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 2 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index c38680685a7a7..02e909d655102 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -743,12 +743,129 @@ static int ane_fw_cold_cycle_boot(struct ane_fw_backend *b) return ret; } -/* The cold way down: nothing in flight, then power off under the firmware. */ +/* + * macOS stops this firmware before its power goes (ANE_deInit, 25G83): command + * 1, then SCRATCH0 armed and POWER_DOWN, which is answered in SCRATCH0 and not + * on the ring. SUSPEND (0x24) is for chips with a warm sleep, not this one. + */ +#define ANE_FW_CMD_STOP 0x1 +#define ANE_FW_STOP_SIZE 12 +#define ANE_FW_CMD_POWER_DOWN 0xb +#define ANE_FW_POWER_DOWN_SIZE 8 +#define ANE_FW_STOP_ARM 0xf7fbdff9 +#define ANE_FW_STOP_DONE 0x08042006 + +static bool clean_stop = true; +module_param(clean_stop, bool, 0644); +MODULE_PARM_DESC(clean_stop, "Stop the H17 firmware before powering it off for system sleep"); + +#define ane_fw_clean_stop(b) (clean_stop && (b)->soc->exclave_mode_stop) + +/* Until SCRATCH0 answers, keep serving what the firmware posts on its way down. */ +static int ane_fw_stop_wait(struct ane_fw_backend *b) +{ + struct ane_fw_transport *t = &b->transport; + void __iomem *scratch0 = b->engine + b->soc->scratch; + u64 deadline = ktime_get_ns() + NSEC_PER_SEC; + u8 report[ANE_FW_SUSPEND_REPORT_MAX]; + u32 pending; + size_t size; + int ret; + + while (ane_rd32(b, scratch0) != ANE_FW_STOP_DONE) { + if (ktime_get_ns() >= deadline) + return -ETIMEDOUT; + ret = ane_fw_transport_consume(t, report, sizeof(report), &size); + if (ret && ret != -EAGAIN) + return ret; + ret = ane_fw_transport_irq(t, &pending); + if (ret) + return ret; + ret = ane_fw_transport_shared(t); + if (ret && ret != -EAGAIN) + return ret; + usleep_range(100, 200); + } + return 0; +} + +/* Every port of every ANE DART idle: what macOS checks before it powers one down. */ +static int ane_fw_dart_idle(struct ane_fw_backend *b) +{ + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) { + u32 params = ane_rd32(b, b->dart[i] + 4); + unsigned int ports = max((params >> 16) & 7, (params >> 20) & 7); + + for (unsigned int p = 0; p < min(ports, 4U); p++) { + int ret = ane_poll32(b, b->dart[i] + 0x230 + 4 * p, U32_MAX, 0, 1, 10000); + + if (ret) { + dev_err(&b->pdev->dev, "H17 ANE: DART %u port %u not idle: %#x\n", + i, p, ane_rd32(b, b->dart[i] + 0x230 + 4 * p)); + return ret; + } + } + } + return 0; +} + +static int ane_fw_firmware_stop(struct ane_fw_backend *b) +{ + struct ane_fw_transport *t = &b->transport; + struct device *dev = &b->pdev->dev; + void __iomem *status = b->engine + b->soc->wrapper + 0x48; + u8 stop[ANE_FW_STOP_SIZE] = {}; + u8 down[ANE_FW_POWER_DOWN_SIZE] = {}; + u32 val; + int ret; + + /* A core in WFI can't finish an allocation request, so answer those first. */ + ret = ane_fw_transport_settle(t, 2 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC); + if (ret < 0) + return ret; + ret = ane_fw_exclave_mode_stop(b); + if (ret) + return ret; + put_unaligned_le16(ANE_FW_CMD_STOP, stop + 4); + ret = ane_fw_command(b, ANE_FW_IO, stop, sizeof(stop), ktime_get_ns() + ANE_FW_TIMEOUT_NS); + if (ret) { + dev_err(dev, "H17 ANE: STOP failed: %d\n", ret); + return ret; + } + put_unaligned_le16(ANE_FW_CMD_POWER_DOWN, down + 4); + ane_wr32(b, ANE_FW_STOP_ARM, b->engine + b->soc->scratch); + ret = ane_fw_transport_publish(t, ANE_FW_IO, down, sizeof(down), + ktime_get_ns() + ANE_FW_TIMEOUT_NS); + if (!ret) + ret = ane_fw_stop_wait(b); + if (ret) { + ane_fw_sleep_trace(b, "POWER_DOWN failed", ane_fw_transport_slot(t, ANE_FW_IO)); + ane_h16_session_dump_fwlog(&b->session); + return ret; + } + /* It never completes on the ring; the boot after the wake starts a new one. */ + t->pending[ANE_FW_IO].active = false; + /* macOS waits up to 200 ms for WFI and goes on without it. */ + if (readl_poll_timeout(status, val, val & 3, 10000, 200000)) + dev_warn(dev, "H17 ANE: no WFI after POWER_DOWN, status %#x\n", ane_rd32(b, status)); + dev_info(dev, "H17 ANE: firmware stopped, status %#x\n", ane_rd32(b, status)); + return ane_fw_dart_idle(b); +} + +/* The cold way down: nothing in flight, stop the firmware, then power off. */ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) { + int ret; + for (unsigned int ch = 0; ch < ANE_FW_CHANNELS; ch++) if (b->transport.pending[ch].active) return -EBUSY; + if (ane_fw_clean_stop(b)) { + ret = ane_fw_firmware_stop(b); + /* Past this point the firmware may be stopped: never a clean refusal. */ + if (ret) + return ret == -EBUSY ? -EIO : ret; + } return ane_fw_power_down(b); } @@ -1493,7 +1610,7 @@ static int ane_fw_startup(struct ane_fw_backend *b) if (commands[i].id == 0x22) dev_dbg(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); /* macOS order: START, then SET_SNE_PMU_BASE2, then CONFIG_GET. */ - if (commands[i].id == 0 && b->soc->sleep_warm) { + if (commands[i].id == 0 && (b->soc->sleep_warm || ane_fw_clean_stop(b))) { ret = ane_fw_pmu_base(b); if (ret) return ret; From 0a80c389aaef1869633949da9c55cb2aebc1b51e Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 02:28:55 -0500 Subject: [PATCH 11/29] accel/ane: h17: take the deep path instead of refusing a sleep A refused sleep is a failed suspend. If a power word stays BUSY after a PS reset, or the firmware doesn't stop, ask the sleep driver for the deep path for this sleep, which cuts the block's power. Refuse only if that isn't available. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 33 +++++++++++++++++++++++++-------- drivers/soc/apple/sleep.c | 20 ++++++++++++++++++-- include/linux/soc/apple/sleep.h | 6 ++++++ 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 02e909d655102..a70c02ac6752c 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "ane_fw_cold.h" @@ -514,17 +515,31 @@ static int ane_fw_rvbar_ready(struct ane_fw_backend *b); /* A power word goes off whole, and off means target, actual and BUSY clear. */ #define ANE_FW_PS_OFF 0x300 #define ANE_FW_PS_OFF_MASK 0x8ff +#define ANE_FW_PS_BUSY BIT(11) +#define ANE_FW_PS_UNLATCH BIT(12) +/* A word left BUSY stalls a wake that keeps DRAM up: clear it, or send this sleep deep. */ static int ane_fw_ps_off(struct ane_fw_backend *b, u32 offset) { + void __iomem *reg = b->pmgr + offset; int ret; - ane_wr32(b, ANE_FW_PS_OFF, b->pmgr + offset); - ret = ane_poll32(b, b->pmgr + offset, ANE_FW_PS_OFF_MASK, 0, 1, 100000); - if (ret) - dev_err(&b->pdev->dev, "H17 ANE: power word %#x stuck at %#x\n", offset, - ane_rd32(b, b->pmgr + offset)); - return ret; + ane_wr32(b, ANE_FW_PS_OFF, reg); + ret = ane_poll32(b, reg, ANE_FW_PS_OFF_MASK, 0, 1, 100000); + if (!ret) + return 0; + dev_warn(&b->pdev->dev, "H17 ANE: power word %#x stuck at %#x\n", offset, ane_rd32(b, reg)); + if (!(ane_rd32(b, reg) & 0xff)) { + ane_wr32(b, ANE_FW_PS_OFF | ANE_FW_PS_UNLATCH, reg); + ane_poll32(b, reg, ANE_FW_PS_BUSY, 0, 1, 1000); + ane_wr32(b, ANE_FW_PS_OFF, reg); + if (!ane_poll32(b, reg, ANE_FW_PS_OFF_MASK, 0, 1, 1000)) + return 0; + } + if (!apple_sleep_request_deep_once()) + return ret; + dev_warn(&b->pdev->dev, "H17 ANE: power word %#x still stuck, taking the deep path\n", offset); + return 0; } /* The order macOS uses for this engine: sub-block, PWGATE, its ack, CPU, then ANE_SYS. */ @@ -862,8 +877,10 @@ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) return -EBUSY; if (ane_fw_clean_stop(b)) { ret = ane_fw_firmware_stop(b); - /* Past this point the firmware may be stopped: never a clean refusal. */ - if (ret) + /* A refused sleep is a failed suspend: the deep path cuts its power cleanly. */ + if (ret && apple_sleep_request_deep_once()) + dev_warn(&b->pdev->dev, "H17 ANE: no clean stop (%d), taking the deep path\n", ret); + else if (ret) return ret == -EBUSY ? -EIO : ret; } return ane_fw_power_down(b); diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index cca6b3a785b53..180849fee5775 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -656,6 +656,19 @@ static void sleep_slp_ddr_write(void) writel(val, sleep_slp_ddr_word); } +/* A driver found something the DRAM-kept wake would stall on: this sleep goes deep. */ +static bool sleep_deep_requested; + +bool apple_sleep_request_deep_once(void) +{ + if (!READ_ONCE(apple_sleep)) + return false; + WRITE_ONCE(sleep_deep_requested, true); + + return true; +} +EXPORT_SYMBOL_GPL(apple_sleep_request_deep_once); + static void sleep_slp_ddr_set_deep_once(bool deep) { guard(spinlock_irqsave)(&sleep_slp_ddr_lock); @@ -2886,7 +2899,9 @@ static int apple_sleep_enter(suspend_state_t state) apple_sleep_coprocs_stop(s, false); /* Done by now unless no SMC driver is bound. */ apple_sleep_usb_down(s); - s->deep_once = false; + s->deep_once = READ_ONCE(sleep_deep_requested); + if (s->deep_once) + sleep_slp_ddr_set_deep_once(true); s->cntkctl = read_sysreg(cntkctl_el1); pm_set_resume_via_firmware(); @@ -2927,6 +2942,7 @@ static int apple_sleep_begin(suspend_state_t state) static void apple_sleep_end(void) { + WRITE_ONCE(sleep_deep_requested, false); if (pm_suspend_via_firmware()) writel(0, apple_sleep->wdt + WDT_WD2_CTL); } @@ -3016,7 +3032,7 @@ static void apple_sleep_report(void) dev_warn(s->dev, "power word %#x was off with BUSY latched at the commit: %#x\n", s->ps_unlatched[i].offset, s->ps_unlatched[i].val); if (s->deep_once) - dev_info(s->dev, "took the deep path: a power word stayed latched\n"); + dev_info(s->dev, "took the deep path: a power word stayed latched or stuck\n"); if (s->deep_ran) dev_info(s->dev, "the deep path ran\n"); for (i = 0; i < s->soc->nwake_names; i++) { diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 91be63b7aa18f..4c408dbd798f6 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -34,6 +34,7 @@ int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), void *data); void apple_sleep_coproc_unregister(struct device *dev); +bool apple_sleep_request_deep_once(void); /* * For a registered coprocessor: the last of its stop that may still refuse the sleep, run @@ -154,6 +155,11 @@ static inline int apple_sleep_coproc_register_first(struct device *dev, return -EOPNOTSUPP; } +static inline bool apple_sleep_request_deep_once(void) +{ + return false; +} + static inline void apple_sleep_coproc_unregister(struct device *dev) { } From 7302a10bd881f6f103e91efd6557839486244fbf Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 03:04:26 -0500 Subject: [PATCH 12/29] accel/ane: h17: stop the firmware and its DARTs as macOS does On the Neo the stop command got no answer. A recording of macOS's sleep under aura with the ANE visible (25G83) shows three things the static reading missed: - once the firmware is up, macOS sets SCRATCH14 = 1 and SCRATCH16 = 0, and before the stop command it sets them the other way; - after SCRATCH0 answers it writes 0xfeedbabe to SCRATCH1; - before the power words it clears each DART's counters and error status, disables every enabled stream one at a time, and flushes the TLB. Do the same, in that order. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 46 ++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index a70c02ac6752c..b260da8f60438 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -824,6 +824,38 @@ static int ane_fw_dart_idle(struct ane_fw_backend *b) return 0; } +/* + * Each DART as macOS leaves it before the power words (recorded, 25G83): its counters + * cleared, error status acknowledged, every enabled stream disabled one at a time, then + * the TLB flushed. + */ +static void ane_fw_dart_deactivate(struct ane_fw_backend *b) +{ + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) { + void __iomem *dart = b->dart[i]; + u32 enabled; + + ane_wr32(b, U32_MAX, dart + 0x4060); + ane_rd32(b, dart + 0x4060); + ane_rd32(b, dart + 0x4000); + ane_wr32(b, 0, dart + 0x4000); + ane_wr32(b, U32_MAX, dart + 0x704); + ane_rd32(b, dart + 0x704); + enabled = ane_rd32(b, dart + 0xc00); + for (unsigned int bit = 0; bit < 16; bit++) { + if (!(enabled & BIT(bit)) && bit != 11 && bit != 15) + continue; + ane_wr32(b, BIT(bit), dart + 0xc20); + ane_rd32(b, dart + 0xc20); + } + /* Flush all: V2 | STT, as macOS writes it. */ + for (unsigned int j = 0; j < ANE_FW_DARTS; j++) + ane_wr32(b, 0xa000, b->dart[j] + 0x80); + for (unsigned int j = 0; j < ANE_FW_DARTS; j++) + ane_rd32(b, b->dart[j] + 0x80); + } +} + static int ane_fw_firmware_stop(struct ane_fw_backend *b) { struct ane_fw_transport *t = &b->transport; @@ -841,6 +873,11 @@ static int ane_fw_firmware_stop(struct ane_fw_backend *b) ret = ane_fw_exclave_mode_stop(b); if (ret) return ret; + /* The host flags as macOS sets them before its stop: SCRATCH14 = 0, SCRATCH16 = 1. */ + ane_wr32(b, 0, b->engine + b->soc->scratch + 0x38); + ane_rd32(b, b->engine + b->soc->scratch + 0x3c); + ane_wr32(b, 1, b->engine + b->soc->scratch + 0x40); + ane_rd32(b, b->engine + b->soc->scratch + 0x44); put_unaligned_le16(ANE_FW_CMD_STOP, stop + 4); ret = ane_fw_command(b, ANE_FW_IO, stop, sizeof(stop), ktime_get_ns() + ANE_FW_TIMEOUT_NS); if (ret) { @@ -864,7 +901,11 @@ static int ane_fw_firmware_stop(struct ane_fw_backend *b) if (readl_poll_timeout(status, val, val & 3, 10000, 200000)) dev_warn(dev, "H17 ANE: no WFI after POWER_DOWN, status %#x\n", ane_rd32(b, status)); dev_info(dev, "H17 ANE: firmware stopped, status %#x\n", ane_rd32(b, status)); - return ane_fw_dart_idle(b); + ane_wr32(b, 0xfeedbabe, b->engine + b->soc->scratch + 4); + ret = ane_fw_dart_idle(b); + if (!ret) + ane_fw_dart_deactivate(b); + return ret; } /* The cold way down: nothing in flight, stop the firmware, then power off. */ @@ -1633,6 +1674,9 @@ static int ane_fw_startup(struct ane_fw_backend *b) return ret; } } + /* The host flags macOS sets once the firmware is up (recorded, 25G83). */ + ane_wr32(b, 1, b->engine + b->soc->scratch + 0x38); + ane_wr32(b, 0, b->engine + b->soc->scratch + 0x40); return 0; } From ce8192d44a990f1f38f5ca946d2454d8b51c271a Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 03:15:32 -0500 Subject: [PATCH 13/29] accel/ane: h17: don't send EXCLAVE_MODE_STOP before the stop The recording of macOS's sleep shows its AP ringing only the IO, and three other, channels: never the one EXCLAVE_MODE_STOP (0x31) goes on. With 0x31 sent first, the Neo's firmware didn't answer the stop command on two boots. Send the stop as macOS does, without it. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index b260da8f60438..4609d16c7977a 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -870,9 +870,10 @@ static int ane_fw_firmware_stop(struct ane_fw_backend *b) ret = ane_fw_transport_settle(t, 2 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC); if (ret < 0) return ret; - ret = ane_fw_exclave_mode_stop(b); - if (ret) - return ret; + /* + * No EXCLAVE_MODE_STOP here: macOS's AP never rings that channel around its sleep + * (recorded), and after it this firmware doesn't answer the stop. + */ /* The host flags as macOS sets them before its stop: SCRATCH14 = 0, SCRATCH16 = 1. */ ane_wr32(b, 0, b->engine + b->soc->scratch + 0x38); ane_rd32(b, b->engine + b->soc->scratch + 0x3c); From 942c008874b2aed3b58725c796ccd90d9ba747db Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 03:25:16 -0500 Subject: [PATCH 14/29] accel/ane: h17: keep the engine off until a request needs it macOS starts the ANE for its first client and keeps it off otherwise, so its sleep has nothing to stop. This driver booted the firmware at probe and again after every wake with no client, so every sleep had to stop a running firmware, and the stop this firmware answers isn't known yet. After the bring-up at boot, and after each wake, park the engine: powered off with no firmware and its session closed. The first LOAD or EXECUTE boots it from the preload, as the wake did, and replays any open program. A sleep with the engine parked has nothing to do. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 92 ++++++++++++++++++++++++++++- drivers/accel/ane/ane_h16_program.c | 10 +++- drivers/accel/ane/ane_h16_program.h | 9 +++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 4609d16c7977a..3c1b2adc31657 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -76,6 +76,15 @@ struct ane_fw_backend { u8 data_digest[SHA256_DIGEST_SIZE]; /* Powered down for system sleep; a (cold or warm) boot is due. */ bool slept; + /* + * Parked: powered off with no firmware, as macOS keeps it without a + * client, until a request brings it up (ane_fw_session_wake()). + */ + bool parked; + /* Between PM_SUSPEND_PREPARE and PM_POST_SUSPEND. */ + bool in_sleep; + /* This power-down is for a system sleep: a stuck one may send it deep. */ + bool down_for_sleep; /* The last cold boot after sleep: firmware ready, first replay, count. */ u64 resume_ready_ns, resume_first_ns; unsigned int resume_replayed; @@ -536,6 +545,10 @@ static int ane_fw_ps_off(struct ane_fw_backend *b, u32 offset) if (!ane_poll32(b, reg, ANE_FW_PS_OFF_MASK, 0, 1, 1000)) return 0; } + if (!b->down_for_sleep) { + dev_warn(&b->pdev->dev, "H17 ANE: power word %#x still stuck while parking\n", offset); + return 0; + } if (!apple_sleep_request_deep_once()) return ret; dev_warn(&b->pdev->dev, "H17 ANE: power word %#x still stuck, taking the deep path\n", offset); @@ -919,8 +932,11 @@ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) return -EBUSY; if (ane_fw_clean_stop(b)) { ret = ane_fw_firmware_stop(b); + /* Parking at runtime: the PMGR clears a latch outside a system sleep. */ + if (ret && !b->down_for_sleep) + dev_warn(&b->pdev->dev, "H17 ANE: no clean stop (%d) while parking\n", ret); /* A refused sleep is a failed suspend: the deep path cuts its power cleanly. */ - if (ret && apple_sleep_request_deep_once()) + else if (ret && apple_sleep_request_deep_once()) dev_warn(&b->pdev->dev, "H17 ANE: no clean stop (%d), taking the deep path\n", ret); else if (ret) return ret == -EBUSY ? -EIO : ret; @@ -928,11 +944,62 @@ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) return ane_fw_power_down(b); } +/* + * Off with no firmware until a request needs it, as macOS keeps the ANE + * without a client. Called with b->lock held, the engine up at the program + * stage. Its session stays closed: ane_h16_session_enter() wakes it. + */ +static int ane_fw_park(struct ane_fw_backend *b) +{ + int ret; + + ane_h16_session_sleep(&b->session, true); + b->down_for_sleep = false; + ret = ane_fw_cold_cycle_down(b); + if (ret) { + ane_fw_quarantine(b, ret); + b->session.quarantined = true; + ane_h16_session_sleep(&b->session, false); + return ret; + } + WRITE_ONCE(b->parked, true); + WRITE_ONCE(b->session.parked, true); + return 0; +} + +/* A request for a parked engine: boot it from the preload, as after a sleep. */ +static int ane_fw_session_wake(void *ctx) +{ + struct ane_fw_backend *b = ctx; + int ret; + + guard(mutex)(&b->lock); + if (!b->parked) + return 0; + /* The system is going to sleep or coming back: PM_POST_SUSPEND wakes waiters. */ + if (b->in_sleep) + return -EAGAIN; + ret = ane_fw_cold_cycle_boot(b); + WRITE_ONCE(b->parked, false); + WRITE_ONCE(b->session.parked, false); + if (ret) { + ane_fw_quarantine(b, ret); + b->session.quarantined = true; + } + ane_h16_session_sleep(&b->session, false); + dev_info(&b->pdev->dev, "H17 ANE: up for a request%s\n", ret ? " FAILED" : ""); + return ret; +} + static int ane_fw_sleep_prepare(struct ane_fw_backend *b) { int ret; guard(mutex)(&b->lock); + /* Parked: already off, with nothing to stop. */ + if (b->parked) + return 0; + b->down_for_sleep = true; /* * A quarantined engine that never came up has nothing to lose; give * back an ANE_SYS reference a failed power-up kept. @@ -1021,6 +1088,13 @@ static void ane_fw_sleep_resume(struct ane_fw_backend *b) if (!b->slept) return; b->slept = false; + /* Stay off until a request needs the engine, as macOS does. */ + if (!b->session.quarantined) { + WRITE_ONCE(b->parked, true); + WRITE_ONCE(b->session.parked, true); + dev_info(&b->pdev->dev, "H17 ANE: resume ok: parked until the next request\n"); + return; + } start = ktime_get_ns(); b->resume_ready_ns = 0; b->resume_first_ns = 0; @@ -1066,8 +1140,16 @@ static int ane_fw_pm_event(struct notifier_block *nb, unsigned long event, void if (event == PM_POST_SUSPEND) { ane_fw_sleep_resume(b); + WRITE_ONCE(b->in_sleep, false); + WRITE_ONCE(b->session.wake_blocked, false); + /* Requests that waited out the sleep now wake the parked engine. */ + wake_up_all(&b->session.wake); return NOTIFY_DONE; } + if (event == PM_SUSPEND_PREPARE) { + WRITE_ONCE(b->in_sleep, true); + WRITE_ONCE(b->session.wake_blocked, true); + } if (event == PM_SUSPEND_PREPARE && retained) { if (!ane_fw_sleep_prepare(b)) return NOTIFY_DONE; @@ -1728,6 +1810,7 @@ static const struct ane_h16_session_ops ane_fw_session_ops = { .log = ane_fw_program_log, .sync = ane_fw_program_sync, .busy = ane_fw_program_busy, + .wake = ane_fw_session_wake, }; /* @@ -1971,6 +2054,13 @@ static void ane_fw_start_work(struct work_struct *work) } } dev_info(&b->pdev->dev, "H17 ANE: up at boot, accel node ready\n"); + /* Then off until a client's request, as macOS keeps it. */ + flags = lock_system_sleep(); + mutex_lock(&b->lock); + ret = ane_fw_park(b); + mutex_unlock(&b->lock); + unlock_system_sleep(flags); + dev_info(&b->pdev->dev, "H17 ANE: parked until the first request%s\n", ret ? " FAILED" : ""); } static void ane_fw_observer_free(struct ane_fw_backend *b) diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index 7a47f842e7b7d..432efc1c9bdc8 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -1656,10 +1656,18 @@ int ane_h16_session_enter(struct ane_h16_session *s, bool nonblock) int ret; for (;;) { + /* A parked engine comes up for its first request. */ + if (READ_ONCE(s->parked) && s->ops->wake) { + ret = s->ops->wake(s->ctx); + if (ret && ret != -EAGAIN) + return ret; + } /* O_NONBLOCK: a clean -EAGAIN while the engine is cycled. */ if (nonblock && READ_ONCE(s->asleep)) return -EAGAIN; - ret = wait_event_freezable(s->wake, !READ_ONCE(s->asleep)); + ret = wait_event_freezable(s->wake, !READ_ONCE(s->asleep) || + (READ_ONCE(s->parked) && s->ops->wake && + !READ_ONCE(s->wake_blocked))); if (ret) return ret; mutex_lock(s->lock); diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h index 468e83c7101be..50ad92a0470a2 100644 --- a/drivers/accel/ane/ane_h16_program.h +++ b/drivers/accel/ane/ane_h16_program.h @@ -167,6 +167,11 @@ struct ane_h16_session_ops { * hold a performance vote while any request is in flight. */ void (*busy)(void *ctx, bool busy); + /* + * Optional. The engine is parked (off, no firmware) until a request needs + * it: bring it up, or return an error if it can't come up now. + */ + int (*wake)(void *ctx); }; struct ane_h16_program { @@ -282,6 +287,10 @@ struct ane_h16_session { * ane_h16_session_enter() and closes are deferred to the reopen. */ bool asleep; + /* Off until the next request, which calls ops->wake first. */ + bool parked; + /* A system sleep is under way: a parked engine can't come up until it ends. */ + bool wake_blocked; struct list_head deferred; /* * Firmware boot count: a cold restart across system sleep loses every From 589646b6cec31b42f2a632e3b7dcd916ee43ae34 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 03:33:48 -0500 Subject: [PATCH 15/29] accel/ane: h17: send SET_SNE_PMU_BASE2 on the IO ring, and wait for the stop flags The H17 stop command gets no answer. The likely cause is the ring that SET_SNE_PMU_BASE2 (0x29) goes on. macOS's command helper takes a queue number, not a table position. In aneFirmwareCommandSend (25G83) the queue indexes a table of three channel fields, which ANE_Init fills by name: queue 0 is IO, 1 is DEBUG, and 2 is DATA_CHAIN_H2T. The argument this driver read as "channel selector 2" is the command's timeout in seconds. Every start-up command, 0x29, the Exclave mode commands 0x30 and 0x31, the stop (1) and POWER_DOWN (0x0b) use queue 0, IO. Only the firmware debug query uses DEBUG. The recording agrees: macOS's boot rings the IO doorbell for about sixteen commands and never rings DEBUG's. This driver sent 0x29 on DEBUG. That ring completes it, but likely without acting on it, so the firmware may never have had the PMU base. This driver's comment says the suspend handler's DM-bus idle query never completes without that base, and the stop handler is likely the same. Send 0x29 on IO, after START, as macOS does. A refusal still fails the start-up, since the transport doesn't carry on after a refused command. Without the base the clean stop isn't tried. Before the stop, macOS turns the h17 push telemetry off (engine + 0x196c000: wait for bit 4 to clear, then clear bit 0). It then sets SCRATCH14 to 0 and waits up to 50 ms for SCRATCH15 bit 0, and sets SCRATCH16 to 1 and waits for SCRATCH17 bit 0. Do the same, in that order, going on after a miss as macOS does. Not changed: the stop and POWER_DOWN commands, their ring, the SCRATCH0 handshake and SCRATCH1, all of which already match macOS byte for byte. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 55 ++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 3c1b2adc31657..80c127957c9ea 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -869,6 +869,32 @@ static void ane_fw_dart_deactivate(struct ane_fw_backend *b) } } +/* h17 push telemetry control; macOS sets bit 0 at start-up. */ +#define ANE_FW_TELEMETRY 0x196c000 + +/* + * macOS's step right before the stop on this chip (ANE_deInit, its h17 case): + * push telemetry off once bit 4 reads clear, then SCRATCH14 = 0 and wait for + * SCRATCH15 bit 0, then SCRATCH16 = 1 and wait for SCRATCH17 bit 0. Each wait + * is 50 ms and macOS goes on after a miss. + */ +static void ane_fw_stop_handshake(struct ane_fw_backend *b) +{ + void __iomem *scratch = b->engine + b->soc->scratch; + void __iomem *telemetry = b->engine + ANE_FW_TELEMETRY; + struct device *dev = &b->pdev->dev; + + if (ane_poll32(b, telemetry, BIT(4), 0, 10, 50000)) + dev_warn(dev, "H17 ANE: telemetry busy before the stop: %#x\n", ane_rd32(b, telemetry)); + ane_wr32(b, ane_rd32(b, telemetry) & ~BIT(0), telemetry); + ane_wr32(b, 0, scratch + 0x38); + if (ane_poll32(b, scratch + 0x3c, 1, 1, 10, 50000)) + dev_warn(dev, "H17 ANE: SCRATCH15 not set: %#x\n", ane_rd32(b, scratch + 0x3c)); + ane_wr32(b, 1, scratch + 0x40); + if (ane_poll32(b, scratch + 0x44, 1, 1, 10, 50000)) + dev_warn(dev, "H17 ANE: SCRATCH17 not set: %#x\n", ane_rd32(b, scratch + 0x44)); +} + static int ane_fw_firmware_stop(struct ane_fw_backend *b) { struct ane_fw_transport *t = &b->transport; @@ -887,11 +913,10 @@ static int ane_fw_firmware_stop(struct ane_fw_backend *b) * No EXCLAVE_MODE_STOP here: macOS's AP never rings that channel around its sleep * (recorded), and after it this firmware doesn't answer the stop. */ - /* The host flags as macOS sets them before its stop: SCRATCH14 = 0, SCRATCH16 = 1. */ - ane_wr32(b, 0, b->engine + b->soc->scratch + 0x38); - ane_rd32(b, b->engine + b->soc->scratch + 0x3c); - ane_wr32(b, 1, b->engine + b->soc->scratch + 0x40); - ane_rd32(b, b->engine + b->soc->scratch + 0x44); + /* Without the PMU base the stop can't finish: its DM-bus idle query needs it. */ + if (!b->pmu_base_set) + return -ENODEV; + ane_fw_stop_handshake(b); put_unaligned_le16(ANE_FW_CMD_STOP, stop + 4); ret = ane_fw_command(b, ANE_FW_IO, stop, sizeof(stop), ktime_get_ns() + ANE_FW_TIMEOUT_NS); if (ret) { @@ -1703,22 +1728,24 @@ static int ane_fw_command(struct ane_fw_backend *b, unsigned int channel, } /* - * SET_SNE_PMU_BASE2 (0x29) on the control ring with the DT "pmgr" range - * base, once per cold firmware boot. Without it the 0x24 handler's DM-bus - * idle query never completes (C2-C7; SPEC-ANE-SLEEP-H17-r2.md). + * SET_SNE_PMU_BASE2 (0x29) with the DT "pmgr" range base, once per firmware + * boot, right after START, on the IO ring. macOS's command helper takes a + * queue, not a table position: 0 is IO, 1 DEBUG, 2 DATA_CHAIN_H2T + * (aneFirmwareCommandSend, 25G83), and 0x29 goes on queue 0, never DEBUG. */ static int ane_fw_pmu_base(struct ane_fw_backend *b) { - struct ane_fw_transport *t = &b->transport; + u8 command[ANE_FW_PMU_BASE_SIZE] = {}; u64 base = b->window[1].pa; int ret; b->pmu_base_set = false; - ret = ane_fw_transport_publish_pmu_base(t, base, ktime_get_ns() + ANE_FW_TIMEOUT_NS); - if (!ret) - ret = ane_fw_transport_wait(t, t->control_ch); - dev_info(&b->pdev->dev, "H17 ANE: SET_SNE_PMU_BASE2 (0x29) %#llx on ring %s: %d\n", - base, ane_fw_channel_name(t->control_ch), ret); + put_unaligned_le16(ANE_FW_CMD_SET_PMU_BASE, command + 4); + put_unaligned_le64(base, command + 8); + ret = ane_fw_command(b, ANE_FW_IO, command, sizeof(command), + ktime_get_ns() + ANE_FW_TIMEOUT_NS); + dev_info(&b->pdev->dev, "H17 ANE: SET_SNE_PMU_BASE2 (0x29) %#llx on ring IO: %d\n", + base, ret); if (!ret) b->pmu_base_set = true; return ret; From 74b070cce06e703cc9ae309b2560e8da1625b0ae Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 03:43:42 -0500 Subject: [PATCH 16/29] accel/ane: h17: register the node without starting the firmware macOS has no ANE firmware running until its first client. The parking at boot still started the firmware once and then cut its power with the stop unanswered, and on the first sleep after it ane_sys went off with BUSY latched. Register the session and node at probe with the transport not yet armed, and leave every domain alone. The first request maps, boots and starts the engine, as the boot did. Until then a sleep has nothing to do. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 124 ++++++++++++++++++---------- drivers/accel/ane/ane_h16_program.c | 28 +++++-- drivers/accel/ane/ane_h16_program.h | 4 + 3 files changed, 109 insertions(+), 47 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 80c127957c9ea..ea8be6982dfa9 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -519,6 +519,8 @@ static int ane_fw_suspend_firmware(struct ane_fw_backend *b) static const u8 ane_fw_text_pin[SHA256_DIGEST_SIZE]; static int ane_fw_boot(struct ane_fw_backend *b); static int ane_fw_startup(struct ane_fw_backend *b); +static int ane_fw_first_bringup(struct ane_fw_backend *b); +static ssize_t ane_fw_stage_action(struct device *dev, const char *buf, size_t count); static int ane_fw_rvbar_ready(struct ane_fw_backend *b); /* A power word goes off whole, and off means target, actual and BUSY clear. */ @@ -969,35 +971,14 @@ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) return ane_fw_power_down(b); } -/* - * Off with no firmware until a request needs it, as macOS keeps the ANE - * without a client. Called with b->lock held, the engine up at the program - * stage. Its session stays closed: ane_h16_session_enter() wakes it. - */ -static int ane_fw_park(struct ane_fw_backend *b) -{ - int ret; - - ane_h16_session_sleep(&b->session, true); - b->down_for_sleep = false; - ret = ane_fw_cold_cycle_down(b); - if (ret) { - ane_fw_quarantine(b, ret); - b->session.quarantined = true; - ane_h16_session_sleep(&b->session, false); - return ret; - } - WRITE_ONCE(b->parked, true); - WRITE_ONCE(b->session.parked, true); - return 0; -} - /* A request for a parked engine: boot it from the preload, as after a sleep. */ static int ane_fw_session_wake(void *ctx) { struct ane_fw_backend *b = ctx; int ret; + if (READ_ONCE(b->stage) == ANE_STAGE_PROBED) + return READ_ONCE(b->in_sleep) ? -EAGAIN : ane_fw_first_bringup(b); guard(mutex)(&b->lock); if (!b->parked) return 0; @@ -1193,6 +1174,9 @@ static int ane_fw_retain(struct ane_fw_backend *b) { int ret; + if (b->retained) + return 0; + /* * The kexec blocker takes built-in users only. A module, like the H16 * backend, pins itself and refuses system sleep, but cannot veto kexec. @@ -1923,6 +1907,68 @@ static int ane_fw_program_stage(struct ane_fw_backend *b) return 0; } +/* + * The node without the firmware: macOS has no ANE running until its first + * client, so the domains stay off and a sleep has nothing to stop. The first + * request maps, boots and starts the engine (ane_fw_first_bringup()). + */ +static int ane_fw_register_parked(struct ane_fw_backend *b) +{ + int ret; + + ret = ane_fw_retain(b); + if (ret) + return ret; + b->transport.soc = b->soc; + ret = ane_h16_session_init_parked(&b->session, &b->transport, &ane_fw_session_ops, b, + ANE_FW_TIMEOUT_NS); + if (ret) + return ret; + ret = apple_clpc_ane_get(&b->pdev->dev, &b->clpc); + if (ret) + return ret; + b->session.gate_hold_ns = ANE_FW_GATE_HOLD_NS; + ret = ane_h16_accel_register(&b->pdev->dev, &b->session); + if (ret) { + apple_clpc_ane_put(b->clpc); + b->clpc = NULL; + return ret; + } + ane_fw_debugfs(b); + WRITE_ONCE(b->parked, true); + return 0; +} + +/* The first request: what the boot used to do, map, boot and start-up, then admit it. */ +static int ane_fw_first_bringup(struct ane_fw_backend *b) +{ + static const char *const steps[] = { "map", "boot", "startup" }; + unsigned int flags, i; + ssize_t ret = 0; + + flags = lock_system_sleep(); + if (READ_ONCE(b->stage) != ANE_STAGE_PROBED) + goto out; + for (i = 0; ret >= 0 && i < ARRAY_SIZE(steps); i++) + ret = ane_fw_stage_action(&b->pdev->dev, steps[i], strlen(steps[i])); + mutex_lock(&b->lock); + if (ret >= 0) { + b->transport.allow_free = true; + b->stage = ANE_STAGE_PROGRAM; + ret = 0; + } else { + b->session.quarantined = true; + } + WRITE_ONCE(b->parked, false); + WRITE_ONCE(b->session.parked, false); + ane_h16_session_sleep(&b->session, false); + mutex_unlock(&b->lock); + dev_info(&b->pdev->dev, "H17 ANE: first bring-up for a request: %zd\n", ret); +out: + unlock_system_sleep(flags); + return ret; +} + static ssize_t firmware_stage_show(struct device *dev, struct device_attribute *attr, char *buf) { static const char * const names[] = { @@ -2064,30 +2110,24 @@ static bool ane_fw_handoff_present(struct platform_device *pdev, */ static void ane_fw_start_work(struct work_struct *work) { - static const char *const steps[] = { "probe", "map", "boot", "startup", "program" }; struct ane_fw_backend *b = container_of(work, struct ane_fw_backend, start); - unsigned int i, flags; + unsigned int flags; ssize_t ret; - for (i = 0; i < ARRAY_SIZE(steps); i++) { - flags = lock_system_sleep(); - ret = ane_fw_stage_action(&b->pdev->dev, steps[i], strlen(steps[i])); - unlock_system_sleep(flags); - if (ret < 0) { - dev_err(&b->pdev->dev, "H17 ANE: boot bring-up stopped at %s, error %zd\n", - steps[i], ret); - ane_fw_bind_release(b); - return; - } - } - dev_info(&b->pdev->dev, "H17 ANE: up at boot, accel node ready\n"); - /* Then off until a client's request, as macOS keeps it. */ flags = lock_system_sleep(); - mutex_lock(&b->lock); - ret = ane_fw_park(b); - mutex_unlock(&b->lock); + ret = ane_fw_stage_action(&b->pdev->dev, "probe", strlen("probe")); + if (ret >= 0) { + mutex_lock(&b->lock); + ret = ane_fw_register_parked(b); + mutex_unlock(&b->lock); + } unlock_system_sleep(flags); - dev_info(&b->pdev->dev, "H17 ANE: parked until the first request%s\n", ret ? " FAILED" : ""); + /* Nothing holds ANE_SYS now: with no firmware, genpd keeps it as macOS does. */ + ane_fw_bind_release(b); + if (ret < 0) + dev_err(&b->pdev->dev, "H17 ANE: node registration stopped: %zd\n", ret); + else + dev_info(&b->pdev->dev, "H17 ANE: accel node ready; firmware starts on the first request\n"); } static void ane_fw_observer_free(struct ane_fw_backend *b) diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index 432efc1c9bdc8..ba4077793169a 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -511,12 +511,12 @@ static void h16_drain(struct ane_h16_session *s) wake_up_all(&s->wake); } -int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, - const struct ane_h16_session_ops *ops, void *ctx, - u64 timeout_ns) +static int h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns, bool parked) { - if (!s || !transport || !transport->armed || !ops || !ops->map || !ops->log || - !timeout_ns) + if (!s || !transport || !transport->soc || (!parked && !transport->armed) || !ops || + !ops->map || !ops->log || !timeout_ns || (parked && !ops->wake)) return -EINVAL; memset(s, 0, sizeof(*s)); s->transport = transport; @@ -531,10 +531,28 @@ int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *tra INIT_LIST_HEAD(&s->programs); spin_lock_init(&s->stats_lock); INIT_DELAYED_WORK(&s->gate_work, h16_gate_work); + s->asleep = parked; + s->parked = parked; return 0; } + +int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns) +{ + return h16_session_init(s, transport, ops, ctx, timeout_ns, false); +} EXPORT_SYMBOL_GPL(ane_h16_session_init); +/* Closed and parked, its transport armed only by ops->wake on the first request. */ +int ane_h16_session_init_parked(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns) +{ + return h16_session_init(s, transport, ops, ctx, timeout_ns, true); +} +EXPORT_SYMBOL_GPL(ane_h16_session_init_parked); + static int h16_fail(struct ane_h16_session *s, int error) { if (!s->error) diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h index 50ad92a0470a2..8dee541cc2fd9 100644 --- a/drivers/accel/ane/ane_h16_program.h +++ b/drivers/accel/ane/ane_h16_program.h @@ -340,6 +340,10 @@ struct ane_h16_session { int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, const struct ane_h16_session_ops *ops, void *ctx, u64 timeout_ns); +int ane_h16_session_init_parked(struct ane_h16_session *s, + struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns); int ane_h16_session_load(struct ane_h16_session *s, struct ane_h16_hwx_snapshot *snapshot, const void *const aux[ANE_H16_AUX_COUNT], From 992923c96a5440c8724b0952a09b73813d0ebeff Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 04:16:39 -0500 Subject: [PATCH 17/29] accel/ane: h17: deactivate the ANE DARTs before ANE_SYS powers off With the engine parked and no firmware started after a wake, ane_sys still went off with BUSY latched on 47 of 52 sleeps, and the PMGR driver had to clear it each time. macOS never needs that clear. In the recording with the ANE visible (25G83) it deactivates the three ANE DARTs before every ane_sys power-off. It does this after its firmware stop, and also when it has powered ane_sys up only for DART work, with no firmware: on, DART setup, TLB flushes by address, then the deactivation and off. The deactivation, per DART: error mask 0x4060 = all ones, exception word 0x4000 = 0, error status 0x704 = all ones, each enabled stream (and 11 and 15) disabled one at a time through 0xc20, then a full flush of all three DARTs. The off is one write, the same word the PMGR driver writes here. It reads back with BUSY clear. Here the ANE DARTs keep their streams as the loader left them (cold ANE DARTs are neither saved nor reset at suspend), they hold ane_sys on while the engine is parked, and genpd turns it off in the noirq phase with every stream still enabled. The DART deactivation already exists, but it runs only after a firmware stop, which a parked engine never does. Run it from the ANE_SYS genpd notifier, on GENPD_NOTIFY_PRE_OFF, so it precedes every power-off, runtime or noirq, as on macOS. It is MMIO only and safe under genpd's lock. The next boot of the engine re-attaches SID0, which enables that stream again; it uses no other stream. ane.dart_off=0 turns it off for an A/B. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index ea8be6982dfa9..ca72c8355867b 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -1064,13 +1064,35 @@ static int ane_fw_sleep_prepare(struct ane_fw_backend *b) return 0; } -/* Called by genpd under its lock, in the noirq phase too: record only. */ +static bool dart_off = true; +module_param(dart_off, bool, 0644); +MODULE_PARM_DESC(dart_off, "Deactivate the H17 ANE DARTs before ANE_SYS powers off, as macOS does"); + +/* + * Called by genpd under its lock, in the noirq phase too: nothing that sleeps. + * + * macOS deactivates the three ANE DARTs before every ANE_SYS power-off, with + * or without firmware: it also powers ANE_SYS up for DART work alone and down + * again this way (recorded, 25G83). Here ANE_SYS goes off in genpd, held on + * by the DARTs until then, with their streams still enabled; on most sleeps + * it then reaches off with BUSY latched. Do macOS's step right before the off. + */ static int ane_fw_genpd_event(struct notifier_block *nb, unsigned long action, void *data) { struct ane_fw_backend *b = container_of(nb, struct ane_fw_backend, genpd_nb); u64 now = ktime_get_boottime_ns(); - if (action == GENPD_NOTIFY_OFF) { + if (action == GENPD_NOTIFY_PRE_OFF && dart_off && b->soc->exclave_mode_stop && + b->dart[0] && b->dart[1] && b->dart[2]) { + u32 streams[ANE_FW_DARTS]; + + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) + streams[i] = ane_rd32(b, b->dart[i] + 0xc00); + ane_fw_dart_deactivate(b); + dev_info(&b->pdev->dev, + "H17 ANE: DARTs off before ANE_SYS: streams were %#x %#x %#x, PMGR %#x\n", + streams[0], streams[1], streams[2], ane_rd32(b, b->pmgr + 0x290)); + } else if (action == GENPD_NOTIFY_OFF) { b->sys_offs++; b->sys_off_pmgr = ane_rd32(b, b->pmgr + 0x290); b->sys_off_at = now; From 05799966486c23e73732b0ca99cc039f02d3199e Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 05:21:10 -0500 Subject: [PATCH 18/29] pmdomain: apple: count a T8140 domain off only once BUSY clears macOS waits for (word & 0x8ff) == 0 after lowering a domain, which takes in the BUSY bit. The actual state reads 0 a moment before the switch is done, and the driver stopped there: ane_sys then read 0x2800, BUSY with no history bits yet, and was taken for a latch and pulsed. Wait up to 1 ms for BUSY to clear first, as macOS does, and treat only a BUSY that stays as a latch. The same in the sleep's last check. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmgr-pwrstate.c | 12 ++++++++++++ drivers/soc/apple/sleep.c | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 454e43e9faca2..f59bd64fa41f9 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -43,6 +43,8 @@ #define APPLE_PMGR_PS_PWRGATE 0x0 #define APPLE_PMGR_PS_SET_TIMEOUT 100 +/* How long a lowered domain may report BUSY while its switch finishes */ +#define APPLE_PMGR_BUSY_SETTLE_US 1000 #define APPLE_PMGR_PS_CONSUMER_TIMEOUT 1000 #define APPLE_PMGR_PS_MAX_RETENTION 4 #define APPLE_PMGR_PS_MAX_CHAIN 4 @@ -308,6 +310,16 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a * reset clears it, and the domain is off, so pulse one as on the power-on * path (not in a system sleep transition, see apple_pmgr_ps_clear_latch()). */ + /* + * Off only once BUSY clears too: macOS waits for (word & 0x8ff) == 0. The + * actual state reads 0 a moment before the switch is done. BUSY still set + * after this is a latch. + */ + if (!ret && ps->macos_lower && pstate != APPLE_PMGR_PS_ACTIVE && (cur & APPLE_PMGR_BUSY)) + regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, cur, + !(cur & APPLE_PMGR_BUSY), 1, + APPLE_PMGR_BUSY_SETTLE_US); + if (!ret && pstate != APPLE_PMGR_PS_ACTIVE && (cur & APPLE_PMGR_BUSY) && !FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) && (!ps->system_sleep || ps->macos_lower) && !ps->wait_parent && !ps->strict_state && !ps->externally_clocked) { diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 180849fee5775..ae378e3fc4744 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -80,6 +80,7 @@ struct aura_s2r_mailbox { /* PS word */ #define PS_AUTO_ENABLE BIT(28) #define PS_UNLATCH BIT(12) +#define PS_BUSY_SETTLE_US 1000 #define PS_DEV_DISABLE BIT(10) #define PS_FLAGS GENMASK(9, 8) #define PS_ACTUAL GENMASK(7, 4) @@ -2467,6 +2468,9 @@ static bool apple_sleep_ps_unlatch(struct apple_sleep *s) if (!(val & PS_BUSY) || (val & (PS_ACTUAL | PS_TARGET))) continue; + /* A switch still finishing reads BUSY for a moment: only one that stays is a latch. */ + if (!readl_poll_timeout_atomic(reg, val, !(val & PS_BUSY), 1, PS_BUSY_SETTLE_US)) + continue; s->ps_unlatched[s->ps_unlatched_count].offset = words->offset[i]; s->ps_unlatched[s->ps_unlatched_count++].val = val; off = val & ~(PS_AUTO_ENABLE | PS_FLAGS | PS_BUSY | PS_UNLATCH | PS_DEV_DISABLE); From 0b5617bfb97ae01cff31dbb6aa7c8aaac3d297e9 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 06:16:59 -0500 Subject: [PATCH 19/29] pmdomain: apple: finish a T8140 domain's power-off before a sleep isp_dprx comes up from the loader powered, while genpd has it off. On the first sleep of each boot the sleep driver lowered it at the commit, since macOS has it off there, raised it again at the wake because it had been on, and the wake's resync then lowered it a second time with BUSY left, which the latch clear pulsed. In three recordings of macOS's sleep (25G83) macOS has isp_dprx off at the commit and neither reads nor writes it after a wake. When a system sleep starts, power off a domain of the main PMGR that genpd has off but that is still powered. The sleep then finds it off, nothing raises it at the wake, and nothing lowers it again. Not in the always-on PMGR, whose words the AOP raises for itself. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmgr-pwrstate.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index f59bd64fa41f9..06e10ac43b36b 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -529,6 +529,17 @@ static int apple_pmgr_ps_resume_early(struct device *dev) * whose target is still on was raised by a coprocessor's firmware: on T8140 the AOP * raises audio domains of its own in the always-on PMGR, and the sleep keeps them. */ +/* T8140's always-on PMGR, the block the AOP shares */ +#define APPLE_PMGR_T8140_AON_BASE 0x308280000ULL + +static bool apple_pmgr_ps_in_aon(struct device_node *node) +{ + struct resource res; + + return node->parent && !of_address_to_resource(node->parent, 0, &res) && + res.start == APPLE_PMGR_T8140_AON_BASE; +} + static int apple_pmgr_ps_suspend_noirq(struct device *dev) { struct apple_pmgr_ps *ps = dev_get_drvdata(dev); @@ -541,6 +552,20 @@ static int apple_pmgr_ps_suspend_noirq(struct device *dev) spin_lock_irqsave(&ps->genpd.slock, flags); if (!regmap_read(ps->regmap, ps->offset, ®)) { + /* + * Off for genpd but still powered, as the loader left it (T8140 + * isp_dprx): finish the power-off now. Otherwise the sleep lowers it + * at the commit, raises it again at the wake, and the resync lowers + * it a second time with BUSY left. macOS has it off at the commit and + * doesn't touch it after the wake. Not in the always-on PMGR, whose + * words the AOP raises for itself. + */ + if (ps->macos_lower && ps->genpd.status == GENPD_STATE_OFF && + FIELD_GET(APPLE_PMGR_PS_TARGET, reg) && FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) && + !apple_pmgr_ps_in_aon(dev->of_node)) { + apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_PWRGATE, false); + regmap_read(ps->regmap, ps->offset, ®); + } ps->sleep_target = FIELD_GET(APPLE_PMGR_PS_TARGET, reg); ps->sleep_raised_by_other = ps->genpd.status == GENPD_STATE_OFF && ps->sleep_target != APPLE_PMGR_PS_PWRGATE; From 5801e1f1135dd394cbd6c6e584224fcbb797ba11 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 08:08:14 -0500 Subject: [PATCH 20/29] nvme-apple: give a sleep's shutdown notification the controller's time On the Neo about one sleep in 100 without the debug cable was refused at the last check: the controller still reported the shutdown in progress (CSTS 0x5) when the sleep path's fixed 1 s ran out. The NVMe core waits the controller's RTD3E, at least nvme_core.shutdown_timeout (5 s), for the same notification, and macOS waits for CSTS 9. Wait that long here too, and report a shutdown that took over 100 ms after the wake. Signed-off-by: Diljit Singh --- drivers/nvme/host/apple.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index ded0a31e0e3b0..8e953d861e5de 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -253,6 +253,8 @@ struct apple_nvme { bool sleep_stop; bool sleep_idled; u32 sleep_csts; + /* The last sleep's shutdown notification, when it took over 100 ms */ + u64 sleep_shn_slow_us; /* ANS came back across a sleep, or failed to. reset_work picks these up. */ bool sleep_restarted; bool sleep_failed; @@ -2670,17 +2672,33 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) static int apple_nvme_sleep_shutdown(void *data) { struct apple_nvme *anv = data; + /* As long as the controller says shutdown takes (RTD3E), as the NVMe core waits. */ + u64 timeout = max_t(u64, APPLE_ANS_SLEEP_SHN_TIMEOUT, + (u64)anv->ctrl.shutdown_timeout * USEC_PER_SEC); + u64 waited = 0; u32 csts; - int ret; + int ret = 0; anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; anv->ctrl.ctrl_config |= anv->hw->sleep_shn; writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); - ret = readl_poll_timeout_atomic(anv->mmio_nvme + NVME_REG_CSTS, csts, - (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, - 100, APPLE_ANS_SLEEP_SHN_TIMEOUT); + /* Timekeeping is suspended here: count the wait in its own steps. */ + for (;;) { + csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) + break; + if (waited >= timeout) { + ret = -ETIMEDOUT; + break; + } + udelay(100); + waited += 100; + } if (ret) - dev_err(anv->dev, "shutdown notification failed: CSTS %#x\n", csts); + dev_err(anv->dev, "shutdown notification failed after %llu us: CSTS %#x\n", + waited, csts); + else if (waited > 100 * USEC_PER_MSEC) + anv->sleep_shn_slow_us = waited; anv->sleep_csts = csts; return ret; @@ -2860,6 +2878,12 @@ static int apple_nvme_resume_early(struct device *dev) void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; int ret; + if (anv->sleep_shn_slow_us) { + dev_info(dev, "sleep's shutdown notification took %llu ms\n", + div_u64(anv->sleep_shn_slow_us, USEC_PER_MSEC)); + anv->sleep_shn_slow_us = 0; + } + if (!anv->hw->needs_ioq_registers || !anv->suspended || !anv->sleep_idled) return 0; anv->sleep_idled = false; From 8ee99a76b735493077040d8412d31dc2f0014237 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 09:08:28 -0500 Subject: [PATCH 21/29] nvme-apple: send a sleep's shutdown notification at suspend, as macOS does macOS sends the NVMe shutdown notification from IONVMeController's power state change during the device suspend, CC 0x46c001, and waits with interrupts on for up to 35 s (timeout=35000 in its log, 25G83). The controller can take seconds to save its context: 1674 ms in one recorded macOS sleep, 69 ms in another. This driver sent it in the sleep's last check with interrupts off and gave it 1 s, which refused about one sleep in 20 without the debug cable. Send it at suspend, after the I/O quiesce, wait sleepably as long as macOS does, and refuse the sleep if it never completes. The last check now only reads that it did. A shutdown over 1 s is logged. Signed-off-by: Diljit Singh --- drivers/nvme/host/apple.c | 82 +++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 8e953d861e5de..f0892f4986b94 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -253,8 +253,6 @@ struct apple_nvme { bool sleep_stop; bool sleep_idled; u32 sleep_csts; - /* The last sleep's shutdown notification, when it took over 100 ms */ - u64 sleep_shn_slow_us; /* ANS came back across a sleep, or failed to. reset_work picks these up. */ bool sleep_restarted; bool sleep_failed; @@ -2665,43 +2663,19 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } /* - * A sleep's shutdown notification, with interrupts off, before anything else stops. macOS - * finds the controller enabled and shut down when it tells the firmware to idle. A failure - * refuses: the firmware hasn't been told, and resume's controller reset takes it back. + * The sleep's last check, with interrupts off: the shutdown notification sent at suspend + * (apple_nvme_sleep_send_shn()) has completed. Nothing is written here. */ static int apple_nvme_sleep_shutdown(void *data) { struct apple_nvme *anv = data; - /* As long as the controller says shutdown takes (RTD3E), as the NVMe core waits. */ - u64 timeout = max_t(u64, APPLE_ANS_SLEEP_SHN_TIMEOUT, - (u64)anv->ctrl.shutdown_timeout * USEC_PER_SEC); - u64 waited = 0; - u32 csts; - int ret = 0; + u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); - anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; - anv->ctrl.ctrl_config |= anv->hw->sleep_shn; - writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); - /* Timekeeping is suspended here: count the wait in its own steps. */ - for (;;) { - csts = readl(anv->mmio_nvme + NVME_REG_CSTS); - if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) - break; - if (waited >= timeout) { - ret = -ETIMEDOUT; - break; - } - udelay(100); - waited += 100; - } - if (ret) - dev_err(anv->dev, "shutdown notification failed after %llu us: CSTS %#x\n", - waited, csts); - else if (waited > 100 * USEC_PER_MSEC) - anv->sleep_shn_slow_us = waited; anv->sleep_csts = csts; - - return ret; + if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) + return 0; + dev_err(anv->dev, "shutdown notification not complete at the commit: CSTS %#x\n", csts); + return -EIO; } /* @@ -2878,11 +2852,6 @@ static int apple_nvme_resume_early(struct device *dev) void __iomem *control = anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL; int ret; - if (anv->sleep_shn_slow_us) { - dev_info(dev, "sleep's shutdown notification took %llu ms\n", - div_u64(anv->sleep_shn_slow_us, USEC_PER_MSEC)); - anv->sleep_shn_slow_us = 0; - } if (!anv->hw->needs_ioq_registers || !anv->suspended || !anv->sleep_idled) return 0; @@ -3027,6 +2996,36 @@ static bool apple_nvme_sleep_wanted(struct apple_nvme *anv) apple_rtkit_is_running(anv->rtk); } +/* + * As macOS sends it, from its power-state change during the device suspend: the shutdown + * notification, waited for with interrupts on for as long as macOS allows (35 s). The + * controller can take seconds to save its context (1.7 s in one macOS sleep). A sleep + * refused later finds it shut down, and resume's controller reset takes it back. + */ +#define APPLE_ANS_SLEEP_SHN_MACOS_TIMEOUT_US (35 * USEC_PER_SEC) + +static int apple_nvme_sleep_send_shn(struct apple_nvme *anv) +{ + ktime_t start = ktime_get(); + u32 csts; + s64 us; + int ret; + + anv->ctrl.ctrl_config &= ~NVME_CC_SHN_MASK; + anv->ctrl.ctrl_config |= anv->hw->sleep_shn; + writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); + ret = readl_poll_timeout(anv->mmio_nvme + NVME_REG_CSTS, csts, + (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, + 1000, APPLE_ANS_SLEEP_SHN_MACOS_TIMEOUT_US); + us = ktime_us_delta(ktime_get(), start); + if (ret) + dev_err(anv->dev, "shutdown notification failed after %lld ms: CSTS %#x\n", + div_s64(us, USEC_PER_MSEC), csts); + else if (us > USEC_PER_SEC) + dev_info(anv->dev, "shutdown notification took %lld ms\n", div_s64(us, USEC_PER_MSEC)); + return ret; +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -3076,6 +3075,13 @@ static int apple_nvme_suspend(struct device *dev) nvme_try_sched_reset(&anv->ctrl); return -EIO; } + if (anv->sleep_stop && apple_nvme_sleep_send_shn(anv)) { + dev_err(dev, "refusing to sleep: NVMe shutdown notification didn't complete\n"); + anv->sleep_stop = false; + WRITE_ONCE(anv->suspended, false); + nvme_try_sched_reset(&anv->ctrl); + return -EIO; + } if (anv->sleep_stop) { apple_sleep_coproc_set_ready(dev, true); dev_dbg(dev, "NVMe suspended for a sleep\n"); From c2b4f4b46ec38cdaa0bb5317415727fb18c91eae Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 11:12:58 -0500 Subject: [PATCH 22/29] iommu/apple-dart: let the cold ANE DARTs runtime-suspend macOS keeps ANE_SYS off while the ANE has no client. Here the three cold ANE DARTs kept their probe runtime-PM reference, so they held ANE_SYS on from boot, and from every wake, until the next system sleep, even with the engine parked. Drop that reference as the other DARTs do. Their runtime suspend and resume stay no-ops. The ANE backend already deactivates the DARTs on genpd's PRE_OFF, and it attaches SID0 again after each power-up (the first bring-up, and the cold cycle), so nothing depends on their state surviving the power-off. The PMGR domains are IRQ-safe, so the DARTs' IRQ-safe runtime PM doesn't keep the domain on either. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 18 ++++++++++-------- drivers/iommu/apple-dart.c | 15 +++++++++------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index ca72c8355867b..ab59ccfa9e773 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -580,10 +580,10 @@ static int ane_fw_power_down(struct ane_fw_backend *b) b->powered = false; b->power_ref = false; /* - * This only drops our reference. The ANE DARTs share the domain and stay - * active, so genpd turns it off in the noirq phase of the system suspend. - * ane_fw_genpd_event() records that, and the resume reports a domain that - * stayed on. + * Our reference was the last one on ANE_SYS: genpd turns it off once the + * ANE DARTs are idle, or at the latest in the noirq phase of the system + * suspend. ane_fw_genpd_event() records that, and the resume reports a + * domain that stayed on. */ pm_runtime_put_sync(b->power); dev_info(&b->pdev->dev, "H17 ANE: powered down for system sleep\n"); @@ -1073,9 +1073,11 @@ MODULE_PARM_DESC(dart_off, "Deactivate the H17 ANE DARTs before ANE_SYS powers o * * macOS deactivates the three ANE DARTs before every ANE_SYS power-off, with * or without firmware: it also powers ANE_SYS up for DART work alone and down - * again this way (recorded, 25G83). Here ANE_SYS goes off in genpd, held on - * by the DARTs until then, with their streams still enabled; on most sleeps - * it then reaches off with BUSY latched. Do macOS's step right before the off. + * again this way (recorded, 25G83). Here ANE_SYS goes off in genpd, at + * runtime once the ANE and its DARTs are idle or in the noirq phase of a + * system suspend, with the DARTs' streams still enabled; without this step + * it reached off with BUSY latched on most sleeps. Do macOS's step right + * before the off. */ static int ane_fw_genpd_event(struct notifier_block *nb, unsigned long action, void *data) { @@ -2144,7 +2146,7 @@ static void ane_fw_start_work(struct work_struct *work) mutex_unlock(&b->lock); } unlock_system_sleep(flags); - /* Nothing holds ANE_SYS now: with no firmware, genpd keeps it as macOS does. */ + /* Nothing holds ANE_SYS now: with no firmware, genpd turns it off, as macOS keeps it. */ ane_fw_bind_release(b); if (ret < 0) dev_err(&b->pdev->dev, "H17 ANE: node registration stopped: %zd\n", ret); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 7c7c2531868a1..0f393ee4aa8af 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -5526,9 +5526,12 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_sysfs_remove; - /* Retain supplier references; generic suspend/resume is not applicable. */ - if (!dart->retained_mtp && !dart->managed_sids && !dart->cold_ane && - !apple_dart_has_borrowed(dart)) + /* + * Retain supplier references; generic suspend/resume is not applicable. + * Not for a cold ANE DART: ANE_SYS goes off while the ANE has no client, + * as on macOS, instead of staying on until the next system sleep. + */ + if (!dart->retained_mtp && !dart->managed_sids && !apple_dart_has_borrowed(dart)) pm_runtime_put(dev); dev_dbg( @@ -6033,9 +6036,9 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) if (READ_ONCE(dart->commands_gated)) return 0; /* - * The cold ANE DARTs belong to the ANE backend, which refuses system - * sleep up front while it runs the firmware. Otherwise they hold - * nothing to keep: neither save nor reset them. + * The cold ANE DARTs belong to the ANE backend: it deactivates them + * before ANE_SYS goes off and attaches SID0 again once it is back on. + * Neither save nor reset them. */ if (dart->cold_ane) return 0; From 269dec726d3b1ae25a19ec4b8107aafced06640d Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 12:09:17 -0500 Subject: [PATCH 23/29] pmdomain: apple: keep the noirq suspend's comment on its function The always-on PMGR check went in between apple_pmgr_ps_suspend_noirq() and the comment that describes it. Move the check above the comment. No functional change. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmgr-pwrstate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 4fb479143f14f..a97060f2efe16 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -525,11 +525,6 @@ static int apple_pmgr_ps_resume_early(struct device *dev) return 0; } -/* - * By the noirq phase genpd has powered off what it is going to. A domain it has off - * whose target is still on was raised by a coprocessor's firmware: on T8140 the AOP - * raises audio domains of its own in the always-on PMGR, and the sleep keeps them. - */ /* T8140's always-on PMGR, the block the AOP shares */ #define APPLE_PMGR_T8140_AON_BASE 0x308280000ULL @@ -541,6 +536,11 @@ static bool apple_pmgr_ps_in_aon(struct device_node *node) res.start == APPLE_PMGR_T8140_AON_BASE; } +/* + * By the noirq phase genpd has powered off what it is going to. A domain it has off + * whose target is still on was raised by a coprocessor's firmware: on T8140 the AOP + * raises audio domains of its own in the always-on PMGR, and the sleep keeps them. + */ static int apple_pmgr_ps_suspend_noirq(struct device *dev) { struct apple_pmgr_ps *ps = dev_get_drvdata(dev); From a2ec625b2afa55084d749a651b41d5e3c52b82e6 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 12:09:18 -0500 Subject: [PATCH 24/29] soc: apple: sleep: document apple_sleep_request_deep_once() Every other export in the header says what it is for and when to call it; this one didn't. Say so, and move PS_BUSY_SETTLE_US from the PS word bit fields to the other PS timeout. No functional change. Signed-off-by: Diljit Singh --- drivers/soc/apple/sleep.c | 2 +- include/linux/soc/apple/sleep.h | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index ae378e3fc4744..c78be9b28b3da 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -80,12 +80,12 @@ struct aura_s2r_mailbox { /* PS word */ #define PS_AUTO_ENABLE BIT(28) #define PS_UNLATCH BIT(12) -#define PS_BUSY_SETTLE_US 1000 #define PS_DEV_DISABLE BIT(10) #define PS_FLAGS GENMASK(9, 8) #define PS_ACTUAL GENMASK(7, 4) #define PS_TARGET GENMASK(3, 0) #define PS_OFF_TIMEOUT_US 5000 +#define PS_BUSY_SETTLE_US 1000 /* CPU start block, as aura's smp_start_cpu() uses it */ #define CPU_START_STOP 0x0 diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 4c408dbd798f6..6ce00ba6001d9 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -34,6 +34,11 @@ int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), void *data); void apple_sleep_coproc_unregister(struct device *dev); + +/* + * During a suspend: a driver left something the DRAM-kept wake would stall on, so this + * sleep takes the deep path. False if no sleep driver can do that. + */ bool apple_sleep_request_deep_once(void); /* From b8f5b1e7f750a60e86c7bee83eebdfdea85aeb9d Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 12:09:40 -0500 Subject: [PATCH 25/29] accel/ane: h17: drop the branches left from runtime parking The engine no longer parks at runtime: it powers down only for a system sleep, so down_for_sleep is always set when its readers run, and the "while parking" branches are dead. Remove the flag and those branches. No functional change. Signed-off-by: Diljit Singh --- drivers/accel/ane/ane_fw_cold.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index ab59ccfa9e773..d21e3567bbbdd 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -83,8 +83,6 @@ struct ane_fw_backend { bool parked; /* Between PM_SUSPEND_PREPARE and PM_POST_SUSPEND. */ bool in_sleep; - /* This power-down is for a system sleep: a stuck one may send it deep. */ - bool down_for_sleep; /* The last cold boot after sleep: firmware ready, first replay, count. */ u64 resume_ready_ns, resume_first_ns; unsigned int resume_replayed; @@ -547,10 +545,6 @@ static int ane_fw_ps_off(struct ane_fw_backend *b, u32 offset) if (!ane_poll32(b, reg, ANE_FW_PS_OFF_MASK, 0, 1, 1000)) return 0; } - if (!b->down_for_sleep) { - dev_warn(&b->pdev->dev, "H17 ANE: power word %#x still stuck while parking\n", offset); - return 0; - } if (!apple_sleep_request_deep_once()) return ret; dev_warn(&b->pdev->dev, "H17 ANE: power word %#x still stuck, taking the deep path\n", offset); @@ -959,11 +953,8 @@ static int ane_fw_cold_cycle_down(struct ane_fw_backend *b) return -EBUSY; if (ane_fw_clean_stop(b)) { ret = ane_fw_firmware_stop(b); - /* Parking at runtime: the PMGR clears a latch outside a system sleep. */ - if (ret && !b->down_for_sleep) - dev_warn(&b->pdev->dev, "H17 ANE: no clean stop (%d) while parking\n", ret); /* A refused sleep is a failed suspend: the deep path cuts its power cleanly. */ - else if (ret && apple_sleep_request_deep_once()) + if (ret && apple_sleep_request_deep_once()) dev_warn(&b->pdev->dev, "H17 ANE: no clean stop (%d), taking the deep path\n", ret); else if (ret) return ret == -EBUSY ? -EIO : ret; @@ -1005,7 +996,6 @@ static int ane_fw_sleep_prepare(struct ane_fw_backend *b) /* Parked: already off, with nothing to stop. */ if (b->parked) return 0; - b->down_for_sleep = true; /* * A quarantined engine that never came up has nothing to lose; give * back an ANE_SYS reference a failed power-up kept. From 7fedcbe258340c63f5292811b3fcd20d1974dde0 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 15:44:25 -0500 Subject: [PATCH 26/29] media: apple: isp: quiesce the T8140 coprocessor before its power-down Each time a T8140 stream stops, isp_cpu_core0 and isp_cpu_core1 reach off with BUSY latched, and the PMGR driver has to clear them with a PS reset. macOS doesn't need that. Its DisableISPClocksAndPower (25G83), after the firmware stop and the DART flush and before the ISP's power words go off, writes bit 25 to the coprocessor's quiesce register (ASC + 0x40) and waits up to 50 ms for the bit to read back. It does this every time, on the path T8140 takes. Its recording then shows both core words reading off and clean, 0x0f000300. That is the step T8132 and T6050 already take here (coproc_quiesce). Take it on T8140 too. The run bit stays set, as before: macOS clears it here only when a field is set that no code on this path sets. The rule that only restored firmware quiesces now also allows firmware that starts again from its own memory. Signed-off-by: Diljit Singh --- drivers/media/platform/apple/isp/isp-drv-test.c | 6 +++--- drivers/media/platform/apple/isp/isp-drv.c | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv-test.c b/drivers/media/platform/apple/isp/isp-drv-test.c index a1e60e5e29d15..d8a54c1091f72 100644 --- a/drivers/media/platform/apple/isp/isp-drv-test.c +++ b/drivers/media/platform/apple/isp/isp-drv-test.c @@ -113,8 +113,8 @@ kunit_test_suite(isp_dapf_test_suite); /* * A deferred start detects the camera once: its firmware must keep running - * or start again. Firmware that starts again is not resident, and only it - * is quiesced before the power-down or idles after a stream. + * or start again. Only firmware that starts again, from a restored copy or + * from its own memory, is quiesced before the power-down. */ static void isp_test_firmware_lifetime(struct kunit *test) { @@ -129,7 +129,7 @@ static void isp_test_firmware_lifetime(struct kunit *test) KUNIT_EXPECT_FALSE_MSG(test, hw->resident_fw && hw->fw_restore, "%s", match->compatible); if (hw->coproc_quiesce) - KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore, "%s", + KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore || hw->fw_restart, "%s", match->compatible); /* Only firmware that starts again idles before it stops. */ if (hw->fw_idle_ms) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 87b804dfd8c92..80a94d5ac5e11 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -1065,6 +1065,13 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .mbox2_offset = 0x410, .mbox_irq_route = true, .coproc_control = ISP_COPROC_CONTROL_T8140, + /* + * macOS quiesces the coprocessor before the core words go off + * (DisableISPClocksAndPower, 25G83); without it they reach off with BUSY + * latched. + */ + .coproc_status = ISP_COPROC_STATUS_T8140, + .coproc_quiesce = ISP_COPROC_QUIESCE_T8140, .fw_mmio = apple_isp_fw_mmio_t8140, .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8140), .boot_mode = 1, From e0549e4d43ee7e25853dc8e45eb1660e42e7444b Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 15:44:51 -0500 Subject: [PATCH 27/29] media: apple: isp: keep the T8140 firmware up 5 s after a stream, as macOS does T8140's firmware stops at the end of every stream here, and starts again for the next one. macOS keeps it running after a stream. Its camera service asks the ISP driver to power the ISP down later (ISP_PowerOffCamera, 25G83). In the recording with Photo Booth, the firmware stop came 5.07 s after the last frame both times: once on the way into a sleep, and once in a short capture after the wake with no sleep near it. The ISP driver itself has no idle timer; the delay is the service's. Do the same on T8140, with the idle stop T8132 and T6050 already have (fw_idle_ms), for firmware that starts again from its own memory. A stream that starts within the 5 s uses the running firmware. After 5 s it stops and powers down, as the end of a stream did. A system sleep still finds the firmware off. macOS's service powers an idle ISP off before the kernel sleeps, and here a PM notifier stops idle firmware at PM_SUSPEND_PREPARE, before any device suspends. A stream running at sleep is stopped by the suspend callback as before. Signed-off-by: Diljit Singh --- .../media/platform/apple/isp/isp-drv-test.c | 2 +- drivers/media/platform/apple/isp/isp-drv.c | 24 +++++++++++++++++++ drivers/media/platform/apple/isp/isp-drv.h | 4 ++++ drivers/media/platform/apple/isp/isp-fw.c | 10 ++++---- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv-test.c b/drivers/media/platform/apple/isp/isp-drv-test.c index d8a54c1091f72..2ff43d1c7ae26 100644 --- a/drivers/media/platform/apple/isp/isp-drv-test.c +++ b/drivers/media/platform/apple/isp/isp-drv-test.c @@ -133,7 +133,7 @@ static void isp_test_firmware_lifetime(struct kunit *test) match->compatible); /* Only firmware that starts again idles before it stops. */ if (hw->fw_idle_ms) - KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore, "%s", + KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore || hw->fw_restart, "%s", match->compatible); /* The fallback keeps restartable firmware resident. */ if (hw->resident_fallback) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 80a94d5ac5e11..a1a79ad97b4a7 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -635,6 +635,21 @@ static void apple_isp_resume_work(struct work_struct *work) apple_isp_video_resume(isp); } +/* + * Idle firmware that starts again from its own memory is stopped before a + * system sleep begins, as macOS's camera service powers an idle ISP off + * before the kernel sleeps: the sleep then finds it off, as after a stream. + */ +static int apple_isp_pm_notify(struct notifier_block *nb, unsigned long event, + void *unused) +{ + struct apple_isp *isp = container_of(nb, struct apple_isp, pm_nb); + + if (event == PM_SUSPEND_PREPARE || event == PM_HIBERNATION_PREPARE) + apple_isp_firmware_flush(isp); + return NOTIFY_DONE; +} + static int apple_isp_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -813,6 +828,11 @@ static int apple_isp_probe(struct platform_device *pdev) goto halt_firmware; } + if (isp->hw->fw_idle_ms && isp->hw->resident_fw) { + isp->pm_nb.notifier_call = apple_isp_pm_notify; + register_pm_notifier(&isp->pm_nb); + } + return 0; halt_firmware: @@ -837,6 +857,8 @@ static void apple_isp_remove(struct platform_device *pdev) { struct apple_isp *isp = platform_get_drvdata(pdev); + if (isp->pm_nb.notifier_call) + unregister_pm_notifier(&isp->pm_nb); cancel_work_sync(&isp->resume_work); apple_isp_remove_video(isp); /* Stop resident firmware before freeing what it may still use. */ @@ -1080,6 +1102,8 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .fw_restart = true, /* As on macOS, the firmware first starts when a client opens the camera. */ .deferred_start = true, + /* macOS powers it off 5 s after the last frame (twice in its recording). */ + .fw_idle_ms = 5000, .meta_exposure = true, }; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 0b1b34a61ae49..35989fa42683b 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -426,6 +427,9 @@ struct apple_isp { /* configurations the firmware has, once a deferred start detected them */ u32 num_fw_presets; + /* stops idle resident firmware before a system sleep */ + struct notifier_block pm_nb; + /* stopped for system sleep, to start again on resume */ bool fw_sleep_stopped; /* restarts a stream that ran across that sleep, see apple_isp_resume() */ diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index c92add8c20d9c..dc47ce23f2e20 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -1337,11 +1337,13 @@ void apple_isp_firmware_shutdown(struct apple_isp *isp) isp_free_released_surfaces(isp); /* - * Firmware that starts again goes down with the stream, as macOS's camera close - * takes it, and boots again for the next one. A system sleep then finds it off: - * stopped on the way into a sleep that keeps DRAM up, the SoC doesn't wake. + * Firmware that starts again goes down after the stream, at once or, with + * hw->fw_idle_ms, once it has idled that long, as macOS's camera service + * powers it off. A system sleep finds it off either way: idle firmware is + * stopped before the sleep begins (apple_isp_pm_notify()). The DRAM-kept + * wake runs the ISP itself and needs it left that way. */ - if (isp->hw->fw_restart) + if (isp->hw->fw_restart && !isp->hw->fw_idle_ms) apple_isp_firmware_halt(isp); /* Not on isp->wq: the stop flushes that. */ From 24d7b2e73947303f6162530263ccf4119c845432 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 19:56:29 -0500 Subject: [PATCH 28/29] soc: apple: sep: rate-limit the sleep messages, and fix host tests for the merge The SEP's request-path source test from integration (#1590) wants rate-limited or debug messages throughout sep.rs. Five in this series' sleep path were plain: use dev_warn_ratelimited!, dev_err_ratelimited! and dev_info_ratelimited! for them. No logic changes. The last integration merge renamed the DART system-sleep callbacks that tools/asahi/j613-mtp-s2idle-tests.py extracts. Name them apple_dart_system_suspend() and apple_dart_system_resume() again, with the J613 live-MTP code inline as integration has it and this series' firmware-sleep phase choice for every other DART. Give the test stubs for pm_suspend_via_firmware() and the rtkit-helper AP notices, which a firmware-driven sleep alone reaches. The ISP host harness (#1560) didn't model this series' ISP state. Extract isp_cmd_send_chan() with isp_cmd_send(), and take the latch guard out of both for the overwrite control. Give the session fixture fw_restart, fw.started and isp_check_restarted(), and run its capture stop with the firmware running, as a capture only stops while it streams. Give the buffer fixture ISP_FW_RUNNING and clear_bit(). Every check and control stays as it was. Signed-off-by: Diljit Singh --- drivers/iommu/apple-dart.c | 32 ++++++++-------------------- drivers/soc/apple/sep.rs | 10 ++++----- tools/asahi/isp-tests.py | 9 +++++++- tools/asahi/isp/buffers.c | 7 ++++++ tools/asahi/isp/session.c | 9 ++++++++ tools/asahi/j613-mtp-s2idle-tests.py | 5 +++++ 6 files changed, 43 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index fcfae890ab464..2c4350a0f29fe 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -6408,13 +6408,15 @@ EXPORT_SYMBOL_GPL(apple_dart_state_restore); * before any early resume: coprocessors such as the PMP and SIO restart their firmware in * their early resume, and its first DMA has to find the DART already restored. Any other * sleep keeps the normal phase, which the PCIe-C tunnel DARTs need: their host stops and - * restarts the port clock in its own noirq callbacks. + * restarts the port clock in its own noirq callbacks. The adopted J613 MTP stays live: + * system s2idle must retain, never reset it. */ -/* The adopted MTP stays live: system s2idle must retain, never reset it. */ -static int apple_dart_live_mtp_suspend(struct device *dev) +static int apple_dart_system_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); + if (!dart->live_mtp) + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_suspend(dev); if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; if (apple_dart_readl(dart, DART_TCR(dart, 1)) != dart->hw->tcr_bypass) @@ -6426,10 +6428,12 @@ static int apple_dart_live_mtp_suspend(struct device *dev) return 0; } -static int apple_dart_live_mtp_resume(struct device *dev) +static int apple_dart_system_resume(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); + if (!dart->live_mtp) + return pm_suspend_via_firmware() ? 0 : pm_runtime_force_resume(dev); if (!dart->live_mtp_sleep) return 0; dart->live_mtp_sleep = false; @@ -6441,24 +6445,6 @@ static int apple_dart_live_mtp_resume(struct device *dev) return 0; } -static __maybe_unused int apple_dart_sys_suspend(struct device *dev) -{ - struct apple_dart *dart = dev_get_drvdata(dev); - - if (dart->live_mtp) - return apple_dart_live_mtp_suspend(dev); - return pm_suspend_via_firmware() ? 0 : pm_runtime_force_suspend(dev); -} - -static __maybe_unused int apple_dart_sys_resume(struct device *dev) -{ - struct apple_dart *dart = dev_get_drvdata(dev); - - if (dart->live_mtp) - return apple_dart_live_mtp_resume(dev); - return pm_suspend_via_firmware() ? 0 : pm_runtime_force_resume(dev); -} - static __maybe_unused int apple_dart_sys_suspend_noirq(struct device *dev) { return pm_suspend_via_firmware() ? pm_runtime_force_suspend(dev) : 0; @@ -6470,7 +6456,7 @@ static __maybe_unused int apple_dart_sys_resume_noirq(struct device *dev) } static const struct dev_pm_ops apple_dart_pm_ops = { - SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend, apple_dart_sys_resume) + SYSTEM_SLEEP_PM_OPS(apple_dart_system_suspend, apple_dart_system_resume) NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend_noirq, apple_dart_sys_resume_noirq) RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) }; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 9e0b5794c0634..9987fcd18dbab 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1466,7 +1466,7 @@ impl SepData { } // Do not commit a running enclave whose sleep protocol is not ready. if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { - dev_warn!( + dev_warn_ratelimited!( self.dev, "sleep refused: the enclave did not finish coming up\n" ); @@ -1481,7 +1481,7 @@ impl SepData { |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), ); if !idle { - dev_err!( + dev_err_ratelimited!( self.dev, "sleep refused: a request is still waiting for its reply\n" ); @@ -1490,7 +1490,7 @@ impl SepData { } if let Err(e) = self.sleep_leave_for_commit() { - dev_err!( + dev_err_ratelimited!( self.dev, "sleep refused: irreversible request could not be staged ({:?})\n", e @@ -1569,10 +1569,10 @@ impl SepData { Ok(()) => { self.set_sleep_gate(pm::SleepGate::Open); // One line per sleep it was told of: nothing else says it came back. - dev_info!(self.dev, "the enclave is back from its sleep request\n"); + dev_info_ratelimited!(self.dev, "the enclave is back from its sleep request\n"); } Err(e) => { - dev_err!( + dev_err_ratelimited!( self.dev, "the enclave did not come back ({:?}); it stays unused until a reboot\n", e diff --git a/tools/asahi/isp-tests.py b/tools/asahi/isp-tests.py index d31176c125fab..e649d3d83cbd6 100644 --- a/tools/asahi/isp-tests.py +++ b/tools/asahi/isp-tests.py @@ -50,6 +50,9 @@ def main(): 'apple_isp_firmware_halt', 'isp_firmware_boot', 'apple_isp_firmware_boot', 'apple_isp_firmware_shutdown'], } + # #1445 sends through a per-channel helper; the latch guards it too. + if 'static int isp_cmd_send_chan(' in inputs['isp-cmd.c']: + groups['isp-cmd.c'] = ['isp_cmd_send_chan'] + groups['isp-cmd.c'] if args.suite == 'session': groups['isp-ipc.c'] += ['chan_rx_done', 'ipc_chan_handle'] groups['isp-fw.c'] += ['apple_isp_isr_thread'] @@ -66,7 +69,11 @@ def main(): guard = 'if (READ_ONCE(isp->ipc_error))\n\t\treturn READ_ONCE(isp->ipc_error);' if guard not in bodies['isp_cmd_send']: raise RuntimeError('overwrite negative control no longer matches its guard') - bodies['isp_cmd_send'] = bodies['isp_cmd_send'].replace(guard, '') + for name in ('isp_cmd_send', 'isp_cmd_send_chan'): + if name in bodies: + if guard not in bodies[name]: + raise RuntimeError('overwrite negative control no longer matches ' + name) + bodies[name] = bodies[name].replace(guard, '') elif args.control == 'stop': bodies['apple_isp_stop_capture'] = bodies['apple_isp_stop_capture'].replace( 'if (!isp_cmd_ch_stop(isp, 0)) /* TODO channel mask */', diff --git a/tools/asahi/isp/buffers.c b/tools/asahi/isp/buffers.c index d78873ffe27dc..d52cf0f7e47d8 100644 --- a/tools/asahi/isp/buffers.c +++ b/tools/asahi/isp/buffers.c @@ -63,6 +63,11 @@ static void set_bit(unsigned int bit, unsigned long *state) *state |= 1UL << bit; } +static void clear_bit(unsigned int bit, unsigned long *state) +{ + *state &= ~(1UL << bit); +} + #define dev_err(...) ((void)0) #define dev_err_ratelimited(...) ((void)0) #define dev_warn_ratelimited(...) ((void)0) @@ -301,6 +306,8 @@ static void apple_isp_stop_capture(struct apple_isp *isp); /* These fixtures exercise the non-P010/J613 path. P010 has kernel KUnit coverage. */ #define ISP_FW_DEAD 3 +/* The zero-initialized fixtures run the firmware (#1445's stop checks for it). */ +#define ISP_FW_RUNNING 0 #define __set_bit set_bit static bool isp_is_p010(struct apple_isp *isp) { return false; } static bool isp_supports_p010(struct apple_isp *isp) { return false; } diff --git a/tools/asahi/isp/session.c b/tools/asahi/isp/session.c index 08d24f9f124b7..0b9d563b5bfbe 100644 --- a/tools/asahi/isp/session.c +++ b/tools/asahi/isp/session.c @@ -55,6 +55,8 @@ struct isp_channel { struct hardware { bool resident_fw; + /* #1445: resident firmware that starts again from its own memory */ + bool fw_restart; unsigned int fw_idle_ms; }; @@ -70,6 +72,7 @@ struct apple_isp { struct isp_channel *ipc_stale; bool ipc_failed; const struct hardware *hw; + struct { bool started; } fw; void *dev, *wq, *cmd_virt; u64 cmd_iova; unsigned int current_ch; @@ -322,6 +325,8 @@ static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, sizeof(v)); static bool isp_is_p010(struct apple_isp *isp) { return false; } static int isp_ipc_resync(struct apple_isp *isp) { return 0; } static void isp_restore_fw_memory(struct apple_isp *isp) { } +/* #1445's post-restart query; these fixtures never restart (fw.started is clear). */ +static int isp_check_restarted(struct apple_isp *isp) { CHECK(false); return 0; } /* DRIVER FUNCTIONS */ int main(void) { @@ -355,6 +360,9 @@ int main(void) isp.ipc_error = -EIO; CHECK(ipc_chan_send(&isp, &ch, 1) == -EIO); CHECK(writes == 1 && cursors == 0); + /* A capture stops while it streams, with the firmware running. */ + int fw_state = isp.fw_state; + isp.fw_state = ISP_FW_RUNNING; for (unsigned int fail = 0; fail < 2; fail++) { stop_error = fail ? -ETIME : 0; stops = 0; @@ -363,6 +371,7 @@ int main(void) apple_isp_stop_capture(&isp); CHECK(stops == 1 && returns == !fail && watchdog_stops == 1); } + isp.fw_state = fw_state; for (fail_stage = 1; fail_stage <= 5; fail_stage++) { isp.ipc_error = -ETIME; stage_error = -ENOMEM; diff --git a/tools/asahi/j613-mtp-s2idle-tests.py b/tools/asahi/j613-mtp-s2idle-tests.py index 7aa413645303a..21d673862dac6 100644 --- a/tools/asahi/j613-mtp-s2idle-tests.py +++ b/tools/asahi/j613-mtp-s2idle-tests.py @@ -32,6 +32,11 @@ def fn(s,n): static int pm_runtime_force_resume(struct device *d){forced_resume++;return 8;} static bool of_machine_is_compatible(const char *s){return true;} static unsigned readl(void *p){return *(unsigned *)p;} +/* #1445: only a firmware-driven (T8140 S2R) sleep takes the noirq or AP-idle paths. */ +static bool pm_suspend_via_firmware(void){return false;} +static int apple_rtkit_helper_ap_idle(struct apple_rtkit_helper *h){assert(!"firmware-driven path in s2idle");return 0;} +/* Its AP-on notice is a no-op without an earlier idle notice. */ +static void apple_rtkit_helper_ap_on(struct apple_rtkit_helper *h){assert(h);} ''' body=fn(s,'apple_dart_system_suspend')+fn(s,'apple_dart_system_resume')+fn(h,'apple_rtkit_helper_suspend')+fn(h,'apple_rtkit_helper_resume') for forbidden in ['writel','hw_reset','invalidate_tlb','apple_rtkit_wake','apple_rtkit_boot','apple_rtkit_quiesce']: From f1f89c74e7aa4e234d115165d20efeafb276f60a Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Wed, 11 Nov 2026 20:32:55 -0500 Subject: [PATCH 29/29] tools: asahi: sep-suspend-errors: stub the rate-limited log macros The SEP sleep path now logs through dev_err_ratelimited!, dev_warn_ratelimited! and dev_info_ratelimited!, as integration's request-path rule asks, and the harness's Rust prefix had no-op stubs only for the plain macros, so rustc stopped on the extracted code and the test and its 21 mutations all failed to build. Add no-op stubs for the three, beside the plain ones, which other extracted code may still use. Signed-off-by: Diljit Singh --- tools/asahi/sep-suspend-errors-tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/asahi/sep-suspend-errors-tests.py b/tools/asahi/sep-suspend-errors-tests.py index 6c36f56642b63..2ae69ec2c32fc 100644 --- a/tools/asahi/sep-suspend-errors-tests.py +++ b/tools/asahi/sep-suspend-errors-tests.py @@ -458,6 +458,9 @@ def replace_once(source, before, after): macro_rules! dev_warn { ($($a:tt)*) => {{}}; } macro_rules! dev_dbg { ($($a:tt)*) => {{}}; } macro_rules! dev_info { ($($a:tt)*) => {{}}; } +macro_rules! dev_err_ratelimited { ($($a:tt)*) => {{}}; } +macro_rules! dev_warn_ratelimited { ($($a:tt)*) => {{}}; } +macro_rules! dev_info_ratelimited { ($($a:tt)*) => {{}}; } extern "C" { fn fixture(); fn parameter(which: i32, value: i32); fn counter(which: i32) -> i32; fn device_pointer() -> *mut c_void; fn mbox_pointer() -> *mut c_void;