diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index c3a2f54af0387..ce8c4daad85f2 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -119,7 +119,10 @@ properties: - description: Index into the provider's reg property. - description: Index in the firmware display register table. - description: Byte offset of the scratch register within the resource. - description: PMGR bandwidth scratch resource, display index and offset. + description: | + PMGR bandwidth scratch resource, display index and offset. On the T8132 + external DCPs it is the pipe's 8-byte realtime bandwidth request slot, + which lies inside the display register window the index names (disp-4). apple,bw-doorbell: $ref: /schemas/types.yaml#/definitions/phandle-array diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 350e077e779b4..8fb92757fe7e5 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -65,7 +65,18 @@ properties: error-mask policy rather than changing it for these streams. reg: - maxItems: 1 + minItems: 1 + items: + - description: DART registers + - description: + Address filter (DAPF) registers, which restrict the physical + addresses the DART lets its masters reach + + reg-names: + minItems: 1 + items: + - const: dart + - const: dapf interrupts: maxItems: 1 @@ -157,6 +168,17 @@ required: - '#iommu-cells' - interrupts +allOf: + - if: + properties: + reg: + minItems: 2 + required: + - reg + then: + required: + - reg-names + additionalProperties: false examples: @@ -189,3 +211,12 @@ examples: master2 { iommus = <&dart2a 0>, <&dart2b 1>; }; + + - |+ + dart3: iommu@8e8c0000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x8e8c0000 0xc000>, <0x8e8d0000 0x4000>; + reg-names = "dart", "dapf"; + interrupts = <1 387 4>; + #iommu-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/media/apple,t8132-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8132-isp.yaml new file mode 100644 index 0000000000000..3861d13a0bde9 --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,t8132-isp.yaml @@ -0,0 +1,201 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,t8132-isp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8132 Image Signal Processor + +maintainers: + - Erik Bourget + +description: + The T8132 ISP controls the attached camera sensor and processes its frames + through firmware the bootloader loads. The firmware segments are described + as reserved memory mapped at the addresses the firmware runs them at. The + host exchanges commands and image buffers through three DART instances; the + first one also has the address filter registers. + +select: + properties: + compatible: + contains: + const: apple,t8132-isp + required: + - compatible + +properties: + compatible: + items: + - const: apple,t8132-isp + - const: apple,isp + + reg: + items: + - description: Coprocessor registers up to its control block + - description: Mailbox interrupt registers + - description: Firmware handshake registers + - description: Mailbox doorbell registers + + reg-names: + items: + - const: coproc + - const: mbox + - const: gpio + - const: mbox2 + + interrupts: + maxItems: 1 + + iommus: + items: + - description: First ISP DART stream; this DART has the address filter bank + - description: Second ISP DART stream + - description: Third ISP DART stream + + power-domains: + items: + - description: ISP system domain + - description: ISP CPU domain + - description: ISP CPU core 0 domain + - description: ISP CPU core 1 domain + - description: ISP FE domain + - description: ISP DPRX domain + - description: ISP secure domain + - description: ISP STS0 domain + - description: ISP STS1 domain + - description: ISP Pearl domain + - description: ISP BE domain + - description: ISP CLR domain + - description: ISP CVD domain + - description: ISP CVM domain + - description: First camera sensor domain + - description: Second camera sensor domain + + memory-region: + items: + - description: Firmware text loaded by the bootloader + - description: Firmware data loaded by the bootloader + - description: Firmware main memory window; host allocations follow it + + memory-region-names: + items: + - const: text + - const: data + - const: heap + + apple,dart-vm-size: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Size in bytes of the host allocation aperture used by the ISP driver. + + apple,platform-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Platform identifier passed in the firmware boot descriptor. + + apple,cam-connections-scheme: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Camera connection layout of the board passed in the firmware boot + descriptor. + + apple,temporal-filter: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Firmware temporal-filter strength selected for the attached sensor. + + sensor-presets: + type: object + description: + Sensor firmware presets and the host output geometry associated with them. + patternProperties: + "^preset[0-9]+$": + type: object + properties: + apple,config-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Sensor configuration index understood by the firmware + apple,input-size: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Input width in pixels + - description: Input height in pixels + description: Sensor input dimensions + apple,output-size: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Output width in pixels + - description: Output height in pixels + description: Delivered frame dimensions + apple,crop: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Horizontal crop origin in pixels + - description: Vertical crop origin in pixels + - description: Crop width in pixels + - description: Crop height in pixels + description: Cropped sensor rectangle + required: + - apple,config-index + - apple,input-size + - apple,output-size + - apple,crop + additionalProperties: false + additionalProperties: false + +required: + - compatible + - reg + - reg-names + - interrupts + - iommus + - power-domains + - memory-region + - memory-region-names + - apple,dart-vm-size + - apple,platform-id + - sensor-presets + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@48c000000 { + compatible = "apple,t8132-isp", "apple,isp"; + reg = <0x4 0x8c000000 0x0 0x1604000>, + <0x4 0x8e924000 0x0 0x40>, + <0x4 0x8e9241d0 0x0 0x20>, + <0x4 0x8e924464 0x0 0x10>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, + <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, + <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, + <&ps_isp_cvm>, <&ps_ispsens0>, <&ps_ispsens1>; + memory-region = <&isp_fw_text>, <&isp_fw_data>, <&isp_heap>; + memory-region-names = "text", "data", "heap"; + apple,dart-vm-size = /bits/ 64 <0xdec00000>; + apple,platform-id = <0x17>; + + sensor-presets { + preset0 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml index 2eecc48c9b061..1a1051584e8fa 100644 --- a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +++ b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml @@ -20,6 +20,7 @@ properties: - enum: - apple,abbey-pmic - apple,baku-pmic + - apple,chapel-pmic - apple,maverick-pmic - apple,sera-pmic - apple,stowe-pmic diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 4c1d9aefc2519..a3728ed3effee 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -35,8 +35,10 @@ properties: description: | The perf-state floor words of the voter agents ("voters"), and the per-agent words holding the floor the PMC honours for each agent with - the per-rail interface enable words ("rails"). Optionally, the cluster - enable control word followed by its status word ("clusters"). + the per-rail interface enable words ("rails"); on T8132 the rails + block also holds the bandwidth request and boost interface enable + words. Optionally, the cluster enable control word followed by its + status word ("clusters"). reg-names: description: | @@ -178,6 +180,43 @@ patternProperties: - '#power-domain-cells' - apple,cluster-bit + '^interface-[a-z0-9-]+$': + type: object + description: | + A display pipe's interface enables, provided as a power domain. The + bandwidth request (BWR) and boost interfaces each have an enable word + with one bit per agent, and the PMC ignores the pipe's bandwidth and + boost votes while its bits are clear. Both bits are set after the + parent domain powers on, and cleared before it powers off. + additionalProperties: false + + properties: + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + description: Domain that must be powered while the enables are set. + + apple,bwr-bit: + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 31 + description: Bit of the pipe in the BWR interface enable word. + + apple,boost-bit: + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 31 + description: Bit of the pipe in the boost interface enable word. + + required: + - '#power-domain-cells' + - power-domains + - apple,bwr-bit + - apple,boost-bit + '^performance-controller@[0-9a-f]+$': type: object description: The host performance controller's voter agent. @@ -207,6 +246,17 @@ required: - reg - reg-names +allOf: + - if: + properties: + compatible: + not: + contains: + const: apple,t8132-pmc + then: + patternProperties: + '^interface-[a-z0-9-]+$': false + additionalProperties: false examples: @@ -245,3 +295,27 @@ examples: }; }; }; + + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-management@382800000 { + compatible = "apple,t8132-pmc"; + reg = <0x3 0x82800000 0x0 0x10000>, + <0x3 0x828aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + apple,static-voters = <3>; + + interface-dispext0 { + label = "dispext0-interface"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_sys>; + apple,bwr-bit = <3>; + apple,boost-bit = <1>; + }; + }; + }; diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst index eace444abbcd8..8e6dd6bc3a9b6 100644 --- a/Documentation/rust/bindgen-layout-checks.rst +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -19,6 +19,13 @@ pointer as the current mm and Asahi's RSS charge faulted in counter layout; the pointer supplied to them was wrong. Enabling SCHEDSTATS only masks the generator defect and is not a requirement of the driver. +``rust/bindgen_parameters`` declares ``sched_statistics`` opaque. bindgen takes +an opaque type's size and alignment from Clang's layout, so stock bindgen also +emits ``#[repr(align(64))]`` for it and ``task_struct`` agrees with C. Rust code +does not access its fields. Other aligned empty records still need the +corrected generator below, and the layout assertions still check every +emitted type. + Known generator limitations --------------------------- diff --git a/MAINTAINERS b/MAINTAINERS index e0a0070243ae6..4ed4b3dc08ea8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2097,6 +2097,7 @@ F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +F: Documentation/devicetree/bindings/media/apple,t8132-isp.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml diff --git a/arch/arm64/boot/dts/apple/t6041-j616c.dts b/arch/arm64/boot/dts/apple/t6041-j616c.dts index 96cb5281c8b59..423e8b28c7bfe 100644 --- a/arch/arm64/boot/dts/apple/t6041-j616c.dts +++ b/arch/arm64/boot/dts/apple/t6041-j616c.dts @@ -158,6 +158,10 @@ status = "okay"; }; +&mcc { + status = "okay"; +}; + &smc_hwmon { temperature-TCMz { apple,key-id = "TCMz"; diff --git a/arch/arm64/boot/dts/apple/t6041.dtsi b/arch/arm64/boot/dts/apple/t6041.dtsi index 62c2f817f3096..094a7c4bd8fa6 100644 --- a/arch/arm64/boot/dts/apple/t6041.dtsi +++ b/arch/arm64/boot/dts/apple/t6041.dtsi @@ -207,11 +207,56 @@ /delete-property/ power-domain-names; }; +/* + * The main PMIC is a Chapel at 0xb (ADT nub-spmi0 spmi-chapelL1-b), not the + * J616s's Abbey at 0xe. Its nvmem cells and RTC alarm registers are at the + * Abbey offsets (ADT info-rtc_alarm_*). + */ &nub_spmi0 { - /delete-property/ interrupt-controller; - /delete-property/ #interrupt-cells; - /delete-property/ interrupt-parent; - /delete-property/ interrupts; + /delete-node/ pmic@e; + + pmic@b { + compatible = "apple,chapel-pmic", "apple,spmi-nvmem"; + reg = <0xb SPMI_USID>; + + pmu_rtc: rtc { + compatible = "apple,t6041-pmu-rtc"; + interrupt-parent = <&nub_spmi0>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + rtc_offset: rtc-offset@2100 { + reg = <0x2100 0x6>; + }; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; }; &usb_drd0_dart_0 { @@ -588,3 +633,34 @@ apple,always-on; apple,no-auto-pm; }; + +/* + * Four cache planes per AMCC, eight AMCCs on the full part and six on the + * binned one (ADT amcc-enable-mask 0x3f). List the six every T6041 has. + */ +&mcc { + reg = <0x2 0x20000000 0x0 0x4000>, + <0x2 0x20040000 0x0 0x4000>, + <0x2 0x20080000 0x0 0x4000>, + <0x2 0x200c0000 0x0 0x4000>, + <0x2 0x22000000 0x0 0x4000>, + <0x2 0x22040000 0x0 0x4000>, + <0x2 0x22080000 0x0 0x4000>, + <0x2 0x220c0000 0x0 0x4000>, + <0x2 0x24000000 0x0 0x4000>, + <0x2 0x24040000 0x0 0x4000>, + <0x2 0x24080000 0x0 0x4000>, + <0x2 0x240c0000 0x0 0x4000>, + <0x2 0x26000000 0x0 0x4000>, + <0x2 0x26040000 0x0 0x4000>, + <0x2 0x26080000 0x0 0x4000>, + <0x2 0x260c0000 0x0 0x4000>, + <0x2 0x28000000 0x0 0x4000>, + <0x2 0x28040000 0x0 0x4000>, + <0x2 0x28080000 0x0 0x4000>, + <0x2 0x280c0000 0x0 0x4000>, + <0x2 0x2a000000 0x0 0x4000>, + <0x2 0x2a040000 0x0 0x4000>, + <0x2 0x2a080000 0x0 0x4000>, + <0x2 0x2a0c0000 0x0 0x4000>; +}; diff --git a/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi b/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi index ba1f428b4ee04..7f12ea2491967 100644 --- a/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi +++ b/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi @@ -4483,12 +4483,12 @@ apple,output-size = <1920 1080>; apple,crop = <0 0 1920 1080>; }; - /* 1280x720 */ + /* 1280x720: the same window as 1920x1080, scaled down */ preset1 { apple,config-index = <5>; apple,input-size = <1920 2160>; apple,output-size = <1280 720>; - apple,crop = <0 0 1280 720>; + apple,crop = <0 0 1920 1080>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j713-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8132-j713-dcpext.dtsi index 541cf598f29b7..49d2983481b04 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-j713-dcpext.dtsi @@ -5,7 +5,8 @@ * T8132 has two external display pipes, dispext0 and dispext1, each with * its own DCP, mailbox, DCP DART and display DART. They mirror the * internal display (t8132-j713-display.dtsi) at other bases and share its - * firmware clock window (disp-4). + * firmware clock window (disp-4), which also holds their realtime bandwidth + * request slots in the PMC. * * iBoot starts both and leaves them quiesced, as it does the internal one. * When the boot loader hands them over (opt-in, and only after the internal @@ -81,6 +82,7 @@ power-domains = <&ps_dispext0_cpu>; dma-noncoherent; apple,dcp-index = <1>; + apple,bw-scratch = <&pmgr_dcpext 0 4 0>; #address-cells = <2>; #size-cells = <2>; @@ -91,6 +93,17 @@ }; }; + /* + * The external pipes' realtime bandwidth request slots in the PMC, + * 0x2c000 and 0x30000 into the window the dcpexts have as disp-4. + */ + pmgr_dcpext: power-management@38282c000 { + reg = <0x3 0x8282c000 0x0 0x8>, + <0x3 0x82830000 0x0 0x8>; + reg-names = "dcpext0-bw-scratch", "dcpext1-bw-scratch"; + #apple,bw-scratch-cells = <3>; + }; + dcpext1_mbox: mailbox@48ae08000 { compatible = "apple,t8132-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x4 0x8ae08000 0x0 0x4000>; @@ -142,6 +155,7 @@ power-domains = <&ps_dispext1_cpu>; dma-noncoherent; apple,dcp-index = <2>; + apple,bw-scratch = <&pmgr_dcpext 1 4 0>; #address-cells = <2>; #size-cells = <2>; diff --git a/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi index d5d29a55e8fc5..66b69f0f6320c 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi @@ -63,6 +63,14 @@ status = "disabled"; }; +&ps_ispsens0 { + status = "okay"; +}; + +&ps_ispsens1 { + status = "okay"; +}; + &ps_pmgr_soc_ocla { status = "disabled"; }; @@ -385,7 +393,7 @@ apple,firmware-owned; }; -/* No camera driver yet; genpd powers the ISP down as an unused domain. */ +/* The camera powers the ISP from its first stream until it is unbound. */ &ps_isp_sys { status = "okay"; }; @@ -430,6 +438,58 @@ status = "disabled"; }; +&ps_isp_cpu { + status = "okay"; +}; + +&ps_isp_cpu_core0 { + status = "okay"; +}; + +&ps_isp_cpu_core1 { + status = "okay"; +}; + +&ps_isp_fe { + status = "okay"; +}; + +&ps_isp_dprx { + status = "okay"; +}; + +&ps_isp_secure { + status = "okay"; +}; + +&ps_isp_sts0 { + status = "okay"; +}; + +&ps_isp_sts1 { + status = "okay"; +}; + +&ps_isp_pearl { + status = "okay"; +}; + +&ps_isp_be { + status = "okay"; +}; + +&ps_isp_clr { + status = "okay"; +}; + +&ps_isp_cvd { + status = "okay"; +}; + +&ps_isp_cvm { + status = "okay"; +}; + &ps_venc_dma { status = "disabled"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index 33a9fa372da65..eda4f52c23f93 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -127,6 +127,25 @@ }; }; + reserved-memory { + /* + * iBoot loads the ISP firmware outside the kernel's memory, at + * addresses that move with the boot firmware. The boot loader + * fills these placeholders from the ADT's isp segment-ranges: reg, + * and iommu-addresses where the firmware runs them, from + * 0x100_0000_0000 on. The driver checks that the coprocessor's + * reset vector is this text before it starts the firmware. + */ + isp_fw_text: isp-firmware-text { }; + + isp_fw_data: isp-firmware-data { }; + + /* the firmware's 20 MiB main memory, the driver allocates above it */ + isp_heap: isp-heap { + iommu-addresses = <&isp 0x100 0x0 0x0 0x1400000>; + }; + }; + /* * The boot NOR runs from a PMU rail that is on whenever the SoC is * and that no OS switches. The ADT gives the flash no supply. @@ -138,6 +157,34 @@ }; }; +/* + * The boot loader enables the ISP and its DARTs once it has filled the + * firmware placeholders and set apple,platform-id, the ADT isp node's + * sensor-type (the J713 ADT has no cam-connections-scheme). + */ +&isp { + memory-region = <&isp_fw_text>, <&isp_fw_data>, <&isp_heap>; + memory-region-names = "text", "data", "heap"; + + /* The front camera, an IMX958: configuration 5, as on the J616s. */ + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1280x720 */ + preset1 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1280 720>; + }; + }; +}; + /* * m1n1 programs the AOP DART's DAPF and reserves the firmware segments and * the DAPF windows in memory-region when /chosen asks for the AOP's handoff, @@ -541,6 +588,18 @@ apple,static-voters = <3>; }; +/* + * The PMC enables of an external pipe are set once the pipe's SYS state is + * on, and before its FE state powers on. + */ +&ps_dispext0_fe { + power-domains = <&ps_dispext0_sys>, <&pmc_dispext0>; +}; + +&ps_dispext1_fe { + power-domains = <&ps_dispext1_sys>, <&pmc_dispext1>; +}; + &smc { status = "okay"; diff --git a/arch/arm64/boot/dts/apple/t8132-memory.dtsi b/arch/arm64/boot/dts/apple/t8132-memory.dtsi index fea5b3a97dbf5..c6cafa8219d2a 100644 --- a/arch/arm64/boot/dts/apple/t8132-memory.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-memory.dtsi @@ -19,6 +19,27 @@ apple,pmp = <&pmp>; status = "disabled"; + /* + * The external display pipes' bandwidth request (BWR) and + * boost interface enables. The PMC ignores a pipe's bandwidth + * and display clock votes while they are clear. + */ + pmc_dispext0: interface-dispext0 { + label = "dispext0-interface"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_sys>; + apple,bwr-bit = <3>; + apple,boost-bit = <1>; + }; + + pmc_dispext1: interface-dispext1 { + label = "dispext1-interface"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_sys>; + apple,bwr-bit = <4>; + apple,boost-bit = <2>; + }; + clpc: performance-controller@3 { compatible = "apple,t8132-clpc"; reg = <3>; diff --git a/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi index 5e31d6ce69e15..0dafc59196507 100644 --- a/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi @@ -120,6 +120,24 @@ label = "pmgr_soc_ocla"; }; + ps_ispsens0: power-controller@188 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + status = "disabled"; + }; + + ps_ispsens1: power-controller@190 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + status = "disabled"; + }; + ps_ap_tmm: power-controller@1b8 { compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x1b8 4>; @@ -885,6 +903,137 @@ apple,always-on; }; + /* The ISP's own power states, children of ISP_SYS */ + ps_isp_cpu: power-controller@4000 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu"; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + ps_isp_cpu_core0: power-controller@4008 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu_core0"; + power-domains = <&ps_isp_cpu>; + status = "disabled"; + }; + + ps_isp_cpu_core1: power-controller@4010 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu_core1"; + power-domains = <&ps_isp_cpu>; + status = "disabled"; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + ps_isp_dprx: power-controller@4020 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_dprx"; + power-domains = <&ps_isp_fe>; + status = "disabled"; + }; + + ps_isp_secure: power-controller@4028 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_secure"; + power-domains = <&ps_isp_fe>; + status = "disabled"; + }; + + ps_isp_sts0: power-controller@4030 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sts0"; + power-domains = <&ps_isp_fe>; + status = "disabled"; + }; + + ps_isp_sts1: power-controller@4038 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sts1"; + power-domains = <&ps_isp_fe>; + status = "disabled"; + }; + + ps_isp_pearl: power-controller@4040 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_pearl"; + power-domains = <&ps_isp_fe>; + status = "disabled"; + }; + + ps_isp_be: power-controller@4048 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_be"; + power-domains = <&ps_isp_fe>; + status = "disabled"; + }; + + ps_isp_clr: power-controller@4050 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_clr"; + power-domains = <&ps_isp_be>; + status = "disabled"; + }; + + ps_isp_cvd: power-controller@4058 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cvd"; + power-domains = <&ps_isp_be>; + status = "disabled"; + }; + + ps_isp_cvm: power-controller@4060 { + compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cvm"; + power-domains = <&ps_isp_cvd>; + status = "disabled"; + }; + ps_venc_dma: power-controller@8000 { compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x8000 4>; diff --git a/arch/arm64/boot/dts/apple/t8132.dtsi b/arch/arm64/boot/dts/apple/t8132.dtsi index 7a26e67f37a41..6ee608983784e 100644 --- a/arch/arm64/boot/dts/apple/t8132.dtsi +++ b/arch/arm64/boot/dts/apple/t8132.dtsi @@ -1204,6 +1204,67 @@ }; + /* + * ADT isp and dart-isp: three DART instances on one interrupt; streams 1, + * 2, 3, 6, 7, 10 and 11 are remapped to stream 0. The first instance also + * has the address filter registers. + */ + isp: isp@48c000000 { + compatible = "apple,t8132-isp", "apple,isp"; + reg = <0x4 0x8c000000 0x0 0x1604000>, + <0x4 0x8e924000 0x0 0x40>, + <0x4 0x8e9241d0 0x0 0x20>, + <0x4 0x8e924464 0x0 0x10>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, + <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, + <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, + <&ps_isp_cvm>, <&ps_ispsens0>, <&ps_ispsens1>; + /* the DART's 0xe0000000 window above the firmware's 20 MiB */ + apple,dart-vm-size = /bits/ 64 <0xdec00000>; + status = "disabled"; + }; + + isp_dart0: iommu@48e8c0000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x8e8c0000 0x0 0xc000>, + <0x4 0x8e8d0000 0x0 0x4000>; + reg-names = "dart", "dapf"; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, <10 0>, <11 0>; + status = "disabled"; + }; + + isp_dart1: iommu@48e8e0000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x8e8e0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, <10 0>, <11 0>; + status = "disabled"; + }; + + isp_dart2: iommu@48e900000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x8e900000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, <10 0>, <11 0>; + status = "disabled"; + }; + dart_apcie0: iommu@490000000 { status = "disabled"; compatible = "apple,t8132-dart", "apple,t8110-dart"; diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index f6514a09df43b..08df0ae86c366 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -458,7 +458,7 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, * release order frees irq before ioremap (probe-order reverse); * DMA surfaces are wedge-held, never freed. */ - dev_warn(ane->dev, + dev_info(ane->dev, "boot: module pinned while firmware CPU may run\n"); if (r == -ENODATA) diff --git a/drivers/gpu/drm/apple/dcp-m4-callback.h b/drivers/gpu/drm/apple/dcp-m4-callback.h index 2cf3f73790ade..7a6fb73856e1a 100644 --- a/drivers/gpu/drm/apple/dcp-m4-callback.h +++ b/drivers/gpu/drm/apple/dcp-m4-callback.h @@ -21,6 +21,11 @@ struct dcp_m4_callback_ops { int (*memory)(void *cookie, unsigned int method, const void *input, void *output); /* Idle caching: D117 state, D578/D579 fence, D580 cache-surface release. */ int (*idle)(void *cookie, unsigned int method, const void *input, void *output); + /* + * D003 rt_bandwidth_setup_ap: its reply, @output zeroed (0x3c bytes). + * Absent: the optional service is declined (status 1). + */ + int (*bandwidth)(void *cookie, void *output); }; struct dcp_m4_callback_shape { @@ -129,6 +134,8 @@ static inline int dcp_m4_callback_dispatch(const struct dcp_m4_callback_ops *ops put_unaligned_le32(1, out); return 0; case 3: + if (ops->bandwidth) + return dcp_m4_callback_result(ops->bandwidth(cookie, out)); put_unaligned_le32(1, out + 56); /* Decline optional bandwidth service. */ return 0; case 6: diff --git a/drivers/gpu/drm/apple/dcp-m4-frame.h b/drivers/gpu/drm/apple/dcp-m4-frame.h index 0bf00b68d6aec..665f271e3c437 100644 --- a/drivers/gpu/drm/apple/dcp-m4-frame.h +++ b/drivers/gpu/drm/apple/dcp-m4-frame.h @@ -404,4 +404,21 @@ static inline int dcp_m4_swap_take_completion(struct dcp_m4_swap_state *swap) return 1; } +/* + * Hand a replied swap over to @to, which tracks it from then on (its D590 may + * be in already): @swap counts as taken, so that the next swap may begin. + */ +static inline int dcp_m4_swap_hand_off(struct dcp_m4_swap_state *swap, + struct dcp_m4_swap_state *to) +{ + if (swap->error) + return swap->error; + if (!swap->id || !swap->replied || swap->emitted) + return dcp_m4_swap_fail(swap, -EPROTO); + *to = *swap; + swap->completed = true; + swap->emitted = true; + return 0; +} + #endif diff --git a/drivers/gpu/drm/apple/iomfb_m4.c b/drivers/gpu/drm/apple/iomfb_m4.c index 494cfbaa98452..3e639595accda 100644 --- a/drivers/gpu/drm/apple/iomfb_m4.c +++ b/drivers/gpu/drm/apple/iomfb_m4.c @@ -80,8 +80,37 @@ struct iomfb_m4_ext_mode { /* The timing's IsPreferred and firmware score */ bool fw_preferred; s64 score; + /* Its colour element is one the timing lists as needing DSC */ + bool dsc; }; +/* A scratch-only realtime bandwidth mailbox (D003) is this long */ +#define DCP_M4_BW_MAILBOX_SIZE 8 + +/* + * How many external direct swaps may be in flight (A407 answered, the D590 + * not yet in), 1 or 2, read at each frame. At depth 1 every flip event is + * sent with its swap's D590. At depth 2 a frame's flip event can be sent + * before the firmware confirms that the frame replaced the previous one, so a + * compositor may draw into a buffer that is still on screen. + */ +#define DCP_M4_EXT_INFLIGHT 2 +static unsigned int t8132_ext_swap_depth = DCP_M4_EXT_INFLIGHT; + +static int t8132_ext_swap_depth_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 1, DCP_M4_EXT_INFLIGHT); +} + +static const struct kernel_param_ops t8132_ext_swap_depth_ops = { + .set = t8132_ext_swap_depth_set, + .get = param_get_uint, +}; + +module_param_cb(t8132_ext_swap_depth, &t8132_ext_swap_depth_ops, &t8132_ext_swap_depth, 0644); +MODULE_PARM_DESC(t8132_ext_swap_depth, + "J713 external direct swaps in flight at most, 2 (default) or 1, from the next frame. At 2 a flip event can come before the firmware confirms the frame replaced the previous one, so a compositor may draw into a buffer still on screen; at 1 every flip event comes with its swap's completion"); + struct iomfb_m4 { struct apple_dcp *owner; struct device *dev; @@ -89,6 +118,8 @@ struct iomfb_m4 { struct iommu_group *scanout_group; resource_size_t coproc_phys; size_t coproc_size; + /* The realtime bandwidth mailbox from the DT (0: none), see dcp_m4_bw_mailbox() */ + u64 bw_mailbox; struct work_struct backlight_work; void __iomem *coproc; struct apple_rtkit *rtkit; @@ -173,7 +204,6 @@ struct iomfb_m4 { static void dcp_m4_runtime_wake(struct iomfb_m4 *dcp); struct dcp_m4_runtime; -static void dcp_m4_ext_async_done(struct dcp_m4_runtime *r); static void dcp_m4_ext_async_work(struct work_struct *work); static int dcp_m4_set_power_state(struct iomfb_m4 *dcp, bool on); static void dcp_m4_runtime_receive(struct iomfb_m4 *dcp, u64 word); @@ -712,8 +742,8 @@ static int dcp_m4_create_piodma_ext(struct iomfb_m4 *dcp) /* * A host window of SID0 (dva, 0: not claimed) and the framebuffer it maps (key, * NULL: empty), with a reference held by @put's owner; whether the last - * confirmed swap shows it; the given-up swap that may still show it; pinned: - * its unmap failed, kept until reboot. + * confirmed swap shows it (given-up swaps that may still show it are in + * ext_lost_swap); pinned: its unmap failed, kept until reboot. */ #define DCP_M4_EXT_WINDOWS 6 struct dcp_m4_ext_window { @@ -722,10 +752,51 @@ struct dcp_m4_ext_window { void (*put)(void *key); unsigned int count; bool on_screen, pinned; - u32 lost_id; unsigned long used; }; +/* + * A swap given up before its D590 (see dcp_m4_ext_swap_give_up()): its ID (0: + * a free slot), the window it may still put on screen (-1: none, or retired + * by a confirmed swap since), and whether a newer swap was shown before it. + */ +struct dcp_m4_ext_lost { + u32 id; + int win; + bool superseded; +}; + +/* + * A direct swap whose A407 reply came and whose D590 has not: its state, its + * window (-1: none), the link generation and the deadline it went out with, + * whether its flip waits for its completion or the older one's, whether a + * and whether a newer swap was shown before it completed. + */ +struct dcp_m4_ext_inflight { + struct dcp_m4_swap_state swap; + int win; + int gen; + unsigned long deadline; + bool flip_held; + bool superseded; +}; + +/* How a direct swap leaves flight, see dcp_m4_ext_async_done(). */ +enum dcp_m4_ext_end { + DCP_M4_EXT_END_SHOWN, /* its own D590, displayed */ + DCP_M4_EXT_END_KEPT, /* named in a later swap's D590, or shown before a newer one */ + DCP_M4_EXT_END_UNSHOWN, /* its own D590, not displayed */ + DCP_M4_EXT_END_TAKEN, /* its D590 before its A407 reply, taken account of then */ +}; + +/* What the D590 being handled says: shown, and the earlier swaps it completes. */ +struct dcp_m4_ext_done { + bool valid; + bool shown; + u32 count; + u32 hol[4]; +}; + struct dcp_m4_runtime { struct iomfb_m4 *dcp; struct dcp_m4_properties *properties; @@ -812,19 +883,31 @@ struct dcp_m4_runtime { unsigned int ext_back; struct drm_rect ext_prev_damage; /* - * The host windows; the swap in flight without waiting for its D590 - * (its ID, 0: none; its window, -1: none; the link generation and the - * deadline it went out with); whether the next swap_full returns after - * its A407 reply; the work that gives the swap up at its deadline or - * when its link goes. + * The host windows; the direct swaps in flight without waiting for their + * D590, oldest first, and how many; the window of the direct swap going + * out (-1: none); how many the next swap_full may leave in flight, + * returning after its A407 reply (0: it waits for its D590), and whether + * its flip may go at once (none older in flight); what the D590 being + * handled says (dcp_m4_external_callback()), and whether the swap going + * out was shown, by a D590 before its A407 reply; the work that gives the + * oldest up at its deadline or when its link goes. */ struct dcp_m4_ext_window ext_win[DCP_M4_EXT_WINDOWS]; - u32 ext_async_id; - int ext_async_win; - int ext_async_gen; - unsigned long ext_async_deadline; - bool ext_swap_async; + struct dcp_m4_ext_inflight ext_q[DCP_M4_EXT_INFLIGHT]; + unsigned int ext_qn; + int ext_swap_win; + unsigned int ext_swap_async; + bool ext_swap_now; + struct dcp_m4_ext_done ext_done; + bool ext_swap_shown; struct delayed_work ext_async_work; + /* + * Direct swaps completed by a later swap's D590 (named in it), whose own + * D590 may still come: dropped then (0: a free slot). Never evicted: with + * no slot for each swap in flight no direct swap goes out, and A406 + * giving one of these IDs again orphans its start, as for ext_lost_swap. + */ + u32 ext_early[16]; /* * The link's generation, which a released route * (iomfb_m4_external_link_gone()) and a D575(0) move on. @@ -832,10 +915,10 @@ struct dcp_m4_runtime { atomic_t ext_link_gen; /* * Swaps given up before their D590 (see dcp_m4_ext_swap_give_up()), - * each kept until its D590 comes (0: a free slot), so that the late - * D590 is told apart and dropped. + * each kept until its D590 comes (ID 0: a free slot), so that the late + * D590 is told apart and dropped, with the window each may still show. */ - u32 ext_lost_swap[16]; + struct dcp_m4_ext_lost ext_lost_swap[16]; /* * The given-up swap that holds swaps (0: none), and whether a DRM * enable lets one try (a probe); surface of the swap in flight (-1: @@ -858,6 +941,10 @@ struct dcp_m4_runtime { u32 ext_buf_lost[2]; }; +static void dcp_m4_ext_async_done(struct dcp_m4_runtime *r, int i, enum dcp_m4_ext_end end); +static void dcp_m4_ext_shown(struct dcp_m4_runtime *r, u32 id, int win, int older, + enum dcp_m4_ext_end end); + static void dcp_m4_runtime_wake(struct iomfb_m4 *dcp) { struct dcp_m4_runtime *r = smp_load_acquire(&dcp->runtime); @@ -1063,69 +1150,151 @@ static int dcp_m4_runtime_event(void *cookie, const char *name, const void *dict static int dcp_m4_ext_lost_slot(struct dcp_m4_runtime *r, u32 id) { for (unsigned int i = 0; id && i < ARRAY_SIZE(r->ext_lost_swap); i++) - if (r->ext_lost_swap[i] == id) + if (r->ext_lost_swap[i].id == id) return i; return -1; } +/* Whether a given-up swap may still put window @i on screen. */ +static bool dcp_m4_ext_win_lost(struct dcp_m4_runtime *r, int i) +{ + for (unsigned int j = 0; j < ARRAY_SIZE(r->ext_lost_swap); j++) + if (r->ext_lost_swap[j].id && r->ext_lost_swap[j].win == i) + return true; + return false; +} + /* - * The late D590 of the given-up swap @id: dropped, and its - * ID forgotten. A surface it put on screen, which no confirmed swap has - * retired since, is the one on screen now and the other one free; the hold - * it is responsible for, if any, ends. False if @id is no given-up swap's. + * The late D590 of the given-up swap @id: dropped, and its ID forgotten. + * Displayed (@shown): a surface it put on screen, which no confirmed swap has + * retired since, is the one on screen now and the other one free, or, with a + * newer swap shown before it, on screen with that one. Not displayed: what is + * on screen stays, and it no longer makes its surfaces uncertain (others + * given up may still). The hold it is responsible for, if any, ends. False if + * @id is no given-up swap's. */ -static bool dcp_m4_ext_late_d590(struct dcp_m4_runtime *r, u32 id) +static bool dcp_m4_ext_late_d590(struct dcp_m4_runtime *r, u32 id, bool shown) { int slot = dcp_m4_ext_lost_slot(r, id); - bool shown = false, held = r->ext_hold_id == id; + bool front = false, held = r->ext_hold_id == id; + struct dcp_m4_ext_lost lost; if (slot < 0) return false; - r->ext_lost_swap[slot] = 0; + lost = r->ext_lost_swap[slot]; + r->ext_lost_swap[slot] = (struct dcp_m4_ext_lost){ .win = -1 }; for (int i = 0; i < 2; i++) { if (r->ext_buf_lost[i] != id) continue; r->ext_buf_lost[i] = 0; + if (!shown) + continue; r->ext_front = i; r->ext_back = i ^ 1; r->ext_buf_stale[0] = true; r->ext_buf_stale[1] = true; - shown = true; - } - /* A window it put on screen is the one shown now. */ - for (int i = 0; i < DCP_M4_EXT_WINDOWS; i++) { - if (!r->ext_win[i].lost_id || r->ext_win[i].lost_id != id) - continue; - for (int j = 0; j < DCP_M4_EXT_WINDOWS; j++) - r->ext_win[j].on_screen = false; - r->ext_win[i].lost_id = 0; - r->ext_win[i].on_screen = true; - r->ext_front = -1; - r->ext_buf_stale[0] = true; - r->ext_buf_stale[1] = true; - shown = true; + front = true; + } + /* A window it put on screen is the one shown now, or one of them. */ + if (shown && lost.win >= 0) { + if (!lost.superseded) { + for (int j = 0; j < DCP_M4_EXT_WINDOWS; j++) + r->ext_win[j].on_screen = false; + r->ext_front = -1; + r->ext_buf_stale[0] = true; + r->ext_buf_stale[1] = true; + } + r->ext_win[lost.win].on_screen = true; + front = true; } if (held) { r->ext_hold_id = 0; r->ext_hold_probe = false; } dev_info(r->dcp->dev, "swap %u: its D590 came late: dropped%s%s\n", id, - shown ? "; its surface is on screen" : "", held ? "; swaps go on" : ""); + !shown ? "; not displayed, the frame on screen stays" : + front ? "; its surface is on screen" : "", held ? "; swaps go on" : ""); return true; } +/* The slot of @id among the swaps completed early (0: a free one), or -1. */ +static int dcp_m4_ext_early_slot(struct dcp_m4_runtime *r, u32 id) +{ + for (unsigned int i = 0; i < ARRAY_SIZE(r->ext_early); i++) + if (r->ext_early[i] == id) + return i; + return -1; +} + +/* The index of the direct swap @id in flight, or -1. */ +static int dcp_m4_ext_q_find(struct dcp_m4_runtime *r, u32 id) +{ + for (unsigned int i = 0; id && i < r->ext_qn; i++) + if (r->ext_q[i].swap.id == id) + return i; + return -1; +} + static int dcp_m4_runtime_swap_done(void *cookie, u32 id) { struct dcp_m4_runtime *r = cookie; - int ret; - + struct dcp_m4_ext_done done = r->ext_done; + int ret, i, slot; + + if (!r->dcp->external) + return dcp_m4_swap_record_done(&r->swap, id); + r->ext_done.valid = false; + /* Its payload unread: shown, and no earlier swap named. */ + if (!done.valid) + done = (struct dcp_m4_ext_done){ .shown = true }; /* The late D590 of a swap given up is not this swap's. */ - if (r->dcp->external && dcp_m4_ext_late_d590(r, id)) + if (dcp_m4_ext_late_d590(r, id, done.shown)) + return 0; + /* Nor is that of a swap a later one's D590 named. */ + slot = dcp_m4_ext_early_slot(r, id); + if (slot >= 0) { + r->ext_early[slot] = 0; + dev_dbg(r->dcp->dev, "swap %u: its D590 after a later one's: dropped\n", id); + return 0; + } + /* + * The earlier direct swaps in flight it names complete first, kept on + * screen with what is there (macOS completes them as displayed), each + * remembered: its own D590 may still come. Unnamed ones stay in flight. + */ + for (unsigned int j = 0; j < done.count; j++) { + i = done.hol[j] == id ? -1 : dcp_m4_ext_q_find(r, done.hol[j]); + if (i < 0) + continue; + slot = dcp_m4_ext_early_slot(r, 0); + if (slot < 0) { + dev_warn_ratelimited(r->dcp->dev, "swap %u named done by swap %u, with no slot to remember it: left in flight\n", + done.hol[j], id); + continue; + } + r->ext_early[slot] = done.hol[j]; + dcp_m4_ext_async_done(r, i, DCP_M4_EXT_END_KEPT); + } + /* A direct swap in flight is done: its flip, or the next one's. */ + i = dcp_m4_ext_q_find(r, id); + if (i >= 0) { + dcp_m4_ext_async_done(r, i, done.shown ? DCP_M4_EXT_END_SHOWN : + DCP_M4_EXT_END_UNSHOWN); return 0; + } ret = dcp_m4_swap_record_done(&r->swap, id); - /* A direct swap in flight is done: its flip now. */ - if (!ret && r->dcp->external && r->ext_async_id == id && r->swap.replied) - dcp_m4_ext_async_done(r); + /* + * The swap going out, before its A407 reply: whether it was shown, and + * for a direct one what that means for the screen, now, in D590 order; + * only its completion and flip wait for the reply. + */ + if (!ret) { + r->ext_swap_shown = done.shown; + if (r->ext_swap_async) + dcp_m4_ext_shown(r, id, r->ext_swap_win, r->ext_qn, + done.shown ? DCP_M4_EXT_END_SHOWN : + DCP_M4_EXT_END_UNSHOWN); + } return ret; } @@ -1170,6 +1339,27 @@ static int dcp_m4_runtime_idle_cache(void *cookie, unsigned int method, return -EOPNOTSUPP; } +/* + * D003 rt_bandwidth_setup_ap, answered as macOS answers it for a T8132 + * external pipe: the DCP's 8-byte bandwidth request slot as the mailbox, + * whose length (8) selects the scratch-only protocol, with no doorbell. The + * firmware then writes its realtime bandwidth requests there. Without a + * mailbox from the DT (the internal panel) the service is declined. + */ +static int dcp_m4_runtime_bandwidth(void *cookie, void *output) +{ + struct dcp_m4_runtime *r = cookie; + u8 *out = output; + + if (!r->dcp->bw_mailbox) { + put_unaligned_le32(1, out + 0x38); /* status: declined */ + return 0; + } + put_unaligned_le64(r->dcp->bw_mailbox, out + 0x08); + put_unaligned_le32(DCP_M4_BW_MAILBOX_SIZE, out + 0x2c); + return 0; +} + static const struct dcp_m4_callback_ops dcp_m4_runtime_ops = { .call = dcp_m4_runtime_call, .property = dcp_m4_runtime_property, @@ -1177,6 +1367,7 @@ static const struct dcp_m4_callback_ops dcp_m4_runtime_ops = { .event = dcp_m4_runtime_event, .swap_done = dcp_m4_runtime_swap_done, .idle = dcp_m4_runtime_idle_cache, + .bandwidth = dcp_m4_runtime_bandwidth, }; static void dcp_m4_runtime_bad_word(struct dcp_m4_runtime *r, u64 word, int error) @@ -1332,6 +1523,9 @@ static int dcp_m4_external_shape(u32 tag, u32 input, u32 output) if (method == 134) return !input && output == 4 ? method : -EMSGSIZE; + /* D411 mapDeviceMemoryWithIndex, see dcp_m4_external_callback(). */ + if (method == 411) + return input == 0x10 && output == 0x1c ? method : -EMSGSIZE; cb = method < 0 ? NULL : dcp_m4_ext_cb(method); if (!cb) return -EOPNOTSUPP; @@ -1348,6 +1542,46 @@ static int dcp_m4_external_callback(struct dcp_m4_runtime *r, u32 tag, bool remo u8 one[4], out[4]; int ret; + /* + * D590 swap_complete_ap_gated: whether the swap was displayed (+0x4), + * and the earlier swaps it completes, a count at +0x19 of the IDs at + * +0x5, with +0x1d set (and +0x72c, no completion data, clear). Kept for + * its swap_done hook, which the shared dispatcher calls next. + */ + if (method == 590 && input_size == dcp_m4_pf->done_size) { + struct dcp_m4_ext_done *d = &r->ext_done; + u32 count = get_unaligned_le32(input + 0x19); + + *d = (struct dcp_m4_ext_done){ .valid = true, .shown = input[4] != 0 }; + if (input[0x1d] == 1 && !input[0x72c]) { + if (count > ARRAY_SIZE(d->hol)) + dev_warn_ratelimited(r->dcp->dev, "D590 of swap %u names %u earlier swaps: ignored\n", + get_unaligned_le32(input), count); + else + for (d->count = 0; d->count < count; d->count++) + d->hol[d->count] = + get_unaligned_le32(input + 5 + 4 * d->count); + } + return -ENOENT; + } + /* + * D581 swap_complete_head_of_line: a completion too, whose 0xc bytes the + * driver does not decode: logged and acknowledged, its swap left to a + * D590 or to its give-up. + */ + if (method == 581 && input_size == 0xc) { + static DEFINE_RATELIMIT_STATE(rs, 5 * HZ, 4); + u32 id = get_unaligned_le32(input); + + if (__ratelimit(&rs)) { + dev_warn(r->dcp->dev, "D581 swap_complete_head_of_line, word 0 %u (%s): left to a D590 or its give-up\n", + id, dcp_m4_ext_q_find(r, id) >= 0 ? "a swap in flight" : + "no swap in flight"); + print_hex_dump_debug("D581: ", DUMP_PREFIX_OFFSET, 16, 1, input, + input_size, false); + } + return -ENOENT; + } /* * D572 powerUpDART(on): the firmware switches its display DART * on or off, which the host does nothing for but must not touch the @@ -1442,6 +1676,43 @@ static int dcp_m4_external_callback(struct dcp_m4_runtime *r, u32 tag, bool remo put_unaligned_le32(0, output); return 0; } + /* + * D411 mapDeviceMemoryWithIndex, which the firmware asks for its register + * windows (the one holding the bandwidth mailbox among them): "VORP", the + * display register index and flags in; the window's DVA, address and + * length, then a u32 status out. Only a disp-N window the DT describes is + * given. As on the desktop 26.x firmware, the DVA is the physical address: + * the firmware reaches its register windows on its DART-bypass stream. + * Any other index gets status 1. + */ + if (method == 411) { + struct resource *res = NULL; + u8 *o = output; + char name[16]; + u32 index; + + if (input_size != 0x10 || output_size != 0x1c) + return -EMSGSIZE; + index = get_unaligned_le32(input + 4); + if (!memcmp(input, "VORP", 4) && index < MAX_DISP_REGISTERS) { + snprintf(name, sizeof(name), "disp-%u", index); + res = platform_get_resource_byname(to_platform_device(r->dcp->dev), + IORESOURCE_MEM, name); + } + memset(o, 0, 0x1c); + if (!res) { + put_unaligned_le32(1, o + 24); + dev_warn(r->dcp->dev, "D411: register index %u is not described: refused\n", + index); + return 0; + } + put_unaligned_le64(res->start, o); + put_unaligned_le64(res->start, o + 8); + put_unaligned_le64(resource_size(res), o + 16); + dev_dbg(r->dcp->dev, "D411: register index %u (flags %#x) mapped: %pR\n", index, + get_unaligned_le32(input + 8), res); + return 0; + } /* * D575, the hotplug report: the connection state and a 76-byte * tiled-display record (its null flag at 84) in, a 76-byte record out. @@ -1463,8 +1734,8 @@ static int dcp_m4_external_callback(struct dcp_m4_runtime *r, u32 tag, bool remo /* A sink that went takes a new power-on and mode. */ if (!connected) { atomic_inc(&r->ext_link_gen); - /* A direct swap in flight is given up now. */ - if (r->ext_async_id) + /* The direct swaps in flight are given up now. */ + if (r->ext_qn) mod_delayed_work(r->wq, &r->ext_async_work, 0); WRITE_ONCE(r->ext_powered, false); WRITE_ONCE(r->ext_cur_timing, 0); @@ -1743,14 +2014,16 @@ static int dcp_m4_ext_swap_wait(struct dcp_m4_runtime *r, int link_gen, u64 *wor * After one given up on a live link, swaps are held (iomfb_m4_ext_present()) * until its D590 or a confirmed swap after a DRM enable, so that a stuck * pipe does not cost every frame 5 s. With every slot taken the swap is - * not given up. True if the swap was given up so; anything else is the - * session's failure as before. + * not given up. @swap is its state: r->swap, or a direct swap's in flight, + * @win its window (-1: none) and @superseded whether a newer swap was shown + * before it. True if the swap was given up so; anything else is the session's + * failure as before. */ -static bool dcp_m4_ext_swap_give_up(struct dcp_m4_runtime *r, int error) +static bool dcp_m4_ext_swap_give_up(struct dcp_m4_runtime *r, struct dcp_m4_swap_state *swap, + int win, bool superseded, int error) { struct iomfb_m4 *dcp = r->dcp; struct dcp_m4_transaction *t = &dcp->transaction; - struct dcp_m4_swap_state *swap = &r->swap; int slot; if (!dcp->external || (error != -ETIMEDOUT && error != -ENOLINK) || @@ -1762,14 +2035,16 @@ static bool dcp_m4_ext_swap_give_up(struct dcp_m4_runtime *r, int error) return false; slot = -1; for (unsigned int i = 0; slot < 0 && i < ARRAY_SIZE(r->ext_lost_swap); i++) - if (!r->ext_lost_swap[i]) + if (!r->ext_lost_swap[i].id) slot = i; if (slot < 0) { dev_warn(dcp->dev, "swap %u: %d given-up swaps still without their D590: not given up\n", swap->id, (int)ARRAY_SIZE(r->ext_lost_swap)); return false; } - r->ext_lost_swap[slot] = swap->id; + r->ext_lost_swap[slot] = (struct dcp_m4_ext_lost){ + .id = swap->id, .win = win, .superseded = superseded, + }; swap->emitted = true; if (r->ext_swap_surface >= 0) r->ext_buf_lost[r->ext_swap_surface] = swap->id; @@ -1843,9 +2118,9 @@ static int dcp_m4_runtime_swap_full(struct dcp_m4_runtime *r, bool primary, if (!ret) ret = dcp_m4_swap_start_reply(started, sizeof(started), &id); /* - * An ID whose given-up swap's D590 may still come is not reused: the - * two completions could not be told apart, as a D590 names the swap by - * its ID alone. A407 is not sent, and the start is left orphaned: no + * An ID whose given-up (or early-completed) swap's D590 may still come + * is not reused: the two completions could not be told apart, as a D590 + * names the swap by its ID alone. A407 is not sent, and the start is left orphaned: no * host call ends or discards a started swap, and completing it (with a * surfaceless A407) would leave two completions under one ID that a * single D590 could belong to either of, while the firmware may swallow @@ -1853,7 +2128,8 @@ static int dcp_m4_runtime_swap_full(struct dcp_m4_runtime *r, bool primary, * (dcp_m4_runtime_swap_full()) until reboot; the old swap's late D590 * is still dropped, but ends no hold. */ - if (!ret && ext && dcp_m4_ext_lost_slot(r, id) >= 0) { + if (!ret && ext && + (dcp_m4_ext_lost_slot(r, id) >= 0 || dcp_m4_ext_early_slot(r, id) >= 0)) { dev_warn(dcp->dev, "A406 gave swap %u again, whose D590 may still come: not submitted, the start left orphaned; no swap starts until reboot\n", id); r->ext_orphan_start = id; @@ -1876,25 +2152,46 @@ static int dcp_m4_runtime_swap_full(struct dcp_m4_runtime *r, bool primary, if (!ret) ret = dcp_m4_swap_record_reply(&r->swap, submitted, sizeof(submitted)); /* - * A direct swap does not wait for its D590: that completes it - * (dcp_m4_runtime_swap_done()), or its work gives it up at the deadline - * or when the link goes (dcp_m4_ext_async_work()). Nothing of it is - * outstanding on the RPC buffer: A406 and A407 are answered. + * A direct swap does not wait for its D590: it joins the swaps in flight + * and is tracked there (r->swap may begin the next one) until its D590, + * or a later swap's, completes it (dcp_m4_runtime_swap_done()), or its + * work gives it up at the deadline or when the link goes + * (dcp_m4_ext_async_work()). Nothing of it is outstanding on the RPC + * buffer: A406 and A407 are answered. Its flip goes at once with none + * older in flight and a depth over 1 (the previous frame's buffer may still + * be on screen then), else with the completion of the older one, or its + * own. */ if (!ret && ext && r->ext_swap_async) { - r->deadline = 0; - kfree(input); - r->ext_async_id = r->swap.id; - r->ext_async_gen = link_gen; - r->ext_async_deadline = jiffies + msecs_to_jiffies(5000); - if (r->swap.completed) { - dcp_m4_ext_async_done(r); - } else { - queue_delayed_work(r->wq, &r->ext_async_work, msecs_to_jiffies(5000)); - if (!r->ext_connected || atomic_read(&r->ext_link_gen) != link_gen) - mod_delayed_work(r->wq, &r->ext_async_work, 0); + struct dcp_m4_ext_inflight *e = + &r->ext_q[min_t(unsigned int, r->ext_qn, DCP_M4_EXT_INFLIGHT - 1)]; + struct dcp_m4_swap_state handed; + + /* iomfb_m4_ext_present_direct() left room for it. */ + ret = WARN_ON_ONCE(r->ext_qn >= DCP_M4_EXT_INFLIGHT) ? -EBUSY : + dcp_m4_swap_hand_off(&r->swap, &handed); + if (!ret) { + r->deadline = 0; + kfree(input); + *e = (struct dcp_m4_ext_inflight){ + .swap = handed, + .win = r->ext_swap_win, + .gen = link_gen, + .deadline = jiffies + msecs_to_jiffies(5000), + .flip_held = r->ext_qn || r->ext_swap_async < 2, + }; + r->ext_swap_now = !e->flip_held; + r->ext_qn++; + if (e->swap.completed) { + dcp_m4_ext_async_done(r, r->ext_qn - 1, DCP_M4_EXT_END_TAKEN); + } else { + queue_delayed_work(r->wq, &r->ext_async_work, + msecs_to_jiffies(5000)); + if (!r->ext_connected || atomic_read(&r->ext_link_gen) != link_gen) + mod_delayed_work(r->wq, &r->ext_async_work, 0); + } + return 0; } - return 0; } while (!ret && !r->swap.completed) { if (ext) { @@ -1914,7 +2211,7 @@ static int dcp_m4_runtime_swap_full(struct dcp_m4_runtime *r, bool primary, /* Only the copied host input is freed; the firmware-visible RPC stays pinned. */ kfree(input); /* Only its D590 missing: given up, the session kept. */ - if (ret && d590_missing && dcp_m4_ext_swap_give_up(r, ret)) + if (ret && d590_missing && dcp_m4_ext_swap_give_up(r, &r->swap, -1, false, ret)) return ret; if (ret) { dcp_m4_swap_fail(&r->swap, ret); @@ -2016,8 +2313,8 @@ static int dcp_m4_startup(struct iomfb_m4 *dcp) /* No surface on screen or in flight yet. */ r->ext_swap_surface = -1; r->ext_front = -1; - /* No direct swap in flight. */ - r->ext_async_win = -1; + /* No direct swap going out. */ + r->ext_swap_win = -1; INIT_DELAYED_WORK(&r->ext_async_work, dcp_m4_ext_async_work); init_completion(&r->received); INIT_WORK(&r->idle, dcp_m4_runtime_idle); @@ -2187,6 +2484,44 @@ static bool iomfb_m4_firmware_property(struct device_node *np, const char *name) version[0] == 26 && version[1] == 6 && version[2] == 2; } +/* + * The realtime bandwidth mailbox the DT gives @dev, or 0: apple,bw-scratch + * names the provider's reg range, the firmware's display register index and + * an offset into the range. The mailbox is DCP_M4_BW_MAILBOX_SIZE bytes, and + * must lie inside display register window 4, through which the firmware + * reaches a scratch-only mailbox. + */ +static u64 dcp_m4_bw_mailbox(struct device *dev) +{ + struct of_phandle_args args; + struct resource range, *win = NULL; + u64 slot = 0; + int ret; + + ret = of_parse_phandle_with_args(dev->of_node, "apple,bw-scratch", + "#apple,bw-scratch-cells", 0, &args); + if (ret == -ENOENT) + return 0; + if (ret) { + dev_warn(dev, "apple,bw-scratch: %d: realtime bandwidth declined\n", ret); + return 0; + } + if (args.args_count == 3 && args.args[1] == 4 && + !of_address_to_resource(args.np, args.args[0], &range) && + resource_size(&range) >= DCP_M4_BW_MAILBOX_SIZE && + args.args[2] <= resource_size(&range) - DCP_M4_BW_MAILBOX_SIZE) { + win = platform_get_resource_byname(to_platform_device(dev), IORESOURCE_MEM, + "disp-4"); + slot = range.start + args.args[2]; + } + of_node_put(args.np); + if (!win || slot < win->start || slot + DCP_M4_BW_MAILBOX_SIZE - 1 > win->end) { + dev_warn(dev, "apple,bw-scratch: no mailbox inside disp-4: realtime bandwidth declined\n"); + return 0; + } + return slot; +} + int iomfb_m4_init(struct apple_dcp *owner) { struct platform_device *pdev = to_platform_device(owner->dev); @@ -2230,6 +2565,7 @@ int iomfb_m4_init(struct apple_dcp *owner) return -ENOMEM; dcp->owner = owner; dcp->dev = dev; + dcp->bw_mailbox = dcp_m4_bw_mailbox(dev); mutex_init(&dcp->lock); mutex_init(&dcp->ext_modes_lock); spin_lock_init(&dcp->state_lock); @@ -2516,8 +2852,8 @@ void iomfb_m4_external_link_gone(struct apple_dcp *owner) if (r && dcp->external) { /* A swap waiting for its D590 stops waiting (dcp_m4_ext_swap_wait()). */ atomic_inc(&r->ext_link_gen); - /* A direct swap in flight is given up now. */ - if (READ_ONCE(r->ext_async_id)) + /* The direct swaps in flight are given up now. */ + if (READ_ONCE(r->ext_qn)) mod_delayed_work(r->wq, &r->ext_async_work, 0); /* A new link takes a new power-on and mode. */ WRITE_ONCE(r->ext_powered, false); @@ -2567,6 +2903,16 @@ static u64 dcp_m4_ext_capacity(struct apple_dcp *owner) return (u64)lanes * rate * 270 * 1000 * 8 / 10; } +/* + * Whether a cached mode fits the link: at 24 bpp uncompressed, at 8 bpp (the + * lowest DSC target) with a colour that needs DSC, whose rate the firmware + * picks itself. + */ +static bool dcp_m4_ext_fits(const struct iomfb_m4_ext_mode *e, u64 capacity) +{ + return !capacity || (u64)e->mode.clock * (e->dsc ? 8 : 24) <= capacity; +} + /* * The firmware's IDs of a mode iomfb_m4_ext_get_modes() * offered: -ENOENT for any other mode, -ENOSPC for one over the link. @@ -2587,13 +2933,16 @@ int iomfb_m4_ext_mode_ids(struct apple_dcp *owner, const struct drm_display_mode if (!drm_mode_match(&e->mode, mode, DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_CLOCK | DRM_MODE_MATCH_FLAGS)) continue; - if (capacity && (u64)e->mode.clock * 24 > capacity) { + if (!dcp_m4_ext_fits(e, capacity)) { ret = -ENOSPC; continue; } ids->timing = e->timing; ids->color = e->color; ids->bootstrap = e->bootstrap; + dev_dbg(dcp->dev, "mode %ux%u@%d: timing %u, colour %u%s\n", + e->mode.hdisplay, e->mode.vdisplay, drm_mode_vrefresh(&e->mode), + e->timing, e->color, e->dsc ? " (DSC)" : ""); ret = 0; break; } @@ -2601,11 +2950,6 @@ int iomfb_m4_ext_mode_ids(struct apple_dcp *owner, const struct drm_display_mode return ret; } -static bool dcp_m4_ext_fits(const struct drm_display_mode *m, u64 capacity) -{ - return !capacity || (u64)m->clock * 24 <= capacity; -} - /* A mode's refresh rate in mHz, to compare 59.94 with 60 Hz. */ static u64 dcp_m4_ext_mhz(const struct drm_display_mode *m) { @@ -2634,7 +2978,8 @@ static bool dcp_m4_ext_sink_preferred(const struct iomfb_m4 *dcp, /* * The cached mode to mark preferred, -1 for none (the - * 1920x1080 fallback), with the rule that chose it in @why: + * 1920x1080 fallback), never one that needs DSC (the user picks those), with + * the rule that chose it in @why: * 1. the native size is the largest (pixel area) of the sink's preferred * timings: its PreferredTimingElements (their own sizes too) and its * IsPreferred timings; @@ -2670,8 +3015,8 @@ static int dcp_m4_ext_pick_preferred(struct iomfb_m4 *dcp, u64 capacity, const c u64 mhz = dcp_m4_ext_mhz(&e[i].mode), best_mhz; bool pref, best_pref; - if (!dcp_m4_ext_same_size(&e[i].mode, width, height) || - !dcp_m4_ext_fits(&e[i].mode, capacity) || mhz < DCP_M4_EXT_MIN_MHZ) + if (!dcp_m4_ext_same_size(&e[i].mode, width, height) || e[i].dsc || + !dcp_m4_ext_fits(&e[i], capacity) || mhz < DCP_M4_EXT_MIN_MHZ) continue; if (best < 0) { best = i; @@ -2692,8 +3037,8 @@ static int dcp_m4_ext_pick_preferred(struct iomfb_m4 *dcp, u64 capacity, const c u64 a = (u64)e[i].mode.hdisplay * e[i].mode.vdisplay, b; u64 mhz = dcp_m4_ext_mhz(&e[i].mode); - if (!dcp_m4_ext_sink_preferred(dcp, &e[i]) || - !dcp_m4_ext_fits(&e[i].mode, capacity) || mhz < DCP_M4_EXT_MIN_MHZ) + if (!dcp_m4_ext_sink_preferred(dcp, &e[i]) || e[i].dsc || + !dcp_m4_ext_fits(&e[i], capacity) || mhz < DCP_M4_EXT_MIN_MHZ) continue; b = best < 0 ? 0 : (u64)e[best].mode.hdisplay * e[best].mode.vdisplay; if (best < 0 || a > b || (a == b && mhz > dcp_m4_ext_mhz(&e[best].mode))) @@ -2724,7 +3069,7 @@ int iomfb_m4_ext_get_modes(struct apple_dcp *owner, struct drm_connector *connec const struct drm_display_mode *m = &dcp->ext_modes[i].mode; struct drm_display_mode *mode; - if (capacity && (u64)m->clock * 24 > capacity) { + if (!dcp_m4_ext_fits(&dcp->ext_modes[i], capacity)) { skipped++; continue; } @@ -2736,8 +3081,10 @@ int iomfb_m4_ext_get_modes(struct apple_dcp *owner, struct drm_connector *connec mode->type |= DRM_MODE_TYPE_PREFERRED; preferred = mode; } - if (!fallback || (mode->hdisplay == 1920 && mode->vdisplay == 1080 && - drm_mode_vrefresh(mode) == 60)) + /* A DSC mode is never the fallback either: the user picks one. */ + if (!dcp->ext_modes[i].dsc && + (!fallback || (mode->hdisplay == 1920 && mode->vdisplay == 1080 && + drm_mode_vrefresh(mode) == 60))) fallback = mode; drm_mode_probed_add(connector, mode); added++; @@ -3040,9 +3387,10 @@ static_assert(sizeof(struct dcp_m4_mode_info_req) == 0x7c); /* * The colour element of a timing: RGB with the SDR gamma EOTF, not virtual, - * not unsafe or needing DSC for the timing; the lowest depth offered (8 bpc - * if there is one); then sRGB/BT.709 colorimetry and full range over - * others, then the higher score. Never PQ or HLG. + * not unsafe for the timing, at 8 bpc if the timing lists it as needing DSC; + * the lowest depth offered (8 bpc if there is one); at that depth one that + * needs no DSC over one that does, then sRGB/BT.709 colorimetry and full + * range over others, then the higher score. Never PQ or HLG. */ struct dcp_m4_ext_pick { struct dcp_color_element best; @@ -3061,16 +3409,18 @@ static void dcp_m4_ext_pick_color(void *data, const struct dcp_color_element *e) struct dcp_m4_ext_pick *p = data; unsigned int rank, best_rank; - if (e->is_virtual || e->restricted || e->id < 0 || e->id > U32_MAX || e->score < 0 || + if (e->is_virtual || e->unsafe || e->id < 0 || e->id > U32_MAX || e->score < 0 || e->depth <= 0 || e->encoding != DCP_COLOR_FORMAT_RGB || - e->eotf != DCP_EOTF_SDR_GAMMA) + e->eotf != DCP_EOTF_SDR_GAMMA || (e->dsc_required && e->depth != 8)) return; rank = dcp_m4_ext_color_rank(e); best_rank = p->found ? dcp_m4_ext_color_rank(&p->best) : 0; if (p->found && (e->depth > p->best.depth || (e->depth == p->best.depth && - (rank < best_rank || - (rank == best_rank && e->score <= p->best.score))))) + (e->dsc_required > p->best.dsc_required || + (e->dsc_required == p->best.dsc_required && + (rank < best_rank || + (rank == best_rank && e->score <= p->best.score))))))) return; p->best = *e; p->found = true; @@ -3094,7 +3444,8 @@ static bool dcp_m4_ext_timing_color(const void *blob, u32 size, u32 timing, /* * The timings of the sink's TimingElements @blob with an * 8-bpc RGB SDR colour element (see dcp_m4_ext_pick_color()), not - * interlaced, for its DRM connector. Replaces the previous list and starts + * interlaced, for its DRM connector; those whose only such element needs DSC + * are marked so. Replaces the previous list and starts * the sink's catalog (no PreferredTimingElements yet); not if received in * sink generation @gen, before a released route (stale). True if cached. */ @@ -3136,6 +3487,11 @@ static bool dcp_m4_ext_cache_modes(struct iomfb_m4 *dcp, const void *blob, u32 s list[kept].bootstrap = m->has_bootstrap ? m->bootstrap_color_id : 0; list[kept].fw_preferred = m->firmware_preferred; list[kept].score = m->score; + list[kept].dsc = color.dsc_required; + if (color.dsc_required) + dev_dbg(dcp->dev, "timing %u %ux%u@%d: DSC colour %u\n", + m->timing_mode_id, m->mode.hdisplay, m->mode.vdisplay, + drm_mode_vrefresh(&m->mode), (u32)color.id); kept++; } kfree(modes); @@ -3439,17 +3795,16 @@ static bool dcp_m4_ext_ids_current(struct iomfb_m4 *dcp, const struct iomfb_m4_e found = e->timing == ids->timing && e->color == ids->color && e->bootstrap == ids->bootstrap && e->mode.hdisplay == width && - e->mode.vdisplay == height && - (!capacity || (u64)e->mode.clock * 24 <= capacity); + e->mode.vdisplay == height && dcp_m4_ext_fits(e, capacity); } mutex_unlock(&dcp->ext_modes_lock); return found; } /* - * DRM: before an enable or disable, which must not be refused: a direct swap - * in flight completes (D590) or is given up (5 s, or at once when the link - * goes). The session lock is dropped while waiting, so those can run; + * DRM: before an enable or disable, which must not be refused: the direct + * swaps in flight complete (D590) or are given up (5 s, or at once when the + * link goes). The session lock is dropped while waiting, so those can run; * nothing here holds a DRM lock they need. -EBUSY if it has not after 10 s. */ static int dcp_m4_ext_settle(struct iomfb_m4 *dcp, struct dcp_m4_runtime *r, const char *what) @@ -3457,16 +3812,17 @@ static int dcp_m4_ext_settle(struct iomfb_m4 *dcp, struct dcp_m4_runtime *r, con unsigned long end = jiffies + msecs_to_jiffies(10000); lockdep_assert_held(&dcp->lock); - while (r->ext_async_id) { + while (r->ext_qn) { long left = (long)(end - jiffies); if (left <= 0) { - dev_warn(dcp->dev, "%s: a direct swap still under way after 10 s\n", what); + dev_warn(dcp->dev, "%s: %u direct swaps still under way after 10 s\n", what, + r->ext_qn); return -EBUSY; } - dev_dbg(dcp->dev, "%s waits for the direct swap in flight\n", what); + dev_dbg(dcp->dev, "%s waits for the %u direct swaps in flight\n", what, r->ext_qn); mutex_unlock(&dcp->lock); - wait_event_timeout(dcp->ext_settle_wq, !READ_ONCE(r->ext_async_id), left); + wait_event_timeout(dcp->ext_settle_wq, !READ_ONCE(r->ext_qn), left); mutex_lock(&dcp->lock); } return 0; @@ -3507,7 +3863,7 @@ int iomfb_m4_ext_enable(struct apple_dcp *owner, const struct iomfb_m4_ext_ids * /* The link and sink this enable is for, to tell if they changed meanwhile. */ link_gen = atomic_read(&r->ext_link_gen); sink_gen = atomic_read(&dcp->ext_sink_gen); - /* Never refused: a direct swap in flight ends first. */ + /* Never refused: the direct swaps in flight end first. */ ret = dcp_m4_ext_settle(dcp, r, "enable"); if (ret) goto out; @@ -3587,6 +3943,19 @@ int iomfb_m4_ext_enable(struct apple_dcp *owner, const struct iomfb_m4_ext_ids * return ret; } +/* + * Whether the synchronous swap just completed was displayed, as its D590 + * says; logged if not, for @what. + */ +static bool dcp_m4_ext_sync_shown(struct dcp_m4_runtime *r, const char *what) +{ + if (r->ext_swap_shown) + return true; + dev_dbg(r->dcp->dev, "%s: swap %u completed, not displayed: the frame on screen stays\n", + what, r->swap.id); + return false; +} + /* * A swap of surface @shown (-1: surfaceless) completed, * D590 included: that surface is the one on screen, the other one free; @@ -3607,10 +3976,10 @@ static void dcp_m4_ext_retired(struct dcp_m4_runtime *r, int shown) if (shown >= 0) r->ext_back = shown ^ 1; /* No window is on screen, nor may a given-up swap show one. */ - for (int i = 0; i < DCP_M4_EXT_WINDOWS; i++) { + for (int i = 0; i < DCP_M4_EXT_WINDOWS; i++) r->ext_win[i].on_screen = false; - r->ext_win[i].lost_id = 0; - } + for (unsigned int i = 0; i < ARRAY_SIZE(r->ext_lost_swap); i++) + r->ext_lost_swap[i].win = -1; } /* Rows of @rect from @src (@pitch) into @dst (@stride); its pixels. */ @@ -3684,7 +4053,7 @@ int iomfb_m4_ext_present(struct apple_dcp *owner, const struct iosys_map *src, u goto out; ret = -EBUSY; if ((r->ext_hold_id && !r->ext_hold_probe) || r->ext_orphan_start || - r->ext_async_id || r->ext_sleeping) + r->ext_qn || r->ext_sleeping) goto out; r->ext_hold_probe = false; /* @@ -3704,6 +4073,10 @@ int iomfb_m4_ext_present(struct apple_dcp *owner, const struct iosys_map *src, u back ^ 1, r->ext_buf_lost[0], r->ext_buf_lost[1], ret); if (ret) goto out; + /* Not displayed: no surface was freed. */ + ret = -EBUSY; + if (!dcp_m4_ext_sync_shown(r, "copy")) + goto out; dcp_m4_ext_retired(r, -1); r->ext_buf_stale[0] = true; r->ext_buf_stale[1] = true; @@ -3757,9 +4130,17 @@ int iomfb_m4_ext_present(struct apple_dcp *owner, const struct iosys_map *src, u r->ext_buf_stale[back] = true; goto out; } - dcp_m4_ext_retired(r, back); r->ext_buf_stale[back] = false; r->ext_prev_damage = cur; + /* + * Not displayed: what was on screen stays, this surface the back one; the + * other one never got this frame's damage, so it takes a whole frame. + */ + if (!dcp_m4_ext_sync_shown(r, "copy")) { + r->ext_buf_stale[back ^ 1] = true; + goto out; + } + dcp_m4_ext_retired(r, back); /* A frame shown, for the replay after a sleep. */ if (r->ext_restore_pending) WRITE_ONCE(r->ext_restore_frame, true); @@ -3811,14 +4192,20 @@ static void iomfb_m4_ext_put_work(struct work_struct *work) /* * Whether window @i may be unmapped: it maps a framebuffer that no swap shows - * (as last confirmed), may still show (a given-up one) or has in flight, and - * its unmap never failed. + * (as last confirmed), may still show (a given-up one) or has going out or in + * flight, and its unmap never failed. */ static bool dcp_m4_ext_win_idle(struct dcp_m4_runtime *r, int i) { const struct dcp_m4_ext_window *w = &r->ext_win[i]; - return w->key && !w->on_screen && !w->lost_id && !w->pinned && r->ext_async_win != i; + if (!w->key || w->on_screen || w->pinned || r->ext_swap_win == i || + dcp_m4_ext_win_lost(r, i)) + return false; + for (unsigned int j = 0; j < r->ext_qn; j++) + if (r->ext_q[j].win == i) + return false; + return true; } /* @@ -3964,66 +4351,135 @@ static int dcp_m4_ext_win_get(struct dcp_m4_runtime *r, const struct iomfb_m4_ex return pick; } +/* Take the direct swap at @i out of flight. */ +static void dcp_m4_ext_q_remove(struct dcp_m4_runtime *r, int i) +{ + r->ext_qn--; + memmove(&r->ext_q[i], &r->ext_q[i + 1], (r->ext_qn - i) * sizeof(r->ext_q[0])); + if (!r->ext_qn) + cancel_delayed_work(&r->ext_async_work); + wake_up_all(&r->dcp->ext_settle_wq); +} + /* - * The direct swap in flight is done, A407 reply and D590 in: its window is the - * one on screen, every other window and the copy surfaces retired (and those - * surfaces stale), and its flip is sent. Session lock held. + * The direct swap @e out of flight (completed or given up): its flip if it + * waited for that, and the oldest one's if that waited for an older one. */ -static void dcp_m4_ext_async_done(struct dcp_m4_runtime *r) +static void dcp_m4_ext_flip_after(struct dcp_m4_runtime *r, const struct dcp_m4_ext_inflight *e) { - int w = r->ext_async_win; + bool send = e->flip_held; + + if (r->ext_qn && r->ext_q[0].flip_held) { + r->ext_q[0].flip_held = false; + send = true; + } + if (send) + t8132_ext_drm_flip_done(r->dcp->dev); +} - if (dcp_m4_swap_take_completion(&r->swap) != 1) - dev_warn(r->dcp->dev, "direct swap %u: completion not taken\n", - r->ext_async_id); - cancel_delayed_work(&r->ext_async_work); - dcp_m4_ext_retired(r, -1); - if (w >= 0) - r->ext_win[w].on_screen = true; +/* + * What the D590 of direct swap @id (or a later one naming it) means for the + * screen, taken account of as it comes: @win its window (-1: none), @older how + * many direct swaps in flight are older. SHOWN: its window is the one on + * screen, every other window and the copy surfaces retired; the older ones + * complete as KEPT then. KEPT: its window counts as on screen too, nothing + * else is retired. UNSHOWN: what was on screen stays, and its window is free. + * The copy surfaces go stale and a hold ends either way. Session lock held. + */ +static void dcp_m4_ext_shown(struct dcp_m4_runtime *r, u32 id, int win, int older, + enum dcp_m4_ext_end end) +{ + if (end == DCP_M4_EXT_END_SHOWN) { + dcp_m4_ext_retired(r, -1); + for (int j = 0; j < older; j++) + r->ext_q[j].superseded = true; + } else { + if (end == DCP_M4_EXT_END_UNSHOWN) + dev_dbg(r->dcp->dev, "swap %u completed, not displayed: the frame on screen stays\n", + id); + if (r->ext_hold_id) + dev_info(r->dcp->dev, "a swap completed: the hold for swap %u ends\n", + r->ext_hold_id); + r->ext_hold_id = 0; + r->ext_hold_probe = false; + } + if (end != DCP_M4_EXT_END_UNSHOWN) { + if (win >= 0) + r->ext_win[win].on_screen = true; + /* A frame shown, for the replay after a sleep. */ + if (r->ext_restore_pending) + WRITE_ONCE(r->ext_restore_frame, true); + } r->ext_buf_stale[0] = true; r->ext_buf_stale[1] = true; - r->ext_async_id = 0; - r->ext_async_win = -1; - wake_up_all(&r->dcp->ext_settle_wq); - /* A frame shown, for the replay after a sleep. */ - if (r->ext_restore_pending) - WRITE_ONCE(r->ext_restore_frame, true); - t8132_ext_drm_flip_done(r->dcp->dev); } /* - * Give the direct swap in flight up, as a waiting swap would be (its window may - * still show), its link gone (@went) or its time out; its flip goes as lost. - * Session lock held. + * The direct swap in flight at @i is done, A407 reply and D590 in, and leaves + * flight; older ones not completed stay. @end as dcp_m4_ext_shown(), SHOWN + * as KEPT for one superseded; TAKEN: its D590 came before its A407 reply and + * was taken account of then. Its flip, or the next one's. Session lock held. + */ +static void dcp_m4_ext_async_done(struct dcp_m4_runtime *r, int i, enum dcp_m4_ext_end end) +{ + struct dcp_m4_ext_inflight e = r->ext_q[i]; + + if (end == DCP_M4_EXT_END_SHOWN && e.superseded) + end = DCP_M4_EXT_END_KEPT; + e.swap.completed = true; + if (dcp_m4_swap_take_completion(&e.swap) != 1) + dev_warn_ratelimited(r->dcp->dev, "direct swap %u: completion not taken\n", + e.swap.id); + if (i) + dev_warn_ratelimited(r->dcp->dev, "swap %u completed before the older swap %u, which stays in flight\n", + e.swap.id, r->ext_q[0].swap.id); + dcp_m4_ext_q_remove(r, i); + if (end != DCP_M4_EXT_END_TAKEN) + dcp_m4_ext_shown(r, e.swap.id, e.win, i, end); + dcp_m4_ext_flip_after(r, &e); +} + +/* Whether the oldest direct swap in flight is due its give-up, and its link went (@went). */ +static bool dcp_m4_ext_async_due(struct dcp_m4_runtime *r, bool *went) +{ + const struct dcp_m4_ext_inflight *e = &r->ext_q[0]; + + *went = !r->ext_connected || atomic_read(&r->ext_link_gen) != e->gen; + return *went || !time_before(jiffies, e->deadline); +} + +/* + * Give the oldest direct swap in flight up, as a waiting swap would be (its + * window may still show), its link gone (@went) or its time out; its flip goes + * as lost (or the next one's, dcp_m4_ext_flip_after()). Session lock held. */ static void dcp_m4_ext_async_expire(struct dcp_m4_runtime *r, bool went) { struct iomfb_m4 *dcp = r->dcp; - u32 id = r->ext_async_id; - int w = r->ext_async_win; + struct dcp_m4_ext_inflight e = r->ext_q[0]; int ret = went ? -ENOLINK : -ETIMEDOUT; r->ext_swap_surface = -1; - if (!READ_ONCE(dcp->error) && dcp_m4_ext_swap_give_up(r, ret)) { - if (w >= 0) - r->ext_win[w].lost_id = id; - } else if (!READ_ONCE(dcp->error)) { + /* The copy surfaces never got this frame: whole frames, whatever its late D590 says. */ + r->ext_buf_stale[0] = true; + r->ext_buf_stale[1] = true; + if (!READ_ONCE(dcp->error) && + !dcp_m4_ext_swap_give_up(r, &e.swap, e.win, e.superseded, ret)) { dcp_m4_swap_fail(&r->swap, ret); dcp_m4_transaction_fail(&dcp->transaction, ret); dcp_m4_failed(dcp, ret); } - r->ext_async_id = 0; - r->ext_async_win = -1; - wake_up_all(&dcp->ext_settle_wq); - t8132_ext_drm_flip_done(dcp->dev); + dcp_m4_ext_q_remove(r, 0); + dcp_m4_ext_flip_after(r, &e); } /* - * The direct swap in flight at its deadline, or after its link went (D575(0), - * a released route): words already received are taken first (its D590 among - * them maybe); then it is given up as a waiting swap would be - * (dcp_m4_ext_swap_give_up(): its window may still show, and is kept), and - * its flip is sent as lost. + * The oldest direct swap in flight at its deadline, or after its link went + * (D575(0), a released route): words already received are taken first (its + * D590 among them maybe); then each one due is given up as a waiting swap + * would be (dcp_m4_ext_swap_give_up(): its window may still show, and is + * kept), and its flip is sent as lost. The work comes back at the next one's + * deadline. */ static void dcp_m4_ext_async_work(struct work_struct *work) { @@ -4034,21 +4490,20 @@ static void dcp_m4_ext_async_work(struct work_struct *work) int ret; mutex_lock(&dcp->lock); - if (!r->ext_async_id) - goto out; - went = !r->ext_connected || atomic_read(&r->ext_link_gen) != r->ext_async_gen; - if (!went && time_before(jiffies, r->ext_async_deadline)) { - queue_delayed_work(r->wq, &r->ext_async_work, r->ext_async_deadline - jiffies); - goto out; + if (r->ext_qn && dcp_m4_ext_async_due(r, &went)) { + ret = READ_ONCE(dcp->error) ?: dcp_m4_runtime_drain(r, 32); + if (ret < 0) { + dcp_m4_transaction_fail(&dcp->transaction, ret); + dcp_m4_failed(dcp, ret); + } + while (r->ext_qn && dcp_m4_ext_async_due(r, &went)) + dcp_m4_ext_async_expire(r, went); } - ret = READ_ONCE(dcp->error) ?: dcp_m4_runtime_drain(r, 32); - if (ret < 0) { - dcp_m4_transaction_fail(&dcp->transaction, ret); - dcp_m4_failed(dcp, ret); + if (r->ext_qn) { + long left = (long)(r->ext_q[0].deadline - jiffies); + + queue_delayed_work(r->wq, &r->ext_async_work, max(left, 0L)); } - if (r->ext_async_id) - dcp_m4_ext_async_expire(r, went); -out: mutex_unlock(&dcp->lock); } @@ -4056,10 +4511,13 @@ static void dcp_m4_ext_async_work(struct work_struct *work) * One DRM frame scanned out directly * from its framebuffer, mapped into a SID0 host window (kept per * framebuffer, see dcp_m4_ext_win_get()), with a one-surface swap that - * returns after its A407 reply; the flip is sent with its D590, or as lost - * when it is given up. One swap in flight: -EBUSY while one is, checked - * before any A406. -ENOSPC without a window, -EOPNOTSUPP for a framebuffer - * that does not fit the mode or the window: the caller copies it instead. + * returns after its A407 reply. Up to t8132_ext_swap_depth swaps in flight: + * -EBUSY while that many are, checked before any A406. 1 once submitted with + * none older in flight and a depth over 1: its flip now; 0 once submitted + * otherwise: its flip is sent with the older one's completion, or its own + * (D590), or as lost when that is given up. -ENOSPC without a window, + * -EOPNOTSUPP for a framebuffer that does not fit the mode or the window: the + * caller copies it instead. */ int iomfb_m4_ext_present_direct(struct apple_dcp *owner, const struct iomfb_m4_ext_fb *fb) { @@ -4067,6 +4525,7 @@ int iomfb_m4_ext_present_direct(struct apple_dcp *owner, const struct iomfb_m4_e struct dcp_m4_runtime *r; struct dcp_m4_frame f; int ret = -ENOLINK, w; + unsigned int depth, free; size_t need; if (!dcp || !dcp->external) @@ -4082,9 +4541,23 @@ int iomfb_m4_ext_present_direct(struct apple_dcp *owner, const struct iomfb_m4_e if (r->ext_mode_stale) goto out; ret = -EBUSY; + depth = clamp_val(READ_ONCE(t8132_ext_swap_depth), 1, DCP_M4_EXT_INFLIGHT); if ((r->ext_hold_id && !r->ext_hold_probe) || r->ext_orphan_start || - r->ext_async_id || r->ext_sleeping) + r->ext_qn >= depth || r->ext_sleeping) goto out; + /* + * A slot for each swap in flight, this one included, that a later D590 + * may complete (dcp_m4_runtime_swap_done()); else frames are copied. + */ + ret = -ENOSPC; + free = 0; + for (unsigned int i = 0; i < ARRAY_SIZE(r->ext_early); i++) + free += !r->ext_early[i]; + if (free <= r->ext_qn) { + dev_warn_ratelimited(dcp->dev, "%u swaps completed by a later D590 still without their own: frames copied\n", + (unsigned int)ARRAY_SIZE(r->ext_early) - free); + goto out; + } ret = -EOPNOTSUPP; need = PAGE_ALIGN((size_t)fb->pitch * fb->height); if (fb->width != r->ext_width || fb->height != r->ext_height || @@ -4119,12 +4592,14 @@ int iomfb_m4_ext_present_direct(struct apple_dcp *owner, const struct iomfb_m4_e }; r->ext_hold_probe = false; r->ext_swap_surface = -1; - r->ext_async_win = w; - r->ext_swap_async = true; + r->ext_swap_win = w; + r->ext_swap_async = depth; + r->ext_swap_now = false; ret = dcp_m4_runtime_swap_full(r, true, &f, NULL, 0, false, 0, 0, 0); - r->ext_swap_async = false; - if (ret && !r->ext_async_id) - r->ext_async_win = -1; + r->ext_swap_async = 0; + r->ext_swap_win = -1; + if (!ret && r->ext_swap_now) + ret = 1; out: mutex_unlock(&dcp->lock); return ret; @@ -4150,7 +4625,7 @@ int iomfb_m4_ext_blank(struct apple_dcp *owner) mutex_unlock(&dcp->lock); return 0; } - /* Never refused: a direct swap in flight ends first. */ + /* Never refused: the direct swaps in flight end first. */ /* Pairs with the release store of r->done in startup. */ if (r && smp_load_acquire(&r->done)) { int link_gen = atomic_read(&r->ext_link_gen); @@ -4179,7 +4654,8 @@ int iomfb_m4_ext_blank(struct apple_dcp *owner) dcp_m4_runtime_swap_full(r, true, NULL, NULL, 0, false, 0, 0, 0); if (ret == -EBUSY) dev_info(dcp->dev, "disable: swaps held or a start orphaned: the last frame stays\n"); - if (!ret) { + /* Only a displayed blank takes the windows off screen. */ + if (!ret && dcp_m4_ext_sync_shown(r, "disable")) { dcp_m4_ext_retired(r, -1); /* The windows are off screen: let them go. */ if (dcp_m4_ext_win_release_idle(r, 0)) @@ -4223,12 +4699,12 @@ bool iomfb_m4_ext_sleep_mark(struct apple_dcp *owner) /* * PM_SUSPEND_PREPARE, before the tasks freeze, every worker running, after - * iomfb_m4_ext_sleep_mark(): the display down for the sleep: the swap in flight - * settles (its D590, or its give-up); a surfaceless swap; every window let go - * of; the display powered off (A473 0) and its firmware's D572(0) waited for - * (1 s), so that the DCP's suspend finds the display DART off, else it refuses - * the sleep. The DRM state replayed after the sleep powers on and sets the - * mode from scratch. + * iomfb_m4_ext_sleep_mark(): the display down for the sleep: the swaps in + * flight settle (their D590, or their give-up); a surfaceless swap; every + * window let go of; the display powered off (A473 0) and its firmware's + * D572(0) waited for (1 s), so that the DCP's suspend finds the display DART + * off, else it refuses the sleep. The DRM state replayed after the sleep + * powers on and sets the mode from scratch. */ void iomfb_m4_ext_sleep_down(struct apple_dcp *owner) { @@ -4246,9 +4722,10 @@ void iomfb_m4_ext_sleep_down(struct apple_dcp *owner) mutex_unlock(&dcp->lock); return; } - dev_dbg(dcp->dev, "sleep: display going down (display DART %s, swap in flight %u)\n", - READ_ONCE(dcp->ext_dart_down) ? "off" : "on", r->ext_async_id); - /* The workers run: the swap in flight completes or is given up as ever. */ + dev_dbg(dcp->dev, "sleep: display going down (display DART %s, %u swaps in flight, oldest %u)\n", + READ_ONCE(dcp->ext_dart_down) ? "off" : "on", r->ext_qn, + r->ext_qn ? r->ext_q[0].swap.id : 0); + /* The workers run: the swaps in flight complete or are given up as ever. */ ret = dcp_m4_ext_settle(dcp, r, "sleep"); if (!ret && dcp_m4_ext_route_ready(dcp, r) && READ_ONCE(r->ext_cur_timing)) { r->ext_swap_surface = -1; @@ -4256,7 +4733,8 @@ void iomfb_m4_ext_sleep_down(struct apple_dcp *owner) dcp_m4_runtime_swap_full(r, true, NULL, NULL, 0, false, 0, 0, 0); if (ret) dev_warn(dcp->dev, "sleep: surfaceless swap failed: %d\n", ret); - if (!ret) { + /* Only a displayed blank takes the windows off screen. */ + if (!ret && dcp_m4_ext_sync_shown(r, "sleep")) { dcp_m4_ext_retired(r, -1); for (int i = 0; i < DCP_M4_EXT_WINDOWS; i++) mapped += !!r->ext_win[i].key; @@ -4462,7 +4940,7 @@ int iomfb_m4_suspend(struct apple_dcp *owner) mutex_lock(&dcp->lock); if (!READ_ONCE(dcp->ext_dart_down)) why = "the display DART is on (last D572 was 1)"; - else if (dcp->runtime->ext_async_id) + else if (dcp->runtime->ext_qn) why = "a swap is under way"; mutex_unlock(&dcp->lock); if (why) { diff --git a/drivers/gpu/drm/apple/iomfb_m4.h b/drivers/gpu/drm/apple/iomfb_m4.h index f76966b88db41..c45d737b1d726 100644 --- a/drivers/gpu/drm/apple/iomfb_m4.h +++ b/drivers/gpu/drm/apple/iomfb_m4.h @@ -59,7 +59,7 @@ int iomfb_m4_ext_enable(struct apple_dcp *dcp, const struct iomfb_m4_ext_ids *id int iomfb_m4_ext_present(struct apple_dcp *dcp, const struct iosys_map *src, u32 pitch, const struct drm_rect *damage); int iomfb_m4_ext_blank(struct apple_dcp *dcp); -/* 0 once submitted; the flip comes with its D590. */ +/* 0 once submitted, its flip later (t8132_ext_drm_flip_done()); 1: its flip now. */ int iomfb_m4_ext_present_direct(struct apple_dcp *dcp, const struct iomfb_m4_ext_fb *fb); /* Why an external DCP's display refuses system sleep, or NULL. */ const char *iomfb_m4_ext_sleep_veto(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 037c0f4e9ba99..6e48211d69f53 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -2296,7 +2296,7 @@ static int parse_color_element(struct dcp_parse_ctx *handle, struct dcp_color_el *e = (struct dcp_color_element) { .id = -1, .depth = -1, .range = -1, .eotf = -1, .encoding = -1, - .colorimetry = -1, .score = -1, .is_virtual = true, .restricted = false, + .colorimetry = -1, .score = -1, .is_virtual = true, }; if (ret) return ret; @@ -2354,31 +2354,32 @@ static int parse_id_list(struct dcp_parse_ctx *handle, s64 *ids, unsigned int *c /* * Each colour element (virtual ones too) of the timing @timing_id in a - * TimingElements catalog, flagged if the timing lists it as unsafe or as - * needing DSC; whether the timing is interlaced. A malformed timing entry is - * skipped. -ENOENT if the catalog has no such timing. + * TimingElements catalog, flagged if the timing lists it as unsafe and if it + * lists it as needing DSC; whether the timing is interlaced. A malformed + * timing entry is skipped. -ENOENT if the catalog has no such timing. */ int dcp_parse_timing_colors(const void *blob, size_t len, u32 timing_id, dcp_color_element_fn emit, void *data, bool *interlaced) { struct dcp_parse_ctx ctx; struct iterator timings; - s64 *restricted; + s64 *unsafe, *dsc; int ret = parse(blob, len, &ctx); if (!ret) ret = iterator_begin(&ctx, &timings, false); if (ret) return ret; - restricted = kcalloc(DCP_TIMING_RESTRICTED_MAX, sizeof(*restricted), GFP_KERNEL); - if (!restricted) + unsafe = kcalloc(2 * DCP_TIMING_RESTRICTED_MAX, sizeof(*unsafe), GFP_KERNEL); + if (!unsafe) return -ENOMEM; + dsc = unsafe + DCP_TIMING_RESTRICTED_MAX; ret = -ENOENT; for (; timings.idx < timings.len; ++timings.idx) { struct dcp_parse_ctx entry_start = ctx, colors = {}; struct iterator entry, it; bool match = false, have_colors = false, is_interlaced = false; - unsigned int nr_restricted = 0; + unsigned int nr_unsafe = 0, nr_dsc = 0; int err; /* ColorModes may come before the ID: find both, then walk the colours. */ @@ -2398,9 +2399,10 @@ int dcp_parse_timing_colors(const void *blob, size_t len, u32 timing_id, colors = ctx; have_colors = true; err = skip(&ctx); - } else if (!strcmp(key, "UnsafeColorElementIDs") || - !strcmp(key, "DSCRequiredColorElementIDs")) { - err = parse_id_list(&ctx, restricted, &nr_restricted); + } else if (!strcmp(key, "UnsafeColorElementIDs")) { + err = parse_id_list(&ctx, unsafe, &nr_unsafe); + } else if (!strcmp(key, "DSCRequiredColorElementIDs")) { + err = parse_id_list(&ctx, dsc, &nr_dsc); } else if (!strcmp(key, "IsInterlaced")) { err = parse_bool(&ctx, &is_interlaced); } else { @@ -2431,14 +2433,16 @@ int dcp_parse_timing_colors(const void *blob, size_t len, u32 timing_id, ret = parse_color_element(it.handle, &e); if (ret) break; - for (unsigned int i = 0; i < nr_restricted; i++) - e.restricted |= restricted[i] == e.id; + for (unsigned int i = 0; i < nr_unsafe; i++) + e.unsafe |= unsafe[i] == e.id; + for (unsigned int i = 0; i < nr_dsc; i++) + e.dsc_required |= dsc[i] == e.id; emit(data, &e); } goto out; } out: - kfree(restricted); + kfree(unsafe); return ret; } diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index ef2424a2bdaa3..3fa384ecf68c4 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -272,8 +272,8 @@ int parse_system_log_mnits_pairs(const void *blob, size_t size, struct dcp_color_element { s64 id, depth, range, eotf, encoding, colorimetry, score; bool is_virtual; - /* The timing lists it as unsafe or as needing DSC (timing walks only). */ - bool restricted; + /* The timing lists it as unsafe, or as needing DSC (timing walks only). */ + bool unsafe, dsc_required; }; typedef void (*dcp_color_element_fn)(void *data, const struct dcp_color_element *element); diff --git a/drivers/gpu/drm/apple/t8132_ext_drm.c b/drivers/gpu/drm/apple/t8132_ext_drm.c index 6d35ea3ada2d2..f6cc2f7767190 100644 --- a/drivers/gpu/drm/apple/t8132_ext_drm.c +++ b/drivers/gpu/drm/apple/t8132_ext_drm.c @@ -73,7 +73,10 @@ struct t8132_ext_out { struct drm_connector connector; struct platform_device *pdev; struct work_struct hotplug_work; - /* The held flip event, sent with its swap's D590 or later (drm->event_lock) */ + /* + * The held flip event, sent once its frame is on its way (see + * t8132_ext_crtc_flush()) or later (drm->event_lock) + */ struct drm_pending_vblank_event *event; struct delayed_work flip_work; /* The DRM device suspends before this DCP and resumes after it */ @@ -340,9 +343,10 @@ static void t8132_ext_direct_refused(struct t8132_ext_out *out, struct drm_frame } /* - * The frame scanned out directly from @fb: 2 once its swap is - * submitted (the flip comes with its D590), -ENOSPC or -EOPNOTSUPP to copy - * it instead, else the error (the frame lost). A linear XRGB8888 or + * The frame scanned out directly from @fb: 3 once its swap is submitted + * with none older in flight (its flip now), 2 once submitted otherwise (its + * flip comes later, t8132_ext_drm_flip_done()), -ENOSPC or -EOPNOTSUPP to + * copy it instead, else the error (the frame lost). A linear XRGB8888 or * ARGB8888 GEM DMA buffer, own or imported, from its first byte, at the * mode's size, the pitch a multiple of 64 bytes, at most 32 MiB, and every * 16 KiB of it one 16 KiB page (t8132_ext_fb_pages()). @@ -406,15 +410,16 @@ static int t8132_ext_present_direct(struct t8132_ext_out *out, struct apple_dcp ret = iomfb_m4_ext_present_direct(owner, &d); if (ret == -EOPNOTSUPP) t8132_ext_direct_refused(out, fb, t8132_ext_pages_refused); - return ret ?: 2; + return ret < 0 ? ret : ret ? 3 : 2; } /* * Present the primary plane's new frame: scanned out directly where it can - * be, else copied into the DCP and swapped in; 2 once a direct frame's swap - * is submitted, 1 if the commit has no new frame (no plane state), so - * nothing was tried. A copied framebuffer is read through its kernel - * mapping (dma-buf vmap for an imported one) inside CPU access. + * be, else copied into the DCP and swapped in; 2 or 3 once a direct frame's + * swap is submitted (see t8132_ext_present_direct()), 1 if the commit has no + * new frame (no plane state), so nothing was tried. A copied framebuffer is + * read through its kernel mapping (dma-buf vmap for an imported one) inside + * CPU access. */ static int t8132_ext_present(struct t8132_ext_out *out, struct drm_atomic_commit *state) { @@ -503,7 +508,7 @@ static void t8132_ext_crtc_flush(struct drm_crtc *crtc, struct drm_atomic_commit /* Its damage is lost with it: whole frames until one is shown. */ if (ret <= 0) out->frame_lost = !!ret; - if (fb && !out->first_frame && (ret <= 0 || ret == 2)) { + if (fb && !out->first_frame && (ret <= 0 || ret >= 2)) { out->first_frame = true; dev_dbg(&out->pdev->dev, "%s: first frame %ux%u %p4cc, modifier %#llx: %d\n", crtc->name, fb->width, fb->height, &fb->format->format, fb->modifier, ret); @@ -512,9 +517,10 @@ static void t8132_ext_crtc_flush(struct drm_crtc *crtc, struct drm_atomic_commit crtc->name, ret); } /* - * A copied frame's swap has completed (D590): its flip now, else a - * frame later. A direct frame's comes with its D590 (or when its swap - * is given up). + * A copied frame's swap has completed (D590), or a direct one went out + * with none older in flight (3): its flip now, else a frame later. Any + * other direct frame's (2) comes when the older swap in flight + * completes, or its own at a depth of 1 (or when that is given up). */ if (event && ret != 2) { /* Torn down: no work queued again, the event now. */ @@ -709,10 +715,12 @@ bool t8132_ext_drm_in_commit(struct drm_atomic_commit *state) /* * Every flip of these CRTCs in @state, before its old planes are cleaned up: - * a frame's flip comes with its swap's D590, or when the swap is given up - * (5 s, or at once when the link goes), so 6 s bound it. Until then the - * old framebuffer may still be scanned out: blocking commits return, and - * DRM lets go of it, only after. + * a frame's flip comes once its swap is submitted, or with the completion of + * the older swap in flight (or its own), or when that swap is given up (5 s, + * or at once when the link goes), so 6 s bound it. Blocking commits return, + * and DRM lets go of the old framebuffer, only after; a framebuffer on screen + * or in flight keeps a reference of its window's (iomfb_m4.c) until no swap + * shows or holds it. */ void t8132_ext_drm_wait_flips(struct drm_atomic_commit *state) { @@ -734,8 +742,9 @@ void t8132_ext_drm_wait_flips(struct drm_atomic_commit *state) } /* - * The swap of @dcpext's last direct frame completed (D590) or was - * given up: its held flip event now. Any context but hard IRQ. + * @dcpext's last direct frame is due its flip (the older swap in flight, or + * its own, completed or was given up): its held flip event now. Any context + * but hard IRQ. */ void t8132_ext_drm_flip_done(struct device *dcpext) { diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b0f2861382ad7..9ee4b6b515d19 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -120,6 +120,17 @@ impl Proof { .all(|&request| self.requests.covers(request)) } + /// The requested ranges this proof does not already cover. + fn uncovered(&self, requests: &Requests) -> Requests { + let mut missing = Requests::new(); + for &request in requests.as_slice() { + if !self.requests.covers(request) { + missing.insert(request); + } + } + missing + } + fn record(&mut self, epoch: Option, requests: Requests) { // Alternating in-flight buffers must not evict each other's proofs. if epoch.is_some() && self.epoch == epoch && !requests.overflow { @@ -153,12 +164,22 @@ impl Proof { let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { - if self.covers(vm.mapping_validation_epoch(), &requests) { + let epoch = vm.mapping_validation_epoch(); + if self.covers(epoch, &requests) { return optimistic; } - let (covered, epoch) = vm.covers_ranges_batch(requests.as_slice()); + // Proofs from an unchanged epoch still hold: walk only the ranges they do not + // cover. A command buffer at a fresh address then no longer re-walks every + // render target it names. + let partial = self.epoch.is_some() && self.epoch == epoch; + let walk = if partial { self.uncovered(&requests) } else { requests }; + let (mut covered, mut walked) = vm.covers_ranges_batch(walk.as_slice()); + if covered && partial && walked != self.epoch { + // A mapping changed since the cached proofs were taken: walk everything. + (covered, walked) = vm.covers_ranges_batch(requests.as_slice()); + } if covered { - self.record(epoch, requests); + self.record(walked, requests); return optimistic; } self.epoch = None; @@ -280,6 +301,40 @@ mod tests { &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); } + #[test] + fn only_uncovered_ranges_are_walked_again() { + let mut proof = Proof::new(); + let mut proven = requests((PAGE, 4 * PAGE, 1)); + proven.insert((8 * PAGE, PAGE, 3)); + proof.record(Some(5), proven); + + let mut next = requests((PAGE + 16, 8, 1)); + next.insert((8 * PAGE, 8, 2)); + next.insert((20 * PAGE, PAGE, 1)); + next.insert((4 * PAGE, 2 * PAGE, 1)); + let missing = proof.uncovered(&next); + assert_eq!(missing.as_slice(), &[(20 * PAGE, PAGE, 1), (4 * PAGE, 2 * PAGE, 1)]); + + // Stronger access than proven is walked again. + assert_eq!(proof.uncovered(&requests((PAGE, 8, 2))).count, 1); + assert_eq!(proof.uncovered(&requests((8 * PAGE, 8, 3))).count, 0); + } + + #[test] + fn a_partial_walk_extends_the_proof() { + let mut proof = Proof::new(); + proof.record(Some(5), requests((PAGE, PAGE, 1))); + let mut next = requests((PAGE, 8, 1)); + next.insert((10 * PAGE, 8, 1)); + assert!(!proof.covers(Some(5), &next)); + // validate() walks proof.uncovered(&next), then records the whole request set. + assert_eq!(proof.uncovered(&next).as_slice(), &[(10 * PAGE, PAGE, 1)]); + proof.record(Some(5), next); + assert!(proof.covers(Some(5), &next)); + assert!(proof.covers(Some(5), &requests((PAGE + 8, 8, 1)))); + assert!(!proof.covers(Some(6), &next)); + } + #[test] fn invalid_rounding_and_full_cache_disable_reuse() { for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f146b2762d812..122b42ccd1b6d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -570,7 +570,7 @@ impl gpuvm::DriverGpuVm for VmInner { Ordering::Acquire, |value| value.checked_add(1), ).map_err(|_| EOVERFLOW)? + 1; - let _mutation = self.mapping_mutation(); + let _mutation = self.mapping_addition(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); let mut left = op.length() as usize; @@ -827,6 +827,19 @@ impl VmInner { self.tlbi_contexts_naming_root(addr, size); } + /// A mutation that only maps addresses nothing maps yet: GPUVM runs the unmap and remap + /// steps for any overlap first, and those are full mutations. Every range a cached proof + /// covers stays mapped, so the epoch keeps its value and the proofs stay valid; cached + /// validation still pauses until the leaf writes finish. + fn mapping_addition(&mut self) -> MappingMutation { + let state = self.mapping_epoch.clone(); + if let Some(state) = state.as_ref() { + state.writers.fetch_add(1, Ordering::Relaxed); + state.visible.swap(u64::MAX, Ordering::AcqRel); + } + MappingMutation(state) + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index d5c6ca690476a..3967e51d472cf 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -28,9 +28,9 @@ config VIDEO_APPLE_ISP_KUNIT_TEST default KUNIT_ALL_TESTS help Builds unit tests for queue setup, P010 layouts and buffer identity, - firmware sensor geometry, the capture command stream and shared - memory retirement. The command tests stub the IPC transport and - need no hardware. Only useful for kernel developers running KUnit. + firmware sensor geometry, the capture command stream, shared memory + retirement and the per-SoC DART address filter windows. The command + tests stub the IPC transport and need no hardware. Only useful for kernel developers running KUnit. If unsure, say N. diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 004a7fa13d019..15b1ca2816fbc 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -209,6 +209,23 @@ static int isp_check_presets(struct apple_isp *isp, u32 num_fw_presets) if (!num_fw_presets) return 0; + /* + * A deferred start detects the camera after the presets were + * advertised: keep them, and refuse a stream on one the firmware + * lacks instead. + */ + if (isp->hw->deferred_start) { + isp->num_fw_presets = num_fw_presets; + for (int i = 0; i < isp->num_presets; i++) + if (isp->presets[i].index >= num_fw_presets) + dev_warn(isp->dev, + "%ux%u preset: config %u, firmware has %u\n", + isp->presets[i].output_dim.x, + isp->presets[i].output_dim.y, + isp->presets[i].index, num_fw_presets); + return 0; + } + for (int i = 0; i < isp->num_presets; i++) { struct isp_preset *preset = &isp->presets[i]; @@ -312,8 +329,10 @@ static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) preset->input_dim.x == config->in_width && preset->input_dim.y == config->in_height && !preset->crop_offset.x && !preset->crop_offset.y && - preset->crop_size.x == preset->output_dim.x && - preset->crop_size.y == preset->output_dim.y) + /* Same aspect ratio, never enlarged: the scaler may shrink it. */ + preset->crop_size.x >= preset->output_dim.x && + preset->crop_size.x * preset->output_dim.y == + preset->crop_size.y * preset->output_dim.x) preset->p010 = preview || reported; } if (!cached->usable) @@ -558,8 +577,12 @@ int apple_isp_detect_camera(struct apple_isp *isp) dev_err(isp->dev, "failed to boot firmware for initial sensor detection: %d\n", err); - /* Resident firmware would fail a deferred retry the same way. */ - return isp->hw->resident_fw ? err : -EPROBE_DEFER; + /* + * Resident firmware would fail a deferred retry the same way, + * and a deferred start is not a probe to defer. + */ + return isp->hw->resident_fw || isp->hw->deferred_start ? + err : -EPROBE_DEFER; } err = isp_detect_camera(isp); @@ -568,6 +591,8 @@ int apple_isp_detect_camera(struct apple_isp *isp) if (isp->hw->resident_fw) { if (err) apple_isp_firmware_halt(isp); + else + isp->camera_detected = true; return err; } @@ -578,6 +603,8 @@ int apple_isp_detect_camera(struct apple_isp *isp) apple_isp_firmware_shutdown(isp); + if (!err) + isp->camera_detected = true; return err; } @@ -955,14 +982,34 @@ static int isp_configure_capture(struct apple_isp *isp) int apple_isp_start_camera(struct apple_isp *isp) { + struct isp_format *fmt = isp_get_current_format(isp); int err; + /* + * A deferred start detects the camera now. Resident firmware is then + * left running; firmware that starts again is stopped after the + * detection like at probe, and booted again here. + */ + if (isp->hw->deferred_start && !isp->camera_detected) { + err = apple_isp_detect_camera(isp); + if (err < 0) { + dev_err(isp->dev, "failed to detect the camera: %d\n", err); + return err; + } + } err = apple_isp_firmware_boot(isp); if (err < 0) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); return err; } + if (isp->num_fw_presets && fmt->preset->index >= isp->num_fw_presets) { + dev_err(isp->dev, "preset config %u: firmware has %u\n", + fmt->preset->index, isp->num_fw_presets); + apple_isp_firmware_shutdown(isp); + return -EINVAL; + } + err = isp_configure_capture(isp); if (err) { dev_err(isp->dev, "failed to configure capture: %d\n", err); diff --git a/drivers/media/platform/apple/isp/isp-drv-test.c b/drivers/media/platform/apple/isp/isp-drv-test.c new file mode 100644 index 0000000000000..dbddb079a0f57 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv-test.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the per-SoC data of the Apple ISP driver, built into + * isp-drv.c: the DART address filter windows and the firmware start. They + * need no hardware. + */ + +#include +#include +#include + +/* Every address filter window must lie inside a window the firmware is mapped. */ +static void isp_test_dapf_in_fw_mmio(struct kunit *test) +{ + const struct of_device_id *match; + + for (match = apple_isp_of_match; match->compatible[0]; match++) { + const struct apple_isp_hw *hw = match->data; + + for (unsigned int i = 0; i < hw->num_dapf; i++) { + const struct isp_dapf_window *w = &hw->dapf[i]; + bool inside = false; + + KUNIT_EXPECT_LE_MSG(test, w->start, w->end, "%s window %u", + match->compatible, i); + for (unsigned int j = 0; j < hw->num_fw_mmio; j++) { + const struct isp_mmio_window *m = &hw->fw_mmio[j]; + + if (w->start >= m->base && w->end < m->base + m->size) + inside = true; + } + KUNIT_EXPECT_TRUE_MSG(test, inside, + "%s window %u 0x%llx-0x%llx is not in fw_mmio", + match->compatible, i, w->start, w->end); + } + } +} + +#define ISP_TEST_DAPF_FILL 0xa5 + +/* + * Each window fills its own 0x40-byte slot with exactly these words, and + * leaves every other byte alone. The bank is plain memory here: the driver + * is arm64 only, where the MMIO accessors work on it unchanged. + */ +static void isp_test_dapf_program(struct kunit *test) +{ + static const struct isp_dapf_window windows[] = { + { 0x380704000, 0x380704063 }, + { 0x382824000, 0x382824004 }, + }; + static const struct apple_isp_hw hw = { + .dapf = windows, + .num_dapf = ARRAY_SIZE(windows), + }; + size_t size = (ARRAY_SIZE(windows) + 1) * DAPF_WINDOW_STRIDE; + u8 expect[DAPF_WINDOW_STRIDE]; + struct apple_isp *isp; + u8 *bank; + + isp = kunit_kzalloc(test, sizeof(*isp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp); + isp->dev = kunit_device_register(test, "apple-isp-dapf-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp->dev); + bank = kunit_kmalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bank); + memset(bank, ISP_TEST_DAPF_FILL, size); + isp->hw = &hw; + isp->dapf = (__force void __iomem *)bank; + + KUNIT_ASSERT_EQ(test, apple_isp_program_dapf(isp), 0); + + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + memset(expect, ISP_TEST_DAPF_FILL, sizeof(expect)); + put_unaligned_le32(0x31, &expect[0x00]); + put_unaligned_le32(0, &expect[0x04]); + put_unaligned_le64(windows[i].start, &expect[0x08]); + put_unaligned_le64(windows[i].end, &expect[0x10]); + put_unaligned_le32(1, &expect[0x20]); + KUNIT_EXPECT_MEMEQ_MSG(test, bank + i * DAPF_WINDOW_STRIDE, expect, + sizeof(expect), "window %u", i); + } + + /* The slot after the last window is untouched. */ + KUNIT_EXPECT_NULL(test, memchr_inv(bank + ARRAY_SIZE(windows) * DAPF_WINDOW_STRIDE, + ISP_TEST_DAPF_FILL, DAPF_WINDOW_STRIDE)); +} + +static struct kunit_case isp_dapf_test_cases[] = { + KUNIT_CASE(isp_test_dapf_in_fw_mmio), + KUNIT_CASE(isp_test_dapf_program), + {} +}; + +static struct kunit_suite isp_dapf_test_suite = { + .name = "apple-isp-dapf", + .test_cases = isp_dapf_test_cases, +}; + +kunit_test_suite(isp_dapf_test_suite); + +/* + * A deferred start detects the camera once: its firmware must keep running + * or start again. Firmware that starts again is not resident, and only it + * is quiesced before the power-down. + */ +static void isp_test_firmware_lifetime(struct kunit *test) +{ + const struct of_device_id *match; + + for (match = apple_isp_of_match; match->compatible[0]; match++) { + const struct apple_isp_hw *hw = match->data; + + if (hw->deferred_start) + KUNIT_EXPECT_TRUE_MSG(test, hw->resident_fw || hw->fw_restore, + "%s", match->compatible); + KUNIT_EXPECT_FALSE_MSG(test, hw->resident_fw && hw->fw_restore, "%s", + match->compatible); + if (hw->coproc_quiesce) + KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore, "%s", + match->compatible); + } +} + +static struct kunit_case isp_hw_test_cases[] = { + KUNIT_CASE(isp_test_firmware_lifetime), + {} +}; + +static struct kunit_suite isp_hw_test_suite = { + .name = "apple-isp-hw", + .test_cases = isp_hw_test_cases, +}; + +kunit_test_suite(isp_hw_test_suite); diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 93a21e901ff6a..a2390964a13d8 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -5,10 +5,12 @@ * Copyright (C) 2023 The Asahi Linux Contributors */ +#include #include #include #include #include +#include #include #include #include @@ -20,6 +22,41 @@ #include "isp-regs.h" #include "isp-v4l2.h" +static void apple_isp_kvfree(void *data) +{ + kvfree(data); +} + +/* + * Keep a copy of the firmware data segment as the boot loader loaded it + * (hw->fw_restore). This must run before the firmware first starts. + */ +static int apple_isp_copy_fw_data(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + struct resource res; + int err; + + err = of_reserved_mem_region_to_resource_byname(dev->of_node, "data", &res); + if (err) + return err; + + isp->fw_data_size = resource_size(&res); + isp->fw_data = devm_memremap(dev, res.start, isp->fw_data_size, MEMREMAP_WB); + if (IS_ERR(isp->fw_data)) + return PTR_ERR(isp->fw_data); + + isp->fw_data_copy = kvmalloc(isp->fw_data_size, GFP_KERNEL); + if (!isp->fw_data_copy) + return -ENOMEM; + err = devm_add_action_or_reset(dev, apple_isp_kvfree, isp->fw_data_copy); + if (err) + return err; + + memcpy(isp->fw_data_copy, isp->fw_data, isp->fw_data_size); + return 0; +} + static void apple_isp_detach_genpd(struct apple_isp *isp) { if (isp->pd_count <= 1) @@ -175,6 +212,78 @@ static int isp_of_region_iova(struct device *dev, const char *name, return 0; } +/* + * The firmware reaches its fixed MMIO windows through the first DART, whose + * address filter the ADT describes for them. Its registers are on that DART. + */ +static int apple_isp_map_dapf(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + /* The first iommus entry, whatever its #iommu-cells */ + struct device_node *dart __free(device_node) = + of_parse_phandle(dev->of_node, "iommus", 0); + struct resource res; + int idx, err; + + if (!dart) + return dev_err_probe(dev, -ENODEV, "no ISP DART\n"); + + /* DAPF_* describe the T8110 DART's address filter */ + if (!of_device_is_compatible(dart, "apple,t8110-dart")) + return dev_err_probe(dev, -ENODEV, + "%pOF has no known address filter\n", dart); + + idx = of_property_match_string(dart, "reg-names", "dapf"); + if (idx < 0) + return dev_err_probe(dev, idx, + "%pOF has no address filter registers\n", dart); + + err = of_address_to_resource(dart, idx, &res); + if (err) + return dev_err_probe(dev, err, "%pOF: bad address filter registers\n", + dart); + + if (resource_size(&res) < isp->hw->num_dapf * DAPF_WINDOW_STRIDE) + return dev_err_probe(dev, -EINVAL, "%pOF: address filter too small\n", + dart); + + isp->dapf = devm_ioremap_resource(dev, &res); + return PTR_ERR_OR_ZERO(isp->dapf); +} + +/* Program the address filter windows, then check that they took. */ +int apple_isp_program_dapf(struct apple_isp *isp) +{ + const struct apple_isp_hw *hw = isp->hw; + void __iomem *w; + unsigned int i; + + for (i = 0; i < hw->num_dapf; i++) { + w = isp->dapf + i * DAPF_WINDOW_STRIDE; + writel(0, w + DAPF_CTRL1); + writeq(hw->dapf[i].start, w + DAPF_START); + writeq(hw->dapf[i].end, w + DAPF_END); + writel(DAPF_CTRL0_ISP, w + DAPF_CTRL0); + writel(DAPF_CTRL2_ISP, w + DAPF_CTRL2); + } + + /* Only bits 7:0 of CTRL0 and bits 63:2 of END read back. */ + for (i = 0; i < hw->num_dapf; i++) { + w = isp->dapf + i * DAPF_WINDOW_STRIDE; + if (readl(w + DAPF_CTRL1) || + readq(w + DAPF_START) != hw->dapf[i].start || + (readq(w + DAPF_END) | 3) != (hw->dapf[i].end | 3) || + (readl(w + DAPF_CTRL0) & 0xff) != DAPF_CTRL0_ISP || + readl(w + DAPF_CTRL2) != DAPF_CTRL2_ISP) { + dev_err(isp->dev, "DAPF window %u (0x%llx-0x%llx) failed readback\n", + i, hw->dapf[i].start, hw->dapf[i].end); + return -EIO; + } + } + + return 0; +} + static int apple_isp_init_iommu(struct apple_isp *isp) { struct device *dev = isp->dev; @@ -509,6 +618,12 @@ static int apple_isp_probe(struct platform_device *pdev) } } + if (isp->hw->num_dapf) { + err = apple_isp_map_dapf(isp); + if (err) + goto detach_genpd; + } + /* Capture watchdog, where the ISP gates its frames on one */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wdt"); if (res) { @@ -558,12 +673,23 @@ static int apple_isp_probe(struct platform_device *pdev) goto free_iommu; } + if (isp->hw->fw_restore) { + err = apple_isp_copy_fw_data(isp); + if (err) { + dev_err(dev, "failed to copy the firmware data: %d\n", err); + apple_isp_free_firmware_surface(isp); + goto free_iommu; + } + } + pm_runtime_enable(dev); - err = apple_isp_detect_camera(isp); - if (err) { - dev_err(dev, "failed to detect camera: %d\n", err); - goto free_surface; + if (!isp->hw->deferred_start) { + err = apple_isp_detect_camera(isp); + if (err) { + dev_err(dev, "failed to detect camera: %d\n", err); + goto free_surface; + } } err = apple_isp_setup_video(isp); @@ -889,6 +1015,74 @@ static const struct apple_isp_hw apple_isp_hw_t6040 = { .meta_exposure = true, }; +/* T8132: the firmware's MMIO windows, from the J713 ADT's DART address filter */ +static const struct isp_mmio_window apple_isp_fw_mmio_t8132[] = { + { 0x220004000, 0x1d4000 }, /* DSID broadcast-clear windows */ + { 0x222004000, 0x1d4000 }, + { 0x3803c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x380704000, 0x4000 }, /* PMGR, ISP power states */ + { 0x380730000, 0x4000 }, + { 0x380e3c000, 0x4000 }, + { 0x382824000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x39062c000, 0x20000 }, + { 0x501660000, 0x4000 }, +}; + +/* The ADT's DART address filter windows, in its order and with its ends */ +static const struct isp_dapf_window apple_isp_dapf_t8132[] = { + { 0x380704000, 0x380704063 }, /* PMGR, ISP power states */ + { 0x380730000, 0x380733fff }, + { 0x380e3c000, 0x380e3c003 }, + { 0x39062c000, 0x390648003 }, + { 0x501660000, 0x501660003 }, + { 0x220004000, 0x2200162fc }, /* DSID broadcast-clear windows */ + { 0x220044000, 0x2200562fc }, + { 0x2201c4000, 0x2201d62fc }, + { 0x222004000, 0x2220162fc }, + { 0x222044000, 0x2220562fc }, + { 0x2221c4000, 0x2221d62fc }, + { 0x3803c0000, 0x3803e4000 }, /* PMGR, with the PMP clock scratch */ + { 0x382824000, 0x382824004 }, /* PMP bandwidth scratch */ +}; + +/* + * T8132 runs T6040's coprocessor and firmware interface, booted the same + * way. Its two AMCCs (ADT mcc amcc_aperture_count) give two broadcast-clear + * windows, passed one per AMCC without padding slots. No PMP_CTRL_SET is + * sent: T8132's PMP scratch offsets are not established. Its firmware + * starts again from its data segment as loaded once the coprocessor was + * quiesced before the power-down, so the ISP is powered only while the + * camera is in use. + */ +static const struct apple_isp_hw apple_isp_hw_t8132 = { + .gen = ISP_GEN_T6040, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 2, + .dsid_clr_base0 = 0x2201d4000, + .dsid_clr_base1 = 0x2221d4000, + .dsid_clr_range0 = 0x2fc, + + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + .scl1 = true, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8140, + .fw_mmio = apple_isp_fw_mmio_t8132, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8132), + .dapf = apple_isp_dapf_t8132, + .num_dapf = ARRAY_SIZE(apple_isp_dapf_t8132), + + .coproc_control = ISP_COPROC_CONTROL_T8140, + .coproc_status = ISP_COPROC_STATUS_T8140, + .coproc_quiesce = ISP_COPROC_QUIESCE_T8140, + .boot_mode = 1, + .shared_size = 0xdec00000, + .fw_restore = true, + .deferred_start = true, +}; + static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, @@ -899,6 +1093,7 @@ static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t6031-isp", .data = &apple_isp_hw_t6031 }, { .compatible = "apple,t8140-isp", .data = &apple_isp_hw_t8140 }, { .compatible = "apple,t6041-isp", .data = &apple_isp_hw_t6040 }, + { .compatible = "apple,t8132-isp", .data = &apple_isp_hw_t8132 }, {}, }; MODULE_DEVICE_TABLE(of, apple_isp_of_match); @@ -939,8 +1134,8 @@ static __maybe_unused int apple_isp_freeze(struct device *dev) /* * Hibernation hands the system from one kernel to another, and * firmware left running would go on using the memory of the kernel - * that started it. Stop it cleanly instead; the camera is then - * unavailable until the next system boot. + * that started it. Stop it cleanly instead; if it ran, the camera is + * then unavailable until the next system boot. */ if (isp->hw->resident_fw) { mutex_lock(&isp->video_lock); @@ -972,6 +1167,23 @@ static const struct dev_pm_ops apple_isp_pm_ops = { RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) }; +/* + * Stop firmware that starts again (hw->fw_restore) before a reboot or a + * kexec, so that the next kernel finds its data segment as loaded. + */ +static void apple_isp_shutdown(struct platform_device *pdev) +{ + struct apple_isp *isp = platform_get_drvdata(pdev); + + if (!isp || !isp->hw->fw_restore) + return; + /* A stream that still runs is stopped cleanly first, as for hibernation */ + apple_isp_video_suspend(isp); + mutex_lock(&isp->video_lock); + apple_isp_firmware_halt(isp); + mutex_unlock(&isp->video_lock); +} + static struct platform_driver apple_isp_driver = { .driver = { .name = "apple-isp", @@ -980,9 +1192,14 @@ static struct platform_driver apple_isp_driver = { }, .probe = apple_isp_probe, .remove = apple_isp_remove, + .shutdown = apple_isp_shutdown, }; module_platform_driver(apple_isp_driver); MODULE_AUTHOR("Eileen Yoon "); MODULE_DESCRIPTION("Apple ISP driver"); MODULE_LICENSE("GPL v2"); + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include "isp-drv-test.c" +#endif diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 30425f6cf483d..738590fec3f35 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -137,6 +137,15 @@ struct isp_mmio_window { u64 size; }; +/* + * A DART address filter window. Both bounds are inclusive and word + * granular, as the hardware takes them. + */ +struct isp_dapf_window { + u64 start; + u64 end; +}; + struct isp_preset { u32 index; struct coord input_dim; @@ -187,11 +196,27 @@ struct apple_isp_hw { bool mbox_irq_route; /* coprocessor control register, 0 for ISP_COPROC_CONTROL */ u32 coproc_control; + /* coprocessor status register, 0 for ISP_COPROC_STATUS */ + u32 coproc_status; + /* + * coprocessor quiesce register, 0 for none. The CPU core domains of + * a coprocessor that stopped only power down cleanly once it has + * quiesced; otherwise they stay busy and never run again. + */ + u32 coproc_quiesce; /* windows mapped 1:1 for the firmware */ const struct isp_mmio_window *fw_mmio; unsigned int num_fw_mmio; + /* + * The windows the ADT gives the first DART's address filter, inside + * the fw_mmio windows. The filter loses them with ISP_SYS, so they + * are programmed after every power-up. + */ + const struct isp_dapf_window *dapf; + unsigned int num_dapf; + /* ISP_GPIO_6 boot mode */ u32 boot_mode; @@ -205,10 +230,27 @@ struct apple_isp_hw { bool meta_exposure; /* - * The firmware can be started only once per system boot, so it is - * started at probe and kept running while the driver is bound. + * The firmware leaves its data segment changed when it stops, and + * does not start from it again. Keep a copy of the segment as the + * boot loader loaded it, taken at probe before the firmware first + * runs, and put it back whenever the firmware has stopped. + */ + bool fw_restore; + + /* + * The firmware can be started only once per system boot, so once + * started it is kept running while the driver is bound. It starts at + * probe, or at the first stream with deferred_start. */ bool resident_fw; + + /* + * Start the firmware, and detect the camera, at the first stream + * instead of at probe, so the ISP stays off until then. The firmware + * must be resident or start again (fw_restore): the camera is + * detected only once. + */ + bool deferred_start; }; enum isp_sensor_id { @@ -330,6 +372,7 @@ struct apple_isp { void __iomem *gpio; void __iomem *mbox2; void __iomem *wdt; + void __iomem *dapf; struct hrtimer wdt_timer; bool wdt_running; @@ -347,11 +390,19 @@ struct apple_isp { } fw; /* changes under video_lock once the video device is registered */ enum isp_fw_state fw_state; + /* the camera was detected; until then a deferred start is pending */ + bool camera_detected; + /* configurations the firmware has, once a deferred start detected them */ + u32 num_fw_presets; struct isp_surf *ipc_surf; struct isp_surf *extra_surf; struct isp_surf *data_surf; struct isp_surf *main_tail_surf; + /* With hw->fw_restore: the firmware data segment and its copy */ + void *fw_data; + void *fw_data_copy; + size_t fw_data_size; struct isp_surf *log_surf; struct isp_surf *bt_surf; struct isp_surf *meta_surfs[ISP_MAX_BUFFERS]; @@ -424,6 +475,8 @@ enum { ISP_STATE_SLEEPING, }; +int apple_isp_program_dapf(struct apple_isp *isp); + #define isp_dbg(isp, fmt, ...) \ dev_dbg((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) @@ -443,7 +496,8 @@ enum { static inline bool isp_supports_p010(struct apple_isp *isp) { - return isp->hw->gen == ISP_GEN_T6040 && + /* Its config discovery would change formats already advertised. */ + return isp->hw->gen == ISP_GEN_T6040 && !isp->hw->deferred_start && isp_get_current_format(isp)->version == 0x958; } diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 22dbf99a4c3ec..28727ef24fbb6 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -53,6 +54,11 @@ static inline u32 isp_coproc_control(struct apple_isp *isp) return isp->hw->coproc_control ?: ISP_COPROC_CONTROL; } +static inline u32 isp_coproc_status(struct apple_isp *isp) +{ + return isp->hw->coproc_status ?: ISP_COPROC_STATUS; +} + /* Wait for the firmware to write @expected to a GPIO word. */ static int isp_gpio_wait(struct apple_isp *isp, u32 reg, u32 expected) { @@ -391,10 +397,47 @@ static int isp_reset_coproc(struct apple_isp *isp) return 0; } +/* + * With hw->fw_restore, put the firmware memory back as the boot loader left + * it: the data segment from its copy, and the main memory above it cleared. + */ +static void isp_restore_fw_memory(struct apple_isp *isp) +{ + struct isp_surf *tail = isp->main_tail_surf; + + if (!isp->fw_data_copy) + return; + + memcpy(isp->fw_data, isp->fw_data_copy, isp->fw_data_size); + if (tail) + for (u32 i = 0; i < tail->num_pages; i++) + clear_highpage(tail->pages[i]); +} + static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { + /* Only a coprocessor that was started has to settle and quiesce */ + bool started = isp->hw->coproc_quiesce && + isp_coproc_read32(isp, isp_coproc_control(isp)); + u32 val; + + /* A coprocessor that powered down waits in WFI */ + if (started && + readl_poll_timeout(isp->coproc + isp_coproc_status(isp), val, + val & ISP_COPROC_IN_WFI, 100, 200 * USEC_PER_MSEC)) + dev_warn(isp->dev, "coproc not in WFI before power-down (status 0x%x)\n", + val); + isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); + if (started) { + /* The request bit alone asks for it, and reads back once done */ + isp_coproc_write32(isp, isp->hw->coproc_quiesce, ISP_COPROC_QUIESCED); + if (readl_poll_timeout(isp->coproc + isp->hw->coproc_quiesce, val, + val & ISP_COPROC_QUIESCED, 10, 50 * USEC_PER_MSEC)) + dev_warn(isp->dev, "coproc did not quiesce (0x%x)\n", val); + } + apple_isp_power_down_domains(isp); } @@ -403,10 +446,18 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) int err; // u32 val; + isp_restore_fw_memory(isp); + err = apple_isp_power_up_domains(isp); if (err < 0) return err; + if (isp->dapf) { + err = apple_isp_program_dapf(isp); + if (err) + goto shutdown; + } + /* ISP17a and T6040 have no clock enable word */ if (isp->hw->gen != ISP_GEN_T8140 && isp->hw->gen != ISP_GEN_T6040) isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); @@ -837,26 +888,26 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) static int isp_stop_command_processor(struct apple_isp *isp) { -#if 0 - int res = isp_cmd_stop(isp, 0); - if (res) { - isp_err(isp, "isp_cmd_stop() failed\n"); - return res; - } + int res, ret; - /* Wait for ISP_GPIO_0 to 0xf7fbdff9 -> 0x8042006 */ isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); - isp_cmd_power_down(isp); -#else - isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); - - int res = isp_cmd_suspend(isp); + /* + * Firmware that starts again (hw->fw_restore) is stopped for good: + * STOP also powers the camera module down, which SUSPEND leaves on, + * even across a reboot. POWER_DOWN follows even if STOP failed. + */ + if (isp->hw->fw_restore) { + res = isp_cmd_stop(isp, 0); + ret = isp_cmd_power_down(isp); + res = res ?: ret; + } else { + res = isp_cmd_suspend(isp); + } if (res) { - isp_err(isp, "isp_cmd_suspend() failed\n"); + isp_err(isp, "failed to stop the firmware: %d\n", res); return res; } -#endif if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { isp_err(isp, "never received magic number from firmware\n"); @@ -971,6 +1022,9 @@ static void isp_firmware_stop(struct apple_isp *isp) isp_firmware_shutdown_stage3(isp); isp_firmware_shutdown_stage2(isp); isp_collect_gc_surface(isp); + + /* Also leave it as loaded for a rebound driver or another kernel. */ + isp_restore_fw_memory(isp); } static int isp_firmware_boot(struct apple_isp *isp) @@ -982,6 +1036,8 @@ static int isp_firmware_boot(struct apple_isp *isp) /* Stage 1 stops the coprocessor and powers it down itself. */ isp_err(isp, "failed firmware boot stage 1: %d\n", err); isp_collect_gc_surface(isp); + /* The firmware may have run: leave its memory as loaded. */ + isp_restore_fw_memory(isp); return err; } @@ -1039,7 +1095,8 @@ static void isp_firmware_shutdown(struct apple_isp *isp) * Resident firmware (hw->resident_fw) is loaded by the bootloader and * does not start again once it has been stopped: after CISP_CMD_SUSPEND, * a coprocessor reset and a power cycle of its domains it never completes - * the first handshake. It is booted once, at probe, and runs until the + * the first handshake. It is booted once, at probe or with + * deferred_start at the first stream, and runs until the * driver is unbound, probe fails or the system hibernates, also across * suspend; after it has stopped the device stays unusable until the next * system boot. @@ -1085,7 +1142,7 @@ int apple_isp_firmware_boot(struct apple_isp *isp) return 0; } -/* The end of a stream or of the camera detection at probe */ +/* The end of a stream or of the camera detection */ void apple_isp_firmware_shutdown(struct apple_isp *isp) { if (!isp->hw->resident_fw) { diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h index f2f6f1c9489f4..b7f9b2fb87faa 100644 --- a/drivers/media/platform/apple/isp/isp-regs.h +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -24,6 +24,10 @@ #define ISP_COPROC_IRQ_MASK_5 0x1400a14 #define ISP_COPROC_CONTROL_T8140 0x1600044 +#define ISP_COPROC_STATUS_T8140 0x1600048 +#define ISP_COPROC_QUIESCE_T8140 0x1600040 +/* Requests the quiesce when set, and reads back once it has happened */ +#define ISP_COPROC_QUIESCED BIT(25) /* * ISP17a (T8140) reset. The fourth fabric word is at 0xa50, off the 0x100 @@ -40,6 +44,20 @@ #define ISP_COPROC_RESET_ACK_0_T8140 0x1600818 #define ISP_COPROC_RESET_ACK_1_T8140 0x160081c +/* + * Address filter (DAPF) registers of a T8110-type DART: an array of windows, + * each with its inclusive start and end and three control words. Every ISP + * window takes the control values its ADT dapf-instance entry gives. + */ +#define DAPF_WINDOW_STRIDE 0x40 +#define DAPF_CTRL0 0x00 +#define DAPF_CTRL0_ISP 0x31 +#define DAPF_CTRL1 0x04 +#define DAPF_START 0x08 +#define DAPF_END 0x10 +#define DAPF_CTRL2 0x20 +#define DAPF_CTRL2_ISP 0x1 + #define ISP_MBOX_IRQ_INTERRUPT 0x00 #define ISP_MBOX_IRQ_ENABLE 0x04 #define ISP_MBOX_IRQ_ENABLE_T6031 0x08 diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 24506ed526a34..81964ab1916c9 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -531,11 +531,12 @@ static int apple_isp_start_streaming(struct apple_isp *isp) } /* - * Resident firmware was not restarted, but the channel's pools are - * configured again for every stream. Metadata buffers still marked - * as submitted to the previous pools would never reach the new ones. + * The channel's pools are configured again for every stream, by + * resident firmware that kept running or by firmware that started + * afresh. Metadata buffers still marked as submitted to the previous + * pools, or to firmware that stopped, would never reach the new ones. */ - if (isp->hw->resident_fw) { + if (isp->hw->resident_fw || isp->hw->fw_restore) { spin_lock_irqsave(&isp->buf_lock, flags); for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) isp->meta_surfs[i]->submitted = false; diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c index f615a5ab54516..0e01e787699ba 100644 --- a/drivers/nvmem/apple-spmi-nvmem.c +++ b/drivers/nvmem/apple-spmi-nvmem.c @@ -60,7 +60,8 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev) if (IS_ERR(provider)) return PTR_ERR(provider); if (!of_device_is_compatible(sdev->dev.of_node, "apple,baku-pmic") && - !of_device_is_compatible(sdev->dev.of_node, "apple,abbey-pmic")) + !of_device_is_compatible(sdev->dev.of_node, "apple,abbey-pmic") && + !of_device_is_compatible(sdev->dev.of_node, "apple,chapel-pmic")) return 0; return devm_of_platform_populate(&sdev->dev); diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index c177665e17a71..52fe2669e6fe8 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -74,8 +74,9 @@ config APPLE_PMC shared rails, such as the DRAM controller and the SoC fabric, between voter agents. This driver enables the static agents and provides power domains that enable a dynamic agent, such as the - GPU, while its device is powered on. It can also limit the DRAM - controller rail as a thermal cooling device. + GPU, or a display pipe's bandwidth and boost interfaces, while + its device is powered on. It can also limit the DRAM controller + rail as a thermal cooling device. config APPLE_PMC_KUNIT_TEST bool "KUnit tests for the Apple PMC voter interface" if !KUNIT_ALL_TESTS diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index 7cf8a43e8b085..d563a178dc0e8 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -8,6 +8,7 @@ #include #include +#include #define APPLE_PMC_TEST_RAILS_SIZE 0x4000 @@ -281,6 +282,209 @@ static void apple_pmc_test_dcs_floor(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_pmc_test_ap_floor(pmc) & ~APPLE_PMC_RAIL_MASK(hw->dcs_rail), 0U); } +/* + * A T8132 interface domain sets and clears its own BWR and boost enables + * only: the internal display's boost enable, bit 0, stays set. + */ +static void apple_pmc_test_interface(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain dom = { + .pmc = pmc, .interface = true, .bwr_bit = 3, .boost_bit = 1, + }; + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + + pmc->hw = hw; + writel(BIT(0), pmc->rails + hw->boost_enable); + + KUNIT_ASSERT_EQ(test, apple_pmc_domain_power_on(&dom.genpd), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), BIT(3)); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); + KUNIT_ASSERT_EQ(test, apple_pmc_domain_power_off(&dom.genpd), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0)); +} + +struct apple_pmc_test_stuck { + u32 offset; + u32 value; +}; + +/* The interface enable word at stuck->offset keeps stuck->value. */ +static u32 apple_pmc_test_stuck_read(struct apple_pmc *pmc, u32 offset) +{ + struct kunit *test = kunit_get_current_test(); + struct apple_pmc_test_stuck *stuck = test->priv; + + if (offset == stuck->offset) + writel(stuck->value, pmc->rails + offset); + return readl(pmc->rails + offset); +} + +/* + * If the BWR enable does not change, the boost enable goes back too, both + * when the domain powers on and when it powers off. + */ +static void apple_pmc_test_interface_stuck(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain dom = { + .pmc = pmc, .interface = true, .bwr_bit = 3, .boost_bit = 1, + }; + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + struct apple_pmc_test_stuck stuck = { .offset = hw->bwr_enable }; + + pmc->hw = hw; + writel(BIT(0), pmc->rails + hw->boost_enable); + test->priv = &stuck; + kunit_activate_static_stub(test, apple_pmc_interface_read, + apple_pmc_test_stuck_read); + + KUNIT_EXPECT_EQ(test, apple_pmc_domain_power_on(&dom.genpd), -EIO); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->boost_enable), BIT(0)); + + stuck.value = BIT(3); + writel(BIT(0) | BIT(1), pmc->rails + hw->boost_enable); + KUNIT_EXPECT_EQ(test, apple_pmc_domain_power_off(&dom.genpd), -EIO); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->bwr_enable), BIT(3)); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->boost_enable), + BIT(0) | BIT(1)); + + kunit_deactivate_static_stub(test, apple_pmc_interface_read); + test->priv = NULL; +} + +static void apple_pmc_test_genpd_remove(void *genpd) +{ + pm_genpd_remove(genpd); +} + +static void apple_pmc_test_genpd_detach(void *dev) +{ + pm_runtime_disable(dev); + pm_genpd_remove_device(dev); +} + +/* A genpd that is removed again when the test ends, children first. */ +static struct generic_pm_domain *apple_pmc_test_genpd(struct kunit *test, + struct generic_pm_domain *genpd, + int ret) +{ + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + apple_pmc_test_genpd_remove, + genpd), 0); + return genpd; +} + +static struct apple_pmc_domain *apple_pmc_test_interface_dom(struct kunit *test, + struct apple_pmc *pmc, + const char *name) +{ + struct apple_pmc_domain *dom = kunit_kzalloc(test, sizeof(*dom), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dom); + dom->pmc = pmc; + dom->interface = true; + dom->bwr_bit = 3; + dom->boost_bit = 1; + dom->genpd.name = name; + apple_pmc_test_genpd(test, &dom->genpd, apple_pmc_interface_init(pmc, dom)); + return dom; +} + +/* + * Link the interface domain as the PMGR driver does: under a SYS power + * state, with an FE power state below both, all of them IRQ safe. Powering + * FE through a device sets the enables after SYS is on, and powering it + * off clears them. + */ +static void apple_pmc_test_interface_genpd(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + struct generic_pm_domain *sys, *fe; + struct apple_pmc_domain *dom; + struct device *dev; + + pmc->hw = hw; + writel(BIT(0), pmc->rails + hw->boost_enable); + pmc->saved_boost = BIT(0); + dev = kunit_device_register(test, "apple-pmc-test-fe"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + sys = kunit_kzalloc(test, sizeof(*sys), GFP_KERNEL); + fe = kunit_kzalloc(test, sizeof(*fe), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sys); + KUNIT_ASSERT_NOT_NULL(test, fe); + sys->name = "apple-pmc-test-sys"; + sys->flags = GENPD_FLAG_IRQ_SAFE; + fe->name = "apple-pmc-test-fe"; + fe->flags = GENPD_FLAG_IRQ_SAFE; + apple_pmc_test_genpd(test, sys, pm_genpd_init(sys, NULL, false)); + dom = apple_pmc_test_interface_dom(test, pmc, "apple-pmc-test-interface"); + KUNIT_EXPECT_EQ(test, dom->genpd.status, GENPD_STATE_OFF); + apple_pmc_test_genpd(test, fe, pm_genpd_init(fe, NULL, true)); + + KUNIT_ASSERT_EQ(test, pm_genpd_add_subdomain(sys, &dom->genpd), 0); + KUNIT_ASSERT_EQ(test, pm_genpd_add_subdomain(sys, fe), 0); + KUNIT_ASSERT_EQ(test, pm_genpd_add_subdomain(&dom->genpd, fe), 0); + + KUNIT_ASSERT_EQ(test, pm_genpd_add_device(fe, dev), 0); + pm_runtime_enable(dev); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + apple_pmc_test_genpd_detach, + dev), 0); + + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(dev), 0); + KUNIT_EXPECT_EQ(test, dom->genpd.status, GENPD_STATE_ON); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), BIT(3)); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); + + KUNIT_EXPECT_EQ(test, pm_runtime_put_sync_suspend(dev), 0); + KUNIT_EXPECT_EQ(test, dom->genpd.status, GENPD_STATE_OFF); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0)); +} + +/* + * A pipe with both enables set at boot starts on. One with only its boost + * enable set starts off, and a power on that fails on the BWR enable keeps + * that boost enable instead of clearing it. + */ +static void apple_pmc_test_interface_boot_state(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + struct apple_pmc_test_stuck stuck = { .offset = hw->bwr_enable }; + struct apple_pmc_domain *dom; + + pmc->hw = hw; + pmc->saved_bwr = BIT(3); + pmc->saved_boost = BIT(0) | BIT(1); + dom = apple_pmc_test_interface_dom(test, pmc, "apple-pmc-test-on"); + KUNIT_EXPECT_EQ(test, dom->genpd.status, GENPD_STATE_ON); + + pmc->saved_bwr = 0; + writel(BIT(0) | BIT(1), pmc->rails + hw->boost_enable); + dom = apple_pmc_test_interface_dom(test, pmc, "apple-pmc-test-mixed"); + KUNIT_EXPECT_EQ(test, dom->genpd.status, GENPD_STATE_OFF); + + test->priv = &stuck; + kunit_activate_static_stub(test, apple_pmc_interface_read, + apple_pmc_test_stuck_read); + KUNIT_EXPECT_EQ(test, apple_pmc_domain_power_on(&dom->genpd), -EIO); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->boost_enable), + BIT(0) | BIT(1)); + kunit_deactivate_static_stub(test, apple_pmc_interface_read); + test->priv = NULL; + + KUNIT_EXPECT_EQ(test, apple_pmc_domain_power_on(&dom->genpd), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), BIT(3)); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); +} + static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_dcs_floor), KUNIT_CASE(apple_pmc_test_partial_enable), @@ -291,6 +495,10 @@ static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), KUNIT_CASE(apple_pmc_test_floor_aggregate), KUNIT_CASE(apple_pmc_test_floor_pending_timeout), + KUNIT_CASE(apple_pmc_test_interface), + KUNIT_CASE(apple_pmc_test_interface_stuck), + KUNIT_CASE(apple_pmc_test_interface_genpd), + KUNIT_CASE(apple_pmc_test_interface_boot_state), {} }; diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 6b1e4c472e399..b6b5ccdb3e53e 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -23,6 +23,15 @@ * at its highest state to encode at full speed; macOS raises the same rail * for every encode session. Floors on one rail combine to the highest. * + * On T8132 a display pipe votes through two more interfaces: bandwidth + * requests (BWR) and boost, which raises the display clock. Each has an + * enable word with one bit per agent, and the PMC ignores the votes of an + * agent whose bit is clear: the external pipes' display clock then stays at + * its lowest state, too slow for modes such as 4K at 120 Hz. macOS sets a + * pipe's two bits when its PMGR power state powers on, and clears them + * before it powers off. The driver models them as a power domain too, that + * sits between that power state and the ones that depend on it. + * * Under thermal pressure the driver can also limit the DRAM controller * (DCS) rail: it stops honouring the other agents on that rail and votes * on their behalf through the AP's own agent, capped. This bounds the heat @@ -75,6 +84,10 @@ * @agent_rails: rails on which an agent is enabled; 0 for every rail * @ap_agent: voter agent of the application processor, which a DCS limit * votes through + * @bwr_enable: offset of the bandwidth request (BWR) interface enable word + * in the "rails" region; 0 if the PMC has none + * @boost_enable: offset of the boost interface enable word in the "rails" + * region; 0 if the PMC has none */ struct apple_pmc_hw { u32 agent_stride; @@ -87,6 +100,8 @@ struct apple_pmc_hw { u32 agent_rails; u32 ap_agent; bool has_pending; + u32 bwr_enable; + u32 boost_enable; }; struct apple_pmc { @@ -107,6 +122,11 @@ struct apple_pmc { u32 static_agents; /* Rails with a floor domain. */ u32 floor_rails; + /* BWR and boost interface enables at probe, and the bits of domains. */ + u32 saved_bwr; + u32 saved_boost; + u32 bwr_bits; + u32 boost_bits; struct list_head domains; struct dentry *debugfs; /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ @@ -129,6 +149,9 @@ struct apple_pmc { * @on: with @floor, whether the floor is held; protected by pmc->lock * @cluster: the domain is a cluster enable bit in the "clusters" region * @bit: with @cluster, the bit in the control and status words + * @interface: the domain is a display pipe's BWR and boost interface enables + * @bwr_bit: with @interface, the bit in the BWR interface enable word + * @boost_bit: with @interface, the bit in the boost interface enable word */ struct apple_pmc_domain { struct generic_pm_domain genpd; @@ -142,6 +165,9 @@ struct apple_pmc_domain { bool on; bool cluster; unsigned int bit; + bool interface; + unsigned int bwr_bit; + unsigned int boost_bit; bool provider; }; @@ -678,10 +704,74 @@ static int apple_pmc_cluster_set(struct apple_pmc_domain *dom, bool on) return ret; } +static u32 apple_pmc_interface_read(struct apple_pmc *pmc, u32 offset) +{ +#if IS_ENABLED(CONFIG_APPLE_PMC_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(apple_pmc_interface_read, pmc, offset); +#endif + return readl(pmc->rails + offset); +} + +/* + * Caller holds pmc->lock. Set or clear @bit in the interface enable word at + * @offset, and read the word back. + */ +static int apple_pmc_interface_update(struct apple_pmc *pmc, u32 offset, + unsigned int bit, bool on) +{ + u32 old = apple_pmc_interface_read(pmc, offset); + u32 val = on ? old | BIT(bit) : old & ~BIT(bit); + + if (val == old) + return 0; + + writel(val, pmc->rails + offset); + if ((apple_pmc_interface_read(pmc, offset) ^ val) & BIT(bit)) { + dev_err_ratelimited(pmc->dev, + "interface enable %#x bit %u did not turn %s\n", + offset, bit, str_on_off(on)); + return -EIO; + } + + return 0; +} + +/* + * Set or clear the boost and BWR enables of a display pipe. If the BWR + * enable does not take the change, put the boost enable back to what it + * was: genpd keeps the domain in its old state when a callback fails, and + * a boost enable that was already set, such as one the boot loader left + * for a running pipe, must not be lost to a failed power on. Called with + * IRQs off for an IRQ-safe domain; everything here is atomic. + */ +static int apple_pmc_interface_set(struct apple_pmc_domain *dom, bool on) +{ + struct apple_pmc *pmc = dom->pmc; + const struct apple_pmc_hw *hw = pmc->hw; + unsigned long flags; + bool boost; + int ret; + + raw_spin_lock_irqsave(&pmc->lock, flags); + boost = apple_pmc_interface_read(pmc, hw->boost_enable) & BIT(dom->boost_bit); + ret = apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, on); + if (!ret) { + ret = apple_pmc_interface_update(pmc, hw->bwr_enable, dom->bwr_bit, on); + if (ret) + apple_pmc_interface_update(pmc, hw->boost_enable, + dom->boost_bit, boost); + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + if (dom->interface) + return apple_pmc_interface_set(dom, true); if (dom->cluster) return apple_pmc_cluster_set(dom, true); if (dom->floor) @@ -693,6 +783,8 @@ static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + if (dom->interface) + return apple_pmc_interface_set(dom, false); if (dom->cluster) return apple_pmc_cluster_set(dom, false); if (dom->floor) @@ -857,6 +949,100 @@ static int apple_pmc_cluster_add(struct apple_pmc *pmc, struct device_node *np) return 0; } +/* + * Initialize the genpd of an interface domain, which starts from the + * hardware state: on if both of its bits are set, so that the enables of a + * pipe the boot loader left running stay set. With only one bit set it + * starts off, and the driver leaves that bit alone until the domain powers + * on, which sets the other one. A failed power on keeps the bit too, so no + * vote the boot loader left is dropped before the pipe powers off. + * + * The domain is IRQ safe, since the PMGR power states that depend on it + * are, and genpd only takes an IRQ-safe subdomain under an IRQ-safe parent. + */ +static int apple_pmc_interface_init(struct apple_pmc *pmc, + struct apple_pmc_domain *dom) +{ + bool on = (pmc->saved_bwr & BIT(dom->bwr_bit)) && + (pmc->saved_boost & BIT(dom->boost_bit)); + + dom->genpd.flags |= GENPD_FLAG_IRQ_SAFE; + dom->genpd.power_on = apple_pmc_domain_power_on; + dom->genpd.power_off = apple_pmc_domain_power_off; + + return pm_genpd_init(&dom->genpd, NULL, !on); +} + +/* A display pipe's BWR and boost interface enables, set while powered. */ +static int apple_pmc_interface_add(struct apple_pmc *pmc, struct device_node *np) +{ + const struct apple_pmc_hw *hw = pmc->hw; + struct device *dev = pmc->dev; + struct apple_pmc_domain *dom; + struct of_phandle_args parent; + const char *label; + u32 bwr, boost; + int i, ret; + + if (!hw->bwr_enable || !hw->boost_enable) + return dev_err_probe(dev, -EINVAL, + "%pOF: no interface enables on this PMC\n", np); + if (of_property_read_u32(np, "apple,bwr-bit", &bwr) || bwr >= 32 || + of_property_read_u32(np, "apple,boost-bit", &boost) || boost >= 32) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid interface bits\n", np); + if ((pmc->bwr_bits & BIT(bwr)) || (pmc->boost_bits & BIT(boost))) + return dev_err_probe(dev, -EINVAL, "%pOF: interface bits in use\n", np); + + dom = kzalloc_obj(*dom); + if (!dom) + return -ENOMEM; + + dom->pmc = pmc; + dom->np = of_node_get(np); + dom->interface = true; + dom->bwr_bit = bwr; + dom->boost_bit = boost; + if (of_property_read_string(np, "label", &label)) + label = np->name; + dom->genpd.name = kasprintf(GFP_KERNEL, "pmc-%s", label); + if (!dom->genpd.name) { + of_node_put(dom->np); + kfree(dom); + return -ENOMEM; + } + + ret = apple_pmc_interface_init(pmc, dom); + if (ret) { + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); + return dev_err_probe(dev, ret, "%pOF: genpd init failed\n", np); + } + list_add_tail(&dom->list, &pmc->domains); + pmc->bwr_bits |= BIT(bwr); + pmc->boost_bits |= BIT(boost); + + ret = of_genpd_add_provider_simple(np, &dom->genpd); + if (ret) + return dev_err_probe(dev, ret, "%pOF: provider failed\n", np); + dom->provider = true; + + for (i = 0; !of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", i, + &parent); i++) { + struct of_phandle_args child = { .np = np }; + + ret = of_genpd_add_subdomain(&parent, &child); + of_node_put(parent.np); + if (ret) + return dev_err_probe(dev, ret, + "%pOF: failed to join parent domain\n", + np); + } + + return 0; +} + /* A rail floor held through the AP's agent while the domain is powered. */ static int apple_pmc_floor_add(struct apple_pmc *pmc, struct device_node *np) { @@ -936,6 +1122,14 @@ static int apple_pmc_status_show(struct seq_file *s, void *unused) i, apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); seq_printf(s, "dcs limit: state %lu of %u\n", READ_ONCE(pmc->dcs_state), hw->dcs_max); + if (hw->bwr_enable && hw->boost_enable) { + seq_printf(s, "bwr enable: %#010x (saved %#010x)\n", + apple_pmc_interface_read(pmc, hw->bwr_enable), + pmc->saved_bwr); + seq_printf(s, "boost enable: %#010x (saved %#010x)\n", + apple_pmc_interface_read(pmc, hw->boost_enable), + pmc->saved_boost); + } list_for_each_entry(dom, &pmc->domains, list) if (dom->floor) seq_printf(s, "%s: rail%u level %u %s\n", dom->genpd.name, @@ -970,6 +1164,16 @@ static void apple_pmc_release(void *data) for (rail = 0; rail < pmc->hw->rails; rail++) writel(pmc->saved_enable[rail], pmc->rails + pmc->hw->rail_enable + 4 * rail); + /* Nor does an interface enable: give the domains' bits back. */ + if (pmc->bwr_bits || pmc->boost_bits) { + void __iomem *bwr = pmc->rails + pmc->hw->bwr_enable; + void __iomem *boost = pmc->rails + pmc->hw->boost_enable; + + writel((readl(bwr) & ~pmc->bwr_bits) | + (pmc->saved_bwr & pmc->bwr_bits), bwr); + writel((readl(boost) & ~pmc->boost_bits) | + (pmc->saved_boost & pmc->boost_bits), boost); + } pmc->dcs_state = 0; apple_pmc_dcs_vote(pmc); raw_spin_unlock_irqrestore(&pmc->lock, flags); @@ -1037,6 +1241,10 @@ static int apple_pmc_probe(struct platform_device *pdev) "PMC is not responding\n"); pmc->enable[rail] = pmc->saved_enable[rail]; } + if (pmc->hw->bwr_enable && pmc->hw->boost_enable) { + pmc->saved_bwr = apple_pmc_interface_read(pmc, pmc->hw->bwr_enable); + pmc->saved_boost = apple_pmc_interface_read(pmc, pmc->hw->boost_enable); + } pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); count = of_property_count_u32_elems(np, "apple,static-voters"); @@ -1078,6 +1286,9 @@ static int apple_pmc_probe(struct platform_device *pdev) ret = apple_pmc_floor_add(pmc, child); else if (of_property_present(child, "apple,cluster-bit")) ret = apple_pmc_cluster_add(pmc, child); + else if (of_property_present(child, "apple,bwr-bit") || + of_property_present(child, "apple,boost-bit")) + ret = apple_pmc_interface_add(pmc, child); else ret = apple_pmc_domain_add(pmc, child); if (ret) { @@ -1111,6 +1322,8 @@ static const struct apple_pmc_hw apple_pmc_hw_t8132 = { .dcs_max = 5, .ap_agent = 3, .has_pending = true, + .bwr_enable = 0x2020, + .boost_enable = 0x2028, }; static const struct apple_pmc_hw apple_pmc_hw_t8140 = { diff --git a/drivers/soc/apple/t8132-platform-data.dts b/drivers/soc/apple/t8132-platform-data.dts index 3dd507182ecb4..52d97f73b81b3 100644 --- a/drivers/soc/apple/t8132-platform-data.dts +++ b/drivers/soc/apple/t8132-platform-data.dts @@ -37,6 +37,13 @@ apple,static-voters = <3>; }; +/* + * The PMC interface domains' parents are PMGR states, which a copied phandle + * cannot reach in the loader's tree, and no loader PMGR state uses them. + */ +/delete-node/ &pmc_dispext0; +/delete-node/ &pmc_dispext1; + &ps_gfx { status = "okay"; }; diff --git a/drivers/thunderbolt/apple-dpin-state.c b/drivers/thunderbolt/apple-dpin-state.c index 2c3eadc4ad0f1..1a2721d6e84a4 100644 --- a/drivers/thunderbolt/apple-dpin-state.c +++ b/drivers/thunderbolt/apple-dpin-state.c @@ -38,14 +38,14 @@ const struct apple_dpin_policy apple_dpin_m3 = { * writes only DPTX_INACTIVE (+0x0c bit 0) and polls its ACK (+0x10 bit 0), * gated on HPD. The DP IN interrupts are not set up (nothing handles them), * and a tunnel that DCP never activated is torn down without touching the - * block at all. Both DP IN streams of an ATC PHY need one link rate, so - * every tunnel gets HBR2 at most. + * block at all. Both DP IN streams of an ATC PHY need one link rate: a + * tunnel alone gets up to HBR3, two get HBR2 (TB_HOST_DP_SHARED_RATE). */ const struct apple_dpin_policy apple_dpin_m4 = { .flow = APPLE_DPIN_CHANGED, .capacity_retry = true, .defer_new_bringup = true, - .host_policy = TB_HOST_DP_NOTIFY | TB_HOST_DP_HBR2_MAX, + .host_policy = TB_HOST_DP_NOTIFY | TB_HOST_DP_SHARED_RATE, .lazy_idle = true, }; diff --git a/drivers/thunderbolt/apple-dpin-test.c b/drivers/thunderbolt/apple-dpin-test.c index c9664ffb5e9a6..0d4891ad74e14 100644 --- a/drivers/thunderbolt/apple-dpin-test.c +++ b/drivers/thunderbolt/apple-dpin-test.c @@ -381,7 +381,7 @@ static void apple_dpin_qualification_profiles(struct kunit *test) /* * M4 (T8132): the M3 flow without DP IN interrupts, a lazy teardown, and - * DP tunnels capped at HBR2. + * DP tunnels sharing one link rate. */ static void apple_dpin_m4_policy(struct kunit *test) { @@ -393,15 +393,15 @@ static void apple_dpin_m4_policy(struct kunit *test) KUNIT_EXPECT_FALSE(test, p->t602x_handshake); KUNIT_EXPECT_TRUE(test, p->defer_new_bringup); KUNIT_EXPECT_TRUE(test, p->lazy_idle); - KUNIT_EXPECT_EQ(test, p->host_policy, TB_HOST_DP_NOTIFY | TB_HOST_DP_HBR2_MAX); + KUNIT_EXPECT_EQ(test, p->host_policy, TB_HOST_DP_NOTIFY | TB_HOST_DP_SHARED_RATE); /* The hooks of the M3 flow: a queue and a display pipeline both. */ KUNIT_EXPECT_EQ(test, apple_dpin_hooks(p, true, true), APPLE_DPIN_CHANGED); KUNIT_EXPECT_EQ(test, apple_dpin_hooks(p, true, false), APPLE_DPIN_DISABLED); KUNIT_EXPECT_EQ(test, apple_dpin_hooks(p, false, true), APPLE_DPIN_DISABLED); - /* No other Apple host is capped. */ - KUNIT_EXPECT_FALSE(test, apple_dpin_m3.host_policy & TB_HOST_DP_HBR2_MAX); - KUNIT_EXPECT_FALSE(test, apple_dpin_m2.host_policy & TB_HOST_DP_HBR2_MAX); - KUNIT_EXPECT_FALSE(test, apple_dpin_m1.host_policy & TB_HOST_DP_HBR2_MAX); + /* No other Apple host shares one rate. */ + KUNIT_EXPECT_FALSE(test, apple_dpin_m3.host_policy & TB_HOST_DP_SHARED_RATE); + KUNIT_EXPECT_FALSE(test, apple_dpin_m2.host_policy & TB_HOST_DP_SHARED_RATE); + KUNIT_EXPECT_FALSE(test, apple_dpin_m1.host_policy & TB_HOST_DP_SHARED_RATE); } static void apple_dpin_all_policy_fields(struct kunit *test) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 813f15723ee88..61325859752d4 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -3323,7 +3323,7 @@ static int apple_cio_probe(struct platform_device *pdev) apple_cio_pcie_tunnel_is_preinitialized( acio->pcie_tunnel_np); if (!acio->pcie_tunnel_preinitialized) { - dev_warn(dev, + dev_info(dev, "PCIe-C tunnel disabled: not initialized by m1n1 or the kernel\n"); } else { ret = apple_cio_map_pcie_intr2axi(acio); diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c index 95896401571cc..24d01880fdb3e 100644 --- a/drivers/thunderbolt/ctl.c +++ b/drivers/thunderbolt/ctl.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "ctl.h" @@ -1111,8 +1112,14 @@ static int tb_cfg_get_error(struct tb_ctl *ctl, enum tb_cfg_space space, int tb_cfg_read(struct tb_ctl *ctl, void *buffer, u64 route, u32 port, enum tb_cfg_space space, u32 offset, u32 length) { - struct tb_cfg_result res = tb_cfg_read_raw(ctl, buffer, route, port, - space, offset, length, ctl->timeout_msec); + struct tb_cfg_result res; + + /* The KUnit tests read adapter registers without hardware */ + KUNIT_STATIC_STUB_REDIRECT(tb_cfg_read, ctl, buffer, route, port, space, offset, + length); + + res = tb_cfg_read_raw(ctl, buffer, route, port, space, offset, length, + ctl->timeout_msec); switch (res.err) { case 0: /* Success */ diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 17020bf90a92a..56ffea80c5981 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -927,6 +927,7 @@ int tb_domain_init(void) ret = bus_register(&tb_bus_type); if (ret) goto err_xdomain; + tb_dp_pm_gate_init(); return 0; @@ -941,6 +942,7 @@ int tb_domain_init(void) void tb_domain_exit(void) { + tb_dp_pm_gate_exit(); bus_unregister(&tb_bus_type); ida_destroy(&tb_domain_ida); tb_nvm_exit(); diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 69aecbd97277c..7201e0c279cc3 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -225,11 +225,12 @@ struct tb_nhi_ops { #define TB_HOST_DP_ADAPTER_QUIRKS BIT(4) #define TB_HOST_DP_INITIAL_BW_GRANT BIT(5) /* - * Both DP IN streams of a host PHY must run at one link rate: DP tunnels - * from the host's DP IN adapters offer at most HBR2, at which two 4K60 - * streams still fit. + * Both DP IN streams of a host PHY must run at one link rate. A DP tunnel + * alone on the host router offers up to HBR3; when more are needed, all of + * them offer HBR2 at most (two 4K60 streams fit), a tunnel up above that + * being set up again at HBR2 first. */ -#define TB_HOST_DP_HBR2_MAX BIT(6) +#define TB_HOST_DP_SHARED_RATE BIT(6) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 3816b12422064..2f512323e44c9 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -13,6 +13,9 @@ #include #include #include +#include +#include +#include #include "tb.h" #include "tb_regs.h" @@ -64,6 +67,20 @@ MODULE_PARM_DESC(asym_threshold, * runtime resume * @dp_tunnel_work: Re-pair DP resources after retired DPRX work releases its paths * @dp_tunnel_pending: A pairing request not yet serviced, retained across sleep + * @dp_relink_work: Waits on the freezable queue until no system sleep + * transition is in progress, then queues @dp_relink_run + * @dp_relink_run: The host DP tunnel re-link work proper, on the domain + * queue: after any DPRX or teardown work queued before it + * @dp_relink_deferred: The pairing that needs a host DP tunnel's re-link + * waits for system sleep to end (@dp_tunnel_pending is + * set with it, for the resume and thaw replays) + * @dp_relink_tries: Runtime resume failures of @dp_relink_run in a row + * @dp_relink_due: When the host DP tunnel left alone may be re-linked at a + * higher rate + * @dp_relink_delay: The debounce before that (ms), longer while it flaps + * @dp_relinked_at: When the last such re-link was made + * @dp_host_hold_shared: The next host DP tunnels are set up at HBR2 at + * most: one re-linked at a higher rate did not come up * @groups: Bandwidth groups used in this domain. */ struct tb_cm { @@ -74,6 +91,14 @@ struct tb_cm { struct delayed_work remove_work; struct work_struct dp_tunnel_work; bool dp_tunnel_pending; + struct delayed_work dp_relink_work; + struct work_struct dp_relink_run; + bool dp_relink_deferred; + unsigned int dp_relink_tries; + unsigned long dp_relink_due; + unsigned int dp_relink_delay; + unsigned long dp_relinked_at; + bool dp_host_hold_shared; struct tb_bandwidth_group groups[MAX_GROUPS]; }; @@ -98,6 +123,7 @@ static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port, static void tb_queue_dp_bandwidth_request(struct tb *tb, u64 route, u8 port, int retry, unsigned long delay); static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data); +static void tb_queue_dp_tunnels(struct tb *tb); static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug) { @@ -612,12 +638,9 @@ static int tb_consumed_usb3_pcie_bandwidth(struct tb *tb, * * Return: %0 on success, negative errno otherwise. */ -static int tb_consumed_dp_bandwidth(struct tb *tb, - struct tb_port *src_port, - struct tb_port *dst_port, - struct tb_port *port, - int *consumed_up, - int *consumed_down) +VISIBLE_IF_KUNIT int tb_consumed_dp_bandwidth(struct tb *tb, struct tb_port *src_port, + struct tb_port *dst_port, struct tb_port *port, + int *consumed_up, int *consumed_down) { int group_reserved[MAX_GROUPS + 1] = {}; struct tb_cm *tcm = tb_priv(tb); @@ -1991,6 +2014,7 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) struct tb_port *in = tunnel->src_port; struct tb_port *out = tunnel->dst_port; struct tb *tb = data; + struct tb_cm *tcm = tb_priv(tb); mutex_lock(&tb->lock); @@ -2047,15 +2071,36 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) * loaded or not all DP cables where connected to the * discrete router. */ - tb_tunnel_warn(tunnel, "not active, tearing down\n"); - tb_dp_resource_unavailable(tb, in, - "DPRX negotiation failed"); + if (tb_dp_relink_activation(tunnel->host_relinked_up, false) == + TB_DP_RELINK_NOW) { + /* + * Set up at a higher rate when it was left alone: set + * it up at HBR2 again, once, from the domain queue + * once this DPRX work has released its paths. The + * adapters stay listed. + */ + tb_tunnel_warn(tunnel, "not active at the higher rate, setting it up again at up to %u Mb/s per lane\n", + TB_DP_HOST_RATE_SHARED); + tb_deactivate_and_free_tunnel(tunnel); + tcm->dp_host_hold_shared = true; + tb_recalc_estimated_bandwidth(tb); + tb_queue_dp_tunnels(tb); + } else { + tb_tunnel_warn(tunnel, "not active, tearing down\n"); + tb_dp_resource_unavailable(tb, in, + "DPRX negotiation failed"); + } } mutex_unlock(&tb->lock); } -static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, - struct tb_port *out) +/* + * Set up a DP tunnel from @in to @out. @rate_limit: the highest link rate + * (Mb/s per lane) under TB_HOST_DP_SHARED_RATE. The tunnel if it is up or + * being activated, NULL if not. + */ +static struct tb_tunnel *tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, + struct tb_port *out, u32 rate_limit) { int available_up, available_down, ret, link_nr; struct tb_cm *tcm = tb_priv(tb); @@ -2117,6 +2162,7 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, tb_port_dbg(out, "could not allocate DP tunnel\n"); goto err_reclaim_usb; } + tunnel->host_rate_limit = rate_limit; list_add_tail(&tunnel->list, &tcm->tunnel_list); @@ -2133,7 +2179,7 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, list_del(&tunnel->list); goto err_free; } - return; + return tunnel; err_free: tb_tunnel_put(tunnel); @@ -2148,11 +2194,12 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, pm_runtime_put_autosuspend(&out->sw->dev); pm_runtime_mark_last_busy(&in->sw->dev); pm_runtime_put_autosuspend(&in->sw->dev); + return NULL; } /* * The link rate (Mb/s per lane) another DP tunnel from @tunnel's host router - * was given under TB_HOST_DP_HBR2_MAX, 0 if there is none. Called with the + * was given under TB_HOST_DP_SHARED_RATE, 0 if there is none. Called with the * domain lock held, from the tunnel's activation. */ u32 tb_dp_host_shared_rate(const struct tb_tunnel *tunnel) @@ -2167,16 +2214,568 @@ u32 tb_dp_host_shared_rate(const struct tb_tunnel *tunnel) return 0; } +/* + * TB_HOST_DP_SHARED_RATE: the highest link rate (Mb/s per lane) each of + * @tunnels DP tunnels from the host router's DP IN adapters may offer: HBR3 + * for one, HBR2 for more (two 4K60 streams fit at HBR2 x4). + */ +u32 tb_dp_host_rate_for(unsigned int tunnels) +{ + return tunnels > 1 ? TB_DP_HOST_RATE_SHARED : TB_DP_HOST_RATE_ALONE; +} + +/* + * TB_HOST_DP_SHARED_RATE: whether a host DP tunnel up at @rate (Mb/s per + * lane) is set up again at HBR2 when @tunnels are wanted on its host router + * in all. Above HBR2 next to another it is, so that both streams of the PHY + * run at one rate; while system sleep is in progress (@sleeping) only once + * it is over. + */ +enum tb_dp_relink tb_dp_relink_down(u32 rate, unsigned int tunnels, bool sleeping) +{ + if (tunnels < 2 || rate <= TB_DP_HOST_RATE_SHARED) + return TB_DP_RELINK_NONE; + return sleeping ? TB_DP_RELINK_WAIT : TB_DP_RELINK_NOW; +} + +/* + * TB_HOST_DP_SHARED_RATE re-links tear a display's DP tunnel down and set it + * up again. None runs during a system sleep transition: from its PREPARE + * event, before any other PM notifier hears of it, until the transition is + * over, its POST callbacks and the rollback of a refused PREPARE included. + * Every PM notifier call is made with the system sleep lock held, so the + * gate is reopened by a work item that takes that lock after the POST event. + * Admission holds tb_dp_pm_gate for reading while it re-links; the notifier + * takes it for writing, so a re-link in progress is over before any other + * notifier hears of the transition. + */ +static DECLARE_RWSEM(tb_dp_pm_gate); +static struct tb_dp_pm_state tb_dp_pm; + +/* + * PREPARE closes the gate, and drops a reopen asked for by an earlier POST + * that has not run yet (the snapshot device lets go of the system sleep lock + * between its PREPARE and POST). The notifier comes first and refuses + * nothing, so it gets the POST event of every PREPARE it got, also when the + * PREPARE is refused after it: POST asks for the reopen (true). Other + * events: nothing. + */ +bool tb_dp_pm_event(struct tb_dp_pm_state *s, unsigned long event) +{ + switch (event) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + s->sleeping = true; + s->reopen = false; + return false; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + s->reopen = true; + return true; + } + return false; +} + +/* No transition in progress (system sleep lock held): reopen if one ended. */ +void tb_dp_pm_reopen(struct tb_dp_pm_state *s) +{ + if (s->reopen) + s->sleeping = false; + s->reopen = false; +} + +static void tb_dp_pm_reopen_fn(struct work_struct *work) +{ + unsigned int flags = lock_system_sleep(); + + down_write(&tb_dp_pm_gate); + tb_dp_pm_reopen(&tb_dp_pm); + up_write(&tb_dp_pm_gate); + unlock_system_sleep(flags); +} + +static DECLARE_WORK(tb_dp_pm_reopen_work, tb_dp_pm_reopen_fn); + +static int tb_dp_pm_notify(struct notifier_block *nb, unsigned long event, void *unused) +{ + bool reopen; + + down_write(&tb_dp_pm_gate); + reopen = tb_dp_pm_event(&tb_dp_pm, event); + up_write(&tb_dp_pm_gate); + if (reopen) + queue_work(system_unbound_wq, &tb_dp_pm_reopen_work); + return NOTIFY_DONE; +} + +static struct notifier_block tb_dp_pm_nb = { + .notifier_call = tb_dp_pm_notify, + .priority = INT_MAX, +}; + +void tb_dp_pm_gate_init(void) +{ + register_pm_notifier(&tb_dp_pm_nb); +} + +void tb_dp_pm_gate_exit(void) +{ + unregister_pm_notifier(&tb_dp_pm_nb); + flush_work(&tb_dp_pm_reopen_work); +} + +/* + * Whether a host DP tunnel may be re-linked now; if so, tb_dp_relink_done() + * once it is. Never waits for a transition to start or end. + */ +static bool tb_dp_relink_admit(void) +{ + if (!down_read_trylock(&tb_dp_pm_gate)) + return false; + if (!tb_dp_pm.sleeping) + return true; + up_read(&tb_dp_pm_gate); + return false; +} + +static void tb_dp_relink_done(void) +{ + up_read(&tb_dp_pm_gate); +} + +/* No system sleep transition in progress, as far as can be told now. */ +static bool tb_dp_pm_open(void) +{ + if (!tb_dp_relink_admit()) + return false; + tb_dp_relink_done(); + return true; +} + +/* + * The delay (ms) before the re-link work is tried again after its @tries-th + * failure to runtime resume the domain in a row, 0 for not any more: the + * request then waits for the next resume, thaw or plug event. + */ +unsigned int tb_dp_relink_retry_delay(unsigned int tries) +{ + return tries < TB_DP_RELINK_RETRIES ? TB_DP_RELINK_DELAY_MS << tries : 0; +} + +/* + * TB_HOST_DP_SHARED_RATE: as tb_dp_host_rate_for(), HBR2 at most while a + * tunnel re-linked at a higher rate that did not come up is set up again + * (@recovering). + */ +u32 tb_dp_host_rate_after(unsigned int tunnels, bool recovering) +{ + return recovering ? TB_DP_HOST_RATE_SHARED : tb_dp_host_rate_for(tunnels); +} + +/* + * TB_HOST_DP_SHARED_RATE: whether, after a hot unplug took DP tunnels from + * the host router, the one left (@tunnels) up at @rate is marked to be set up + * again at the higher rate both its adapters take (@max_rate, HBR3 at most). + * Nothing is decided while system sleep is in progress (@sleeping). + */ +bool tb_dp_relink_up_wanted(unsigned int tunnels, u32 rate, u32 max_rate, bool sleeping) +{ + return !sleeping && tunnels == 1 && rate < min(max_rate, TB_DP_HOST_RATE_ALONE); +} + +/* + * TB_HOST_DP_SHARED_RATE: whether the host DP tunnel @marked by + * tb_dp_relink_up_wanted() is set up again at the higher rate. Not if + * another one is up or could be paired (@pairable) by now, nor if it got + * that rate already; only after @due, the end of the debounce, and not + * while system sleep is in progress. + */ +enum tb_dp_relink tb_dp_relink_up(bool marked, unsigned int tunnels, u32 rate, u32 max_rate, + bool pairable, bool sleeping, unsigned long now, + unsigned long due) +{ + if (!marked || pairable || !tb_dp_relink_up_wanted(tunnels, rate, max_rate, false)) + return TB_DP_RELINK_NONE; + if (sleeping || time_before(now, due)) + return TB_DP_RELINK_WAIT; + return TB_DP_RELINK_NOW; +} + +/* + * The debounce (ms) before the next re-link to a higher rate: doubled, up + * to TB_DP_RELINK_DELAY_MAX_MS, each time a second DP tunnel takes the rate + * back down soon after such a re-link (@flapping), the base one otherwise. + */ +unsigned int tb_dp_relink_backoff(unsigned int delay_ms, bool flapping) +{ + if (!flapping) + return TB_DP_RELINK_DELAY_MS; + return min(delay_ms * 2, TB_DP_RELINK_DELAY_MAX_MS); +} + +/* + * A host DP tunnel's activation ended, @active or not: NOW if it is set up + * at HBR2 again because it was re-linked at a higher rate (@relinked_up) + * and did not come up. Only once: that tunnel is not re-linked up. + */ +enum tb_dp_relink tb_dp_relink_activation(bool relinked_up, bool active) +{ + return relinked_up && !active ? TB_DP_RELINK_NOW : TB_DP_RELINK_NONE; +} + +/* + * TB_HOST_DP_SHARED_RATE: another host DP tunnel is wanted while @tunnel, + * marked by tb_dp_host_tunnel_gone(), waits for its re-link at a higher + * rate. The re-link is dropped and @tunnel stays capped at HBR2, the rate + * the new one takes next to it, so that it is counted at HBR2 when the + * bandwidth for the new one is worked out. True if a re-link was dropped. + */ +bool tb_dp_host_drop_relink_up(struct tb_tunnel *tunnel) +{ + if (!tunnel->host_relink) + return false; + tunnel->host_relink = false; + tunnel->host_rate_limit = TB_DP_HOST_RATE_SHARED; + tunnel->max_rate = min(tunnel->max_rate, TB_DP_HOST_RATE_SHARED); + return true; +} + +static bool tb_dp_host_shares_rate(const struct tb *tb) +{ + return tb->nhi && (tb->nhi->host_dp_policy & TB_HOST_DP_SHARED_RATE); +} + +/* A DP tunnel from a DP IN adapter of the (Apple) host router. */ +static bool tb_tunnel_is_host_dp(const struct tb_tunnel *tunnel) +{ + return tb_tunnel_is_dp(tunnel) && tb_port_is_apple_host_dpin(tunnel->src_port); +} + +/* The DP tunnels from the host router's DP IN adapters; @last: the last one. */ +static unsigned int tb_dp_host_tunnels(struct tb *tb, struct tb_tunnel **last) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel; + unsigned int n = 0; + + list_for_each_entry(tunnel, &tcm->tunnel_list, list) { + if (!tb_tunnel_is_host_dp(tunnel)) + continue; + n++; + if (last) + *last = tunnel; + } + return n; +} + +/* + * How many more DP tunnels from the host router's DP IN adapters pairing + * can set up now: its free DP IN adapters against the free DP OUT adapters + * of the other routers (see tb_find_dp_out()). + */ +static unsigned int tb_dp_host_free_pairs(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + unsigned int ins = 0, outs = 0; + struct tb_port *port; + + list_for_each_entry(port, &tcm->dp_resources, list) { + if (tb_port_is_dpin(port)) { + if (tb_port_is_apple_host_dpin(port) && !tb_port_is_enabled(port)) + ins++; + } else if (tb_port_is_dpout(port) && tb_route(port->sw) && + !tb_port_is_enabled(port)) { + outs++; + } + } + return min(ins, outs); +} + +/* A new DP tunnel comes: none is re-linked to a higher rate any more. */ +static void tb_dp_host_clear_relink(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel; + + list_for_each_entry(tunnel, &tcm->tunnel_list, list) + if (tb_tunnel_is_host_dp(tunnel)) + tunnel->host_relink = false; +} + +/* + * Another host DP tunnel is wanted: a tunnel waiting for its re-link at a + * higher rate is held at HBR2 instead (tb_dp_host_drop_relink_up()) and its + * estimated bandwidth worked out again, before any bandwidth is given to + * the new one. The debounce is stopped unless deferred pairing needs the + * re-link work. Domain lock held. + */ +static void tb_dp_host_cancel_relink_up(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel; + bool dropped = false; + + list_for_each_entry(tunnel, &tcm->tunnel_list, list) { + if (!tb_tunnel_is_host_dp(tunnel) || !tb_dp_host_drop_relink_up(tunnel)) + continue; + tb_tunnel_info(tunnel, "another DP tunnel is wanted: kept at up to %u Mb/s per lane\n", + TB_DP_HOST_RATE_SHARED); + dropped = true; + } + if (!dropped) + return; + if (!tcm->dp_relink_deferred && cancel_delayed_work(&tcm->dp_relink_work)) + tb_domain_put(tb); + tb_recalc_estimated_bandwidth(tb); +} + +/* + * Run the re-link work (dp_relink_run) @delay_ms from now, and only once no + * system sleep transition is in progress. One domain reference for each + * queued execution of either work, as for dp_tunnel_work. Domain lock held. + */ +static void tb_dp_relink_arm(struct tb *tb, unsigned int delay_ms) +{ + struct tb_cm *tcm = tb_priv(tb); + + lockdep_assert_held(&tb->lock); + if (tcm->stopping) + return; + tb_domain_get(tb); + if (mod_delayed_work(system_freezable_wq, &tcm->dp_relink_work, + msecs_to_jiffies(delay_ms))) + tb_domain_put(tb); +} + +/* + * Tear @tunnel down and set it up again between the same adapters, at + * @rate_limit (Mb/s per lane) at most, because of @why. Its display sees an + * unplug and a plug. Only on the domain queue, so that the DPRX work of + * @tunnel is not running (the queue is ordered) and releases its paths when + * canceled; with the domain lock held and the re-link admitted + * (tb_dp_relink_admit()). The new tunnel, NULL if there is none. + */ +static struct tb_tunnel *tb_dp_host_relink(struct tb *tb, struct tb_tunnel *tunnel, + u32 rate_limit, const char *why) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *relinked; + struct tb_port *in = tunnel->src_port; + struct tb_port *out = tunnel->dst_port; + + tb_tunnel_info(tunnel, "%s: re-linking at up to %u Mb/s per lane (was %u)\n", + why, rate_limit, tunnel->host_rate); + tb_deactivate_and_free_tunnel(tunnel); + tb_recalc_estimated_bandwidth(tb); + relinked = tb_tunnel_one_dp(tb, in, out, rate_limit); + if (relinked) + return relinked; + /* Not at the higher rate: at the shared one, as before. */ + if (rate_limit > TB_DP_HOST_RATE_SHARED) { + relinked = tb_tunnel_one_dp(tb, in, out, TB_DP_HOST_RATE_SHARED); + if (relinked) { + tb_port_warn(in, "DP tunnel set up again at up to %u Mb/s per lane\n", + TB_DP_HOST_RATE_SHARED); + return relinked; + } + tcm->dp_host_hold_shared = true; + } + /* The adapters stay listed: pairing tries again, at the next event too. */ + tb_port_warn(in, "DP tunnel not set up again, pairing again\n"); + tb_queue_dp_tunnels(tb); + return NULL; +} + +/* + * TB_HOST_DP_SHARED_RATE: the link rate limit (Mb/s per lane) for the DP + * tunnels pairing sets up now from the host router's DP IN adapters: HBR3 + * when one tunnel will be up, HBR2 when more will. A tunnel already up + * above HBR2 when more are wanted is set up again at HBR2 first, between + * the same adapters, so that the streams of the PHY share one rate. Not + * while system sleep is in progress: then 0, and the host router's DP IN + * adapters are paired once it is over (dp_relink_work), or at the next + * resume or thaw. 0 without the policy. Domain lock held. + */ +VISIBLE_IF_KUNIT u32 tb_dp_host_rate_limit(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel, *above; + bool admitted, recovering = false; + enum tb_dp_relink action; + unsigned int up, want; + unsigned long calm; + + if (!tb_dp_host_shares_rate(tb)) + return 0; + up = tb_dp_host_tunnels(tb, NULL); + want = up + tb_dp_host_free_pairs(tb); + /* + * Another one comes: the one alone is not re-linked to a higher rate, + * and counts at HBR2 for the new one's bandwidth. + */ + if (want > up) { + tb_dp_host_cancel_relink_up(tb); + recovering = tcm->dp_host_hold_shared; + tcm->dp_host_hold_shared = false; + } + + /* Each round sets one tunnel up again at HBR2 at most. */ + for (;;) { + above = NULL; + list_for_each_entry(tunnel, &tcm->tunnel_list, list) { + if (tb_tunnel_is_host_dp(tunnel) && + tunnel->host_rate > TB_DP_HOST_RATE_SHARED) { + above = tunnel; + break; + } + } + if (!above) + break; + + admitted = tb_dp_relink_admit(); + action = tb_dp_relink_down(above->host_rate, want, !admitted); + if (action == TB_DP_RELINK_NOW) { + /* Soon after a re-link up: back off the next one. */ + calm = tcm->dp_relinked_at + msecs_to_jiffies(TB_DP_RELINK_CALM_MS); + tcm->dp_relink_delay = tb_dp_relink_backoff(tcm->dp_relink_delay, + time_before(jiffies, calm)); + tb_dp_host_relink(tb, above, TB_DP_HOST_RATE_SHARED, + "another DP tunnel is wanted"); + } + if (admitted) { + tb_dp_relink_done(); + tcm->dp_relink_deferred = false; + } + if (action == TB_DP_RELINK_NONE) + break; + if (action == TB_DP_RELINK_WAIT) { + if (!tcm->dp_relink_deferred) + tb_tunnel_info(above, "another DP tunnel is wanted during system sleep, re-linking after it\n"); + /* Replayed by dp_relink_work, and at resume or thaw. */ + tcm->dp_relink_deferred = true; + tcm->dp_tunnel_pending = true; + tcm->dp_host_hold_shared |= recovering; + tb_dp_relink_arm(tb, 0); + return 0; + } + } + return tb_dp_host_rate_after(want, recovering); +} + +/* + * TB_HOST_DP_SHARED_RATE: a hot unplug took DP tunnels from the host + * router's DP IN adapters (@before were up). The one left alone below the + * rate it could get is set up again at that rate once the debounce + * (dp_relink_delay) is over, unless another one comes by then, so that an + * unplug and replug or a retraining does not cause a re-link. Until it is + * set up again, it keeps its HBR2 cap and is counted at HBR2. Nothing is + * marked during a system sleep transition. Domain lock held. + */ +VISIBLE_IF_KUNIT void tb_dp_host_tunnel_gone(struct tb *tb, unsigned int before) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel = NULL; + unsigned int up; + + if (!tb_dp_host_shares_rate(tb)) + return; + up = tb_dp_host_tunnels(tb, &tunnel); + if (!tunnel || up >= before || + !tb_dp_relink_up_wanted(up, tunnel->host_rate, tunnel->host_max_rate, + !tb_dp_pm_open())) + return; + + tunnel->host_relink = true; + tcm->dp_relink_due = jiffies + msecs_to_jiffies(tcm->dp_relink_delay); + tb_tunnel_info(tunnel, "alone at %u Mb/s per lane: re-linking at up to %u in %u ms if no other DP tunnel comes\n", + tunnel->host_rate, min(tunnel->host_max_rate, TB_DP_HOST_RATE_ALONE), + tcm->dp_relink_delay); + tb_dp_relink_arm(tb, tcm->dp_relink_delay); +} + +/* A host DP tunnel waits for its re-link to a higher rate. */ +static bool tb_dp_host_relink_marked(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel; + + list_for_each_entry(tunnel, &tcm->tunnel_list, list) + if (tb_tunnel_is_host_dp(tunnel) && tunnel->host_relink) + return true; + return false; +} + +/* + * The debounced part of tb_dp_host_tunnel_gone(), from dp_relink_run on the + * domain queue. Domain lock held. + */ +static void tb_dp_host_relink_up(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel = NULL, *relinked; + enum tb_dp_relink action; + unsigned int up; + bool admitted; + long left; + + if (!tb_dp_host_relink_marked(tb)) + return; + up = tb_dp_host_tunnels(tb, &tunnel); + + admitted = tb_dp_relink_admit(); + action = tb_dp_relink_up(tunnel->host_relink, up, tunnel->host_rate, + tunnel->host_max_rate, tb_dp_host_free_pairs(tb) > 0, + !admitted, jiffies, tcm->dp_relink_due); + switch (action) { + case TB_DP_RELINK_NONE: + tb_dbg(tb, "DP tunnel re-link to a higher rate not needed any more\n"); + tb_dp_host_clear_relink(tb); + break; + case TB_DP_RELINK_WAIT: + left = (long)(tcm->dp_relink_due - jiffies); + tb_dp_relink_arm(tb, left > 0 ? jiffies_to_msecs(left) : TB_DP_RELINK_POLL_MS); + break; + case TB_DP_RELINK_NOW: + tunnel->host_relink = false; + tcm->dp_relinked_at = jiffies; + relinked = tb_dp_host_relink(tb, tunnel, TB_DP_HOST_RATE_ALONE, + "alone on the host router"); + /* Set up at HBR2 again if it does not come up at this rate. */ + if (relinked && relinked->host_rate_limit > TB_DP_HOST_RATE_SHARED) + relinked->host_relinked_up = true; + break; + } + if (admitted) + tb_dp_relink_done(); +} + +/* Replays a re-link that waited, at resume and runtime resume. Domain lock held. */ +static void tb_dp_relink_replay(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + + if (tcm->dp_relink_deferred || tb_dp_host_relink_marked(tb)) + tb_dp_relink_arm(tb, 0); +} + static void tb_tunnel_dp(struct tb *tb) { struct tb_cm *tcm = tb_priv(tb); struct tb_port *port, *in, *out; + bool host_waits; + u32 rate_limit; if (!tb_acpi_may_tunnel_dp()) { tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n"); return; } + /* Under TB_HOST_DP_SHARED_RATE: may set a tunnel up again first. */ + rate_limit = tb_dp_host_rate_limit(tb); + host_waits = tb_dp_host_shares_rate(tb) && !rate_limit; + /* * Find pair of inactive DP IN and DP OUT adapters and then * establish a DP tunnel between them. @@ -2197,9 +2796,14 @@ static void tb_tunnel_dp(struct tb *tb) in = port; tb_port_dbg(in, "DP IN available\n"); + if (host_waits && tb_port_is_apple_host_dpin(in)) { + tb_port_dbg(in, "waiting for system sleep to end\n"); + continue; + } + out = tb_find_dp_out(tb, port); if (out) - tb_tunnel_one_dp(tb, in, out); + tb_tunnel_one_dp(tb, in, out, rate_limit); else tb_port_dbg(in, "no suitable DP OUT adapter available, not tunneling\n"); } @@ -2316,6 +2920,79 @@ static void tb_dp_tunnel_work(struct work_struct *work) tb_domain_put(tb); } +/* + * The timer and gate of the re-link work, on the freezable queue: it does + * not run while tasks are frozen, and hands dp_relink_run to the domain + * queue only while no system sleep transition is in progress. + */ +static void tb_dp_relink_work(struct work_struct *work) +{ + struct tb_cm *tcm = container_of(to_delayed_work(work), struct tb_cm, + dp_relink_work); + struct tb *tb = tcm_to_tb(tcm); + + mutex_lock(&tb->lock); + if (!tcm->stopping) { + if (!tb_dp_pm_open()) { + tb_dp_relink_arm(tb, TB_DP_RELINK_POLL_MS); + } else { + tb_domain_get(tb); + if (!queue_work(tb->wq, &tcm->dp_relink_run)) + tb_domain_put(tb); + } + } + mutex_unlock(&tb->lock); + tb_domain_put(tb); +} + +/* + * Pairing that waited for system sleep to end, see tb_dp_host_rate_limit(), + * and the debounced re-link of tb_dp_host_tunnel_gone(). On the domain + * queue: behind DPRX and teardown work queued before it, so a tunnel torn + * down there has released its paths. + */ +static void tb_dp_relink_run(struct work_struct *work) +{ + struct tb_cm *tcm = container_of(work, struct tb_cm, dp_relink_run); + struct tb *tb = tcm_to_tb(tcm); + unsigned int delay; + int ret; + + /* Runtime resume can acquire the domain lock, so do it first. */ + ret = pm_runtime_resume_and_get(&tb->dev); + + mutex_lock(&tb->lock); + if (tcm->stopping || !tb->root_switch) + goto unlock; + if (ret < 0) { + /* A few more tries; then the next resume, thaw or plug event. */ + delay = tb_dp_relink_retry_delay(tcm->dp_relink_tries++); + if (delay) + tb_dp_relink_arm(tb, delay); + else + tb_dbg(tb, "DP re-link waits for the next event: runtime resume failed: %d\n", + ret); + goto unlock; + } + tcm->dp_relink_tries = 0; + if (tcm->hotplug_active && tcm->dp_relink_deferred) { + tb_dbg(tb, "system sleep over, pairing DP resources\n"); + tcm->dp_tunnel_pending = false; + /* Checks the gate again, and defers again if it is closed. */ + tb_tunnel_dp(tb); + } + if (tcm->hotplug_active) + tb_dp_host_relink_up(tb); +unlock: + mutex_unlock(&tb->lock); + + if (ret >= 0) { + pm_runtime_mark_last_busy(&tb->dev); + pm_runtime_put_autosuspend(&tb->dev); + } + tb_domain_put(tb); +} + static void tb_queue_dp_tunnels(struct tb *tb) { struct tb_cm *tcm = tb_priv(tb); @@ -2614,6 +3291,8 @@ static void tb_handle_hotplug(struct work_struct *work) pm_runtime_get_sync(&sw->dev); if (ev->unplug) { + unsigned int dp_up = tb_dp_host_tunnels(tb, NULL); + tb_retimer_remove_all(port); if (tb_port_has_remote(port)) { @@ -2632,6 +3311,7 @@ static void tb_handle_hotplug(struct work_struct *work) /* Maybe we can create another DP tunnel */ tb_recalc_estimated_bandwidth(tb); tb_tunnel_dp(tb); + tb_dp_host_tunnel_gone(tb, dp_up); } else if (port->xdomain) { struct tb_xdomain *xd = tb_xdomain_get(port->xdomain); @@ -2651,6 +3331,9 @@ static void tb_handle_hotplug(struct work_struct *work) tb_port_unconfigure_xdomain(port); } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) { tb_dp_resource_unavailable(tb, port, "adapter unplug"); + /* A display went: the DP tunnel left may take a higher rate. */ + if (tb_port_is_dpout(port)) + tb_dp_host_tunnel_gone(tb, dp_up); } else if (!port->port) { tb_sw_dbg(sw, "xHCI disconnect request\n"); tb_switch_xhci_disconnect(sw); @@ -3110,6 +3793,10 @@ static void tb_stop(struct tb *tb) tcm->stopping = true; tcm->hotplug_active = false; cancel_delayed_work(&tcm->remove_work); + if (cancel_delayed_work(&tcm->dp_relink_work)) + tb_domain_put(tb); + if (cancel_work(&tcm->dp_relink_run)) + tb_domain_put(tb); list_for_each_entry(tunnel, &tcm->tunnel_list, list) { if (tb_tunnel_is_dp(tunnel)) tb_dp_tunnel_cancel_dprx(tunnel); @@ -3169,6 +3856,11 @@ static void tb_deinit(struct tb *tb) /* Cancel all the release bandwidth workers */ for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) cancel_delayed_work_sync(&tcm->groups[i].release_work); + /* Stopped: nothing queues them again. */ + if (cancel_delayed_work_sync(&tcm->dp_relink_work)) + tb_domain_put(tb); + if (cancel_work_sync(&tcm->dp_relink_run)) + tb_domain_put(tb); } static int tb_scan_finalize_switch(struct device *dev, void *data) @@ -3540,6 +4232,7 @@ static void tb_complete(struct tb *tb) /* A noirq replay may have failed while runtime PM was disabled. */ if (tcm->dp_tunnel_pending) tb_queue_dp_tunnels(tb); + tb_dp_relink_replay(tb); } static int tb_runtime_suspend(struct tb *tb) @@ -3596,6 +4289,7 @@ static int tb_runtime_resume(struct tb *tb) tcm->hotplug_active = true; if (tcm->dp_tunnel_pending) tb_queue_dp_tunnels(tb); + tb_dp_relink_replay(tb); mutex_unlock(&tb->lock); /* @@ -3694,12 +4388,24 @@ static bool tb_apple_add_links(struct tb_nhi *nhi) return ret; } +static void tb_cm_init(struct tb_cm *tcm) +{ + INIT_LIST_HEAD(&tcm->tunnel_list); + INIT_LIST_HEAD(&tcm->dp_resources); + INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work); + INIT_WORK(&tcm->dp_tunnel_work, tb_dp_tunnel_work); + INIT_DELAYED_WORK(&tcm->dp_relink_work, tb_dp_relink_work); + INIT_WORK(&tcm->dp_relink_run, tb_dp_relink_run); + tcm->dp_relink_delay = TB_DP_RELINK_DELAY_MS; + tcm->dp_relinked_at = jiffies - msecs_to_jiffies(TB_DP_RELINK_CALM_MS); + tb_init_bandwidth_groups(tcm); +} + struct tb *tb_probe(struct tb_nhi *nhi) { - struct tb_cm *tcm; struct tb *tb; - tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm)); + tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(struct tb_cm)); if (!tb) return NULL; @@ -3710,12 +4416,7 @@ struct tb *tb_probe(struct tb_nhi *nhi) tb->cm_ops = &tb_cm_ops; - tcm = tb_priv(tb); - INIT_LIST_HEAD(&tcm->tunnel_list); - INIT_LIST_HEAD(&tcm->dp_resources); - INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work); - INIT_WORK(&tcm->dp_tunnel_work, tb_dp_tunnel_work); - tb_init_bandwidth_groups(tcm); + tb_cm_init(tb_priv(tb)); tb_dbg(tb, "using software connection manager\n"); @@ -3730,3 +4431,60 @@ struct tb *tb_probe(struct tb_nhi *nhi) return tb; } EXPORT_SYMBOL_FOR_MODULES(tb_probe, "thunderbolt_apple"); + +#if IS_ENABLED(CONFIG_USB4_KUNIT_TEST) +/* + * For the KUnit tests: the software connection manager of a domain the test + * sets up itself, with no hardware behind it. The domain is allocated with + * tb_test_cm_size() bytes of private data. + */ +size_t tb_test_cm_size(void) +{ + return sizeof(struct tb_cm); +} + +void tb_test_cm_init(struct tb *tb) +{ + tb_cm_init(tb_priv(tb)); +} + +struct list_head *tb_test_cm_tunnels(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + + return &tcm->tunnel_list; +} + +struct list_head *tb_test_cm_dp_resources(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + + return &tcm->dp_resources; +} + +void tb_test_cm_set_relink_deferred(struct tb *tb, bool deferred) +{ + struct tb_cm *tcm = tb_priv(tb); + + tcm->dp_relink_deferred = deferred; +} + +/* The re-link work's timer is armed. */ +bool tb_test_cm_relink_armed(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + + return delayed_work_pending(&tcm->dp_relink_work); +} + +/* Stop the re-link work as tb_deinit() does, dropping its domain references. */ +void tb_test_cm_stop(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + + if (cancel_delayed_work_sync(&tcm->dp_relink_work)) + tb_domain_put(tb); + if (cancel_work_sync(&tcm->dp_relink_run)) + tb_domain_put(tb); +} +#endif diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 8bcba8e8434d5..841d1ccb27042 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -764,6 +764,22 @@ static inline int tb_port_write(struct tb_port *port, const void *buffer, struct tb *icm_probe(struct tb_nhi *nhi); struct tb *tb_probe(struct tb_nhi *nhi); +#if IS_ENABLED(CONFIG_KUNIT) +int tb_consumed_dp_bandwidth(struct tb *tb, struct tb_port *src_port, + struct tb_port *dst_port, struct tb_port *port, + int *consumed_up, int *consumed_down); +u32 tb_dp_host_rate_limit(struct tb *tb); +void tb_dp_host_tunnel_gone(struct tb *tb, unsigned int before); +#endif +#if IS_ENABLED(CONFIG_USB4_KUNIT_TEST) +size_t tb_test_cm_size(void); +void tb_test_cm_init(struct tb *tb); +struct list_head *tb_test_cm_tunnels(struct tb *tb); +struct list_head *tb_test_cm_dp_resources(struct tb *tb); +void tb_test_cm_set_relink_deferred(struct tb *tb, bool deferred); +bool tb_test_cm_relink_armed(struct tb *tb); +void tb_test_cm_stop(struct tb *tb); +#endif extern const struct device_type tb_domain_type; extern const struct device_type tb_retimer_type; @@ -772,6 +788,8 @@ extern const struct device_type usb4_port_device_type; int tb_domain_init(void); void tb_domain_exit(void); +void tb_dp_pm_gate_init(void); +void tb_dp_pm_gate_exit(void); int tb_xdomain_init(void); void tb_xdomain_exit(void); diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index 745489aef1941..f8418132064af 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -7,8 +7,10 @@ */ #include +#include #include #include +#include #include "tb.h" #include "tunnel.h" @@ -3605,8 +3607,8 @@ static void tb_test_apple_right_dpin_wiring(struct kunit *test) } /* - * TB_HOST_DP_HBR2_MAX: a second DP tunnel keeps the rate of the first and - * gives up lanes instead; alone, lanes go before the rate. + * TB_HOST_DP_SHARED_RATE: a second DP tunnel keeps the rate of the first and + * gives up lanes instead; alone at HBR2, lanes go before the rate. */ static void tb_test_dp_reduce_bandwidth_shared(struct kunit *test) { @@ -3646,9 +3648,690 @@ static void tb_test_dp_reduce_bandwidth_shared(struct kunit *test) &rate, &lanes), -ENOSR); } +/* + * TB_HOST_DP_SHARED_RATE: one DP tunnel on the host router may offer HBR3, + * more offer HBR2. Alone, a reduction keeps the most bandwidth; held at + * HBR2 or next to another tunnel, the lanes go first. + */ +static void tb_test_dp_shared_rate_single(struct kunit *test) +{ + u32 rate, lanes; + + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_for(0), 8100); + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_for(1), 8100); + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_for(2), 5400); + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_for(3), 5400); + + /* Alone, the dock's 34650 Mb/s: HBR3 x4 (25920 Mb/s) */ + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(34650, 8100, 4, 8100, 4, 0, 8100, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 8100); + KUNIT_EXPECT_EQ(test, lanes, 4); + /* Alone, 20000 Mb/s: HBR2 x4 (17280), not HBR3 x2 (12960, too little for 4K60) */ + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(20000, 8100, 4, 8100, 4, 0, 8100, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 5400); + KUNIT_EXPECT_EQ(test, lanes, 4); + /* Alone, a sink side that takes HBR2 at most: HBR2 x4 */ + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(34650, 8100, 4, 5400, 4, 0, 8100, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 5400); + KUNIT_EXPECT_EQ(test, lanes, 4); + /* Held at HBR2 for a second tunnel, 10000 Mb/s: HBR2 x2, lanes first */ + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(10000, 5400, 4, 5400, 4, 0, 5400, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 5400); + KUNIT_EXPECT_EQ(test, lanes, 2); +} + +/* + * TB_HOST_DP_SHARED_RATE: with two DP tunnels the second takes the first + * one's rate, lanes first; a first one up above HBR2 is set up again at HBR2. + */ +static void tb_test_dp_shared_rate_second(struct kunit *test) +{ + u32 rate, lanes; + + /* A second one wanted next to one at HBR3: re-link it now */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(8100, 2, false), TB_DP_RELINK_NOW); + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(8100, 3, false), TB_DP_RELINK_NOW); + /* Alone: kept */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(8100, 1, false), TB_DP_RELINK_NONE); + /* At HBR2 or below: kept, the second one takes its rate */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(5400, 2, false), TB_DP_RELINK_NONE); + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(2700, 2, false), TB_DP_RELINK_NONE); + + /* + * The first one, held at HBR2 between adapters that take HBR3, is + * counted at HBR2 whatever its DP IN adapter reports: 34650 - 17280 + * left for the second, not 34650 - 25920 (HBR2 x2 at most) + */ + KUNIT_EXPECT_EQ(test, tb_dp_host_counted_bandwidth(5400, 8100, 4), 17280); + KUNIT_EXPECT_EQ(test, tb_dp_host_counted_bandwidth(5400, 5400, 2), 8640); + KUNIT_EXPECT_EQ(test, tb_dp_host_counted_bandwidth(8100, 8100, 4), 25920); + KUNIT_EXPECT_EQ(test, tb_dp_host_counted_bandwidth(0, 8100, 4), 25920); + + /* Two HBR2 x4 tunnels fit the dock: 34650 - 17280 left for the second */ + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(17370, 5400, 4, 5400, 4, 5400, 5400, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 5400); + KUNIT_EXPECT_EQ(test, lanes, 4); + /* Less room: the second gives up lanes, never its rate */ + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(12000, 5400, 4, 8100, 4, 5400, 5400, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 5400); + KUNIT_EXPECT_EQ(test, lanes, 2); + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(4000, 5400, 4, 5400, 4, 5400, 5400, + &rate, &lanes), -ENOSR); +} + +/* + * TB_HOST_DP_SHARED_RATE: when the other DP tunnel goes, the one left at + * HBR2 is re-linked at HBR3 once the debounce is over, unless another one + * comes by then; once per unplug, and the debounce backs off while it flaps. + */ +static void tb_test_dp_shared_rate_relink_up(struct kunit *test) +{ + const unsigned long due = 1000; + + /* Alone at HBR2, HBR3 possible: marked */ + KUNIT_EXPECT_TRUE(test, tb_dp_relink_up_wanted(1, 5400, 8100, false)); + /* Still two, none left, already at its best, or an HBR2-only sink side: not */ + KUNIT_EXPECT_FALSE(test, tb_dp_relink_up_wanted(2, 5400, 8100, false)); + KUNIT_EXPECT_FALSE(test, tb_dp_relink_up_wanted(0, 0, 0, false)); + KUNIT_EXPECT_FALSE(test, tb_dp_relink_up_wanted(1, 8100, 8100, false)); + KUNIT_EXPECT_FALSE(test, tb_dp_relink_up_wanted(1, 5400, 5400, false)); + + /* Within the debounce: wait; after it: re-link */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(true, 1, 5400, 8100, false, false, due - 1, due), + TB_DP_RELINK_WAIT); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(true, 1, 5400, 8100, false, false, due, due), + TB_DP_RELINK_NOW); + /* Not marked (the re-linked tunnel itself): once only */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(false, 1, 5400, 8100, false, false, due, due), + TB_DP_RELINK_NONE); + /* Another one came, or could be paired: dropped */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(true, 2, 5400, 8100, false, false, due, due), + TB_DP_RELINK_NONE); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(true, 1, 5400, 8100, true, false, due, due), + TB_DP_RELINK_NONE); + /* At HBR3 by now: nothing to do */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(true, 1, 8100, 8100, false, false, due, due), + TB_DP_RELINK_NONE); + + /* Back-off: doubled while it flaps, at most 40 s; reset when calm */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_backoff(2500, false), 2500); + KUNIT_EXPECT_EQ(test, tb_dp_relink_backoff(2500, true), 5000); + KUNIT_EXPECT_EQ(test, tb_dp_relink_backoff(20000, true), 40000); + KUNIT_EXPECT_EQ(test, tb_dp_relink_backoff(40000, true), 40000); + KUNIT_EXPECT_EQ(test, tb_dp_relink_backoff(40000, false), 2500); +} + +/* + * TB_HOST_DP_SHARED_RATE: a tunnel re-linked at HBR3 that does not come up + * is set up again at HBR2, once and without a new re-link up; any other + * tunnel that does not come up is torn down as before. + */ +static void tb_test_dp_shared_rate_recover(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, tb_dp_relink_activation(true, false), TB_DP_RELINK_NOW); + KUNIT_EXPECT_EQ(test, tb_dp_relink_activation(true, true), TB_DP_RELINK_NONE); + KUNIT_EXPECT_EQ(test, tb_dp_relink_activation(false, false), TB_DP_RELINK_NONE); + /* The recovery: HBR2 at most even alone; afterwards HBR3 alone again */ + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_after(1, true), 5400); + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_after(1, false), 8100); + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_after(2, false), 5400); + /* The tunnel set up so (no re-link mark) is never re-linked up again */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(false, 1, 5400, 8100, false, false, 1000, 1000), + TB_DP_RELINK_NONE); +} + +/* + * TB_HOST_DP_SHARED_RATE: no re-link while system sleep is in progress, from + * the PREPARE event until the transition is over, also when it is refused. + */ +static void tb_test_dp_shared_rate_sleep(struct kunit *test) +{ + struct tb_dp_pm_state s = {}; + + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(8100, 2, true), TB_DP_RELINK_WAIT); + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(5400, 2, true), TB_DP_RELINK_NONE); + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(8100, 1, true), TB_DP_RELINK_NONE); + + /* Suspend: closed at PREPARE, open only once the transition is over */ + KUNIT_EXPECT_FALSE(test, tb_dp_pm_event(&s, PM_SUSPEND_PREPARE)); + KUNIT_EXPECT_TRUE(test, s.sleeping); + KUNIT_EXPECT_TRUE(test, tb_dp_pm_event(&s, PM_POST_SUSPEND)); + KUNIT_EXPECT_TRUE(test, s.sleeping); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_FALSE(test, s.sleeping); + /* Refused: still closed during the POST replay, until the reopen */ + tb_dp_pm_event(&s, PM_SUSPEND_PREPARE); + KUNIT_EXPECT_TRUE(test, tb_dp_pm_event(&s, PM_POST_SUSPEND)); + KUNIT_EXPECT_TRUE(test, s.sleeping); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_FALSE(test, s.sleeping); + /* No reopen without a POST event: closed until it comes */ + tb_dp_pm_event(&s, PM_SUSPEND_PREPARE); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_TRUE(test, s.sleeping); + tb_dp_pm_event(&s, PM_POST_SUSPEND); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_FALSE(test, s.sleeping); + /* + * The reopen asked for by one transition runs after the next one began + * (the snapshot device drops the lock in between): closed until that + * transition's POST and the reopen after it + */ + tb_dp_pm_event(&s, PM_HIBERNATION_PREPARE); + tb_dp_pm_event(&s, PM_POST_HIBERNATION); + tb_dp_pm_event(&s, PM_HIBERNATION_PREPARE); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_TRUE(test, s.sleeping); + KUNIT_EXPECT_TRUE(test, tb_dp_pm_event(&s, PM_POST_HIBERNATION)); + KUNIT_EXPECT_TRUE(test, s.sleeping); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_FALSE(test, s.sleeping); + /* Two transitions before the reopen runs: one reopen, at the end */ + tb_dp_pm_event(&s, PM_HIBERNATION_PREPARE); + tb_dp_pm_event(&s, PM_POST_HIBERNATION); + tb_dp_pm_event(&s, PM_RESTORE_PREPARE); + tb_dp_pm_event(&s, PM_POST_RESTORE); + KUNIT_EXPECT_TRUE(test, s.sleeping); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_FALSE(test, s.sleeping); + tb_dp_pm_reopen(&s); + KUNIT_EXPECT_FALSE(test, s.sleeping); + /* Other events change nothing */ + tb_dp_pm_event(&s, PM_SUSPEND_PREPARE); + KUNIT_EXPECT_FALSE(test, tb_dp_pm_event(&s, 0)); + KUNIT_EXPECT_TRUE(test, s.sleeping); + + /* Tunnels going during the sleep mark nothing */ + KUNIT_EXPECT_FALSE(test, tb_dp_relink_up_wanted(1, 5400, 8100, true)); + /* A re-link to HBR3 due during the sleep waits for its end */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(true, 1, 5400, 8100, false, true, 1001, 1000), + TB_DP_RELINK_WAIT); +} + +/* + * TB_HOST_DP_SHARED_RATE: re-link work that cannot runtime resume the domain + * tries again, a few times and later each time; then the request waits for + * the next resume, thaw or plug event. + */ +static void tb_test_dp_shared_rate_retry(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, tb_dp_relink_retry_delay(0), 2500); + KUNIT_EXPECT_EQ(test, tb_dp_relink_retry_delay(1), 5000); + KUNIT_EXPECT_EQ(test, tb_dp_relink_retry_delay(4), 40000); + KUNIT_EXPECT_EQ(test, tb_dp_relink_retry_delay(5), 0); + KUNIT_EXPECT_EQ(test, tb_dp_relink_retry_delay(100), 0); +} + +/* The dock's link: what is left there for DP tunnels (Mb/s) */ +#define TB_TEST_DOCK_DP_BW 34650 + +/* + * Set @t up as pairing and the capabilities exchange do under + * TB_HOST_DP_SHARED_RATE, between adapters that take HBR3 x4: capped at + * @rate_limit, next to a tunnel at @shared_rate (0: alone), with @room Mb/s + * left on the link. The lanes it takes in @lanes. + */ +static void tb_test_host_dp_up(struct kunit *test, struct tb_tunnel *t, u32 rate_limit, + u32 shared_rate, int room, u32 *lanes) +{ + u32 rate; + int ret; + + memset(t, 0, sizeof(*t)); + t->host_rate_limit = rate_limit; + t->max_rate = rate_limit; + t->host_max_rate = TB_DP_HOST_RATE_ALONE; + ret = tb_dp_host_reduce_bandwidth(room, rate_limit, 4, rate_limit, 4, shared_rate, + rate_limit, &rate, lanes); + KUNIT_ASSERT_EQ(test, ret, 0); + t->max_rate = min(t->max_rate, rate); + t->host_rate = rate; +} + +/* + * What @t, up at @lanes, leaves on the dock's link for another DP tunnel, + * its DP IN adapter reporting the rate both adapters take (HBR3). + */ +static int tb_test_host_dp_room(const struct tb_tunnel *t, u32 lanes) +{ + return TB_TEST_DOCK_DP_BW - + tb_dp_host_counted_bandwidth(t->max_rate, TB_DP_HOST_RATE_ALONE, lanes); +} + +/* + * A at HBR2 x4 next to B: B is wanted while A is up at HBR3, so A is set up + * again at HBR2 first, alone on the link, then B takes its rate. + */ +static void tb_test_host_dp_two_up(struct kunit *test, struct tb_tunnel *a, u32 *a_lanes, + struct tb_tunnel *b, u32 *b_lanes) +{ + tb_test_host_dp_up(test, a, tb_dp_host_rate_for(1), 0, TB_TEST_DOCK_DP_BW, a_lanes); + KUNIT_ASSERT_EQ(test, tb_dp_relink_down(a->host_rate, 2, false), TB_DP_RELINK_NOW); + tb_test_host_dp_up(test, a, tb_dp_host_rate_for(2), 0, TB_TEST_DOCK_DP_BW, a_lanes); + tb_test_host_dp_up(test, b, tb_dp_host_rate_for(2), a->host_rate, + tb_test_host_dp_room(a, *a_lanes), b_lanes); +} + +/* TB_HOST_DP_SHARED_RATE, one display: HBR3 x4, counted so, nothing to re-link. */ +static void tb_test_dp_shared_rate_lone(struct kunit *test) +{ + struct tb_tunnel *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + u32 lanes; + + KUNIT_ASSERT_NOT_NULL(test, a); + tb_test_host_dp_up(test, a, tb_dp_host_rate_for(1), 0, TB_TEST_DOCK_DP_BW, &lanes); + KUNIT_EXPECT_EQ(test, a->host_rate, 8100); + KUNIT_EXPECT_EQ(test, lanes, 4); + KUNIT_EXPECT_EQ(test, tb_test_host_dp_room(a, lanes), 34650 - 25920); + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(a->host_rate, 1, false), TB_DP_RELINK_NONE); + KUNIT_EXPECT_FALSE(test, tb_dp_relink_up_wanted(1, a->host_rate, a->host_max_rate, false)); +} + +/* TB_HOST_DP_SHARED_RATE, a second display comes: both HBR2 x4. */ +static void tb_test_dp_shared_rate_two(struct kunit *test) +{ + struct tb_tunnel *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct tb_tunnel *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + u32 a_lanes, b_lanes; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + tb_test_host_dp_two_up(test, a, &a_lanes, b, &b_lanes); + KUNIT_EXPECT_EQ(test, a->host_rate, 5400); + KUNIT_EXPECT_EQ(test, a_lanes, 4); + KUNIT_EXPECT_EQ(test, b->host_rate, 5400); + KUNIT_EXPECT_EQ(test, b_lanes, 4); + /* At HBR2 both: neither is set up again */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(a->host_rate, 2, false), TB_DP_RELINK_NONE); + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(b->host_rate, 2, false), TB_DP_RELINK_NONE); +} + +/* + * TB_HOST_DP_SHARED_RATE, the second display goes: the one left is marked, + * stays capped and counted at HBR2 through the debounce, and is set up again + * at HBR3 x4 once it is over. + */ +static void tb_test_dp_shared_rate_leave(struct kunit *test) +{ + struct tb_tunnel *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct tb_tunnel *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + const unsigned long due = 1000 + TB_DP_RELINK_DELAY_MS; + u32 a_lanes, b_lanes; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + tb_test_host_dp_two_up(test, a, &a_lanes, b, &b_lanes); + + /* B goes at 1000 */ + KUNIT_ASSERT_TRUE(test, tb_dp_relink_up_wanted(1, a->host_rate, a->host_max_rate, false)); + a->host_relink = true; + /* Waiting: still capped at HBR2 and counted so */ + KUNIT_EXPECT_EQ(test, a->max_rate, 5400); + KUNIT_EXPECT_EQ(test, tb_test_host_dp_room(a, a_lanes), 34650 - 17280); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due - 1, due), TB_DP_RELINK_WAIT); + + /* Debounce over: set up again, alone, at HBR3 x4 and counted so */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due, due), TB_DP_RELINK_NOW); + tb_test_host_dp_up(test, a, TB_DP_HOST_RATE_ALONE, 0, TB_TEST_DOCK_DP_BW, &a_lanes); + KUNIT_EXPECT_EQ(test, a->host_rate, 8100); + KUNIT_EXPECT_EQ(test, a_lanes, 4); + KUNIT_EXPECT_EQ(test, tb_test_host_dp_room(a, a_lanes), 34650 - 25920); + /* Once: the tunnel set up again is not marked */ + KUNIT_EXPECT_FALSE(test, a->host_relink); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due + 1, due), TB_DP_RELINK_NONE); + KUNIT_EXPECT_FALSE(test, tb_dp_host_drop_relink_up(a)); +} + +/* + * TB_HOST_DP_SHARED_RATE, a second display that comes and goes quickly: no + * re-link while it flaps within the debounce, the debounce starts again at + * each unplug, and a second display soon after a re-link up doubles it. + */ +static void tb_test_dp_shared_rate_flap(struct kunit *test) +{ + struct tb_tunnel *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct tb_tunnel *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + unsigned int delay = TB_DP_RELINK_DELAY_MS; + u32 a_lanes, b_lanes; + unsigned long due; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + tb_test_host_dp_two_up(test, a, &a_lanes, b, &b_lanes); + + /* B goes at 1000 */ + a->host_relink = tb_dp_relink_up_wanted(1, a->host_rate, a->host_max_rate, false); + KUNIT_ASSERT_TRUE(test, a->host_relink); + due = 1000 + delay; + /* ... is back at 1500: A held at HBR2, B HBR2 x4 */ + KUNIT_EXPECT_TRUE(test, tb_dp_host_drop_relink_up(a)); + tb_test_host_dp_up(test, b, tb_dp_host_rate_for(2), a->host_rate, + tb_test_host_dp_room(a, a_lanes), &b_lanes); + KUNIT_EXPECT_EQ(test, b->host_rate, 5400); + KUNIT_EXPECT_EQ(test, b_lanes, 4); + /* ... goes again at 2000: a new debounce, the first one's end changes nothing */ + a->host_relink = tb_dp_relink_up_wanted(1, a->host_rate, a->host_max_rate, false); + KUNIT_ASSERT_TRUE(test, a->host_relink); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due, 2000 + delay), TB_DP_RELINK_WAIT); + due = 2000 + delay; + /* ... is back at 3000, gone again at 3200 */ + KUNIT_EXPECT_TRUE(test, tb_dp_host_drop_relink_up(a)); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 2, a->host_rate, a->host_max_rate, + false, false, due, due), TB_DP_RELINK_NONE); + a->host_relink = tb_dp_relink_up_wanted(1, a->host_rate, a->host_max_rate, false); + due = 3200 + delay; + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due - 1, due), TB_DP_RELINK_WAIT); + + /* Calm until the debounce is over: A set up again at HBR3 x4 */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due, due), TB_DP_RELINK_NOW); + tb_test_host_dp_up(test, a, TB_DP_HOST_RATE_ALONE, 0, TB_TEST_DOCK_DP_BW, &a_lanes); + KUNIT_EXPECT_EQ(test, a->host_rate, 8100); + + /* B is back within a minute: A set up again at HBR2, the next debounce doubled */ + KUNIT_EXPECT_EQ(test, tb_dp_relink_down(a->host_rate, 2, false), TB_DP_RELINK_NOW); + delay = tb_dp_relink_backoff(delay, true); + KUNIT_EXPECT_EQ(test, delay, 5000); + tb_test_host_dp_up(test, a, tb_dp_host_rate_for(2), 0, TB_TEST_DOCK_DP_BW, &a_lanes); + tb_test_host_dp_up(test, b, tb_dp_host_rate_for(2), a->host_rate, + tb_test_host_dp_room(a, a_lanes), &b_lanes); + KUNIT_EXPECT_EQ(test, a->host_rate, 5400); + KUNIT_EXPECT_EQ(test, a_lanes, 4); + KUNIT_EXPECT_EQ(test, b->host_rate, 5400); + KUNIT_EXPECT_EQ(test, b_lanes, 4); + + /* B goes once more at 10000: HBR3 again only after the doubled debounce */ + a->host_relink = tb_dp_relink_up_wanted(1, a->host_rate, a->host_max_rate, false); + due = 10000 + delay; + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, 10000 + TB_DP_RELINK_DELAY_MS, due), + TB_DP_RELINK_WAIT); + KUNIT_EXPECT_EQ(test, tb_dp_relink_up(a->host_relink, 1, a->host_rate, a->host_max_rate, + false, false, due, due), TB_DP_RELINK_NOW); +} + +/* + * A domain with the software connection manager, a host router whose DP IN + * adapters share one link rate (TB_HOST_DP_SHARED_RATE) and a USB4 device + * router with two DP OUT adapters. Adapter registers (TB_CFG_PORT, from the + * adapter capability) are read from @regs: [0] host, [1] device router. + */ +struct tb_test_cm { + struct tb_nhi nhi; + struct tb *tb; + struct tb_switch *host, *dev; + u32 regs[2][20][16]; +}; + +#define TB_TEST_CM_CAP 0x10 +#define TB_TEST_CM_HBR3_X4 (DP_COMMON_CAP_RATE_HBR3 << DP_COMMON_CAP_RATE_SHIFT | \ + DP_COMMON_CAP_4_LANES << DP_COMMON_CAP_LANES_SHIFT) + +static int tb_test_cm_cfg_read(struct tb_ctl *ctl, void *buffer, u64 route, u32 port, + enum tb_cfg_space space, u32 offset, u32 length) +{ + struct tb_test_cm *cm = kunit_get_current_test()->priv; + u32 (*regs)[16] = cm->regs[route ? 1 : 0]; + + if (space != TB_CFG_PORT || port >= ARRAY_SIZE(cm->regs[0]) || + offset < TB_TEST_CM_CAP || offset - TB_TEST_CM_CAP + length > ARRAY_SIZE(regs[0])) + return -EIO; + memcpy(buffer, ®s[port][offset - TB_TEST_CM_CAP], length * sizeof(u32)); + return 0; +} + +static struct tb_test_cm *tb_test_cm_alloc(struct kunit *test) +{ + struct tb_test_cm *cm; + struct tb *tb; + + cm = kunit_kzalloc(test, sizeof(*cm), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, cm); + tb = kunit_kzalloc(test, sizeof(*tb) + tb_test_cm_size(), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tb); + cm->nhi.host_dp_policy = TB_HOST_DP_NOTIFY | TB_HOST_DP_SHARED_RATE; + tb->nhi = &cm->nhi; + device_initialize(&tb->dev); + tb->dev.release = tb_test_dprx_release; + mutex_init(&tb->lock); + tb_test_cm_init(tb); + cm->tb = tb; + + cm->host = alloc_host_usb4(test); + KUNIT_ASSERT_NOT_NULL(test, cm->host); + cm->dev = alloc_dev_usb4(test, cm->host, 0x1, true); + KUNIT_ASSERT_NOT_NULL(test, cm->dev); + cm->host->tb = tb; + cm->dev->tb = tb; + cm->host->config.thunderbolt_version = FIELD_PREP(USB4_VERSION_MAJOR_MASK, 1); + cm->dev->config.thunderbolt_version = FIELD_PREP(USB4_VERSION_MAJOR_MASK, 1); + cm->host->ports[5].cap_adap = TB_TEST_CM_CAP; + cm->host->ports[6].cap_adap = TB_TEST_CM_CAP; + cm->dev->ports[13].cap_adap = TB_TEST_CM_CAP; + cm->dev->ports[14].cap_adap = TB_TEST_CM_CAP; + + test->priv = cm; + kunit_activate_static_stub(test, tb_cfg_read, tb_test_cm_cfg_read); + return cm; +} + +/* + * What the DP IN adapter @in reports once its DPRX is done: HBR3 x4, the + * rate both adapters take, in its caps or (@bw_mode) as its allocated + * bandwidth (26000 Mb/s at a granularity of 250 Mb/s). + */ +static void tb_test_cm_reports(struct tb_test_cm *cm, struct tb_port *in, bool bw_mode) +{ + u32 *regs = cm->regs[0][in->port]; + + regs[DP_LOCAL_CAP] = TB_TEST_CM_HBR3_X4; + regs[DP_COMMON_CAP] = TB_TEST_CM_HBR3_X4 | DP_COMMON_CAP_DPRX_DONE; + regs[ADP_DP_CS_2] = 0; + regs[ADP_DP_CS_8] = bw_mode ? ADP_DP_CS_8_DPME : 0; + regs[DP_STATUS] = bw_mode ? 104 << DP_STATUS_ALLOCATED_BW_SHIFT : 0; +} + +/* + * A host DP tunnel from @in to @out as the capabilities exchange leaves it: + * capped at @rate_limit (Mb/s per lane), up at @rate. Listed by the + * connection manager, its adapters enabled. + */ +static struct tb_tunnel *tb_test_cm_dp_tunnel(struct kunit *test, struct tb_test_cm *cm, + struct tb_port *in, struct tb_port *out, + u32 rate_limit, u32 rate) +{ + struct tb_tunnel *tunnel; + int i; + + tunnel = tb_tunnel_alloc_dp(cm->tb, in, out, 1, 0, 0, tb_test_dp_tunnel_active, NULL); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + tunnel->host_rate_limit = rate_limit; + tunnel->max_rate = min(rate_limit, rate); + tunnel->host_rate = rate; + tunnel->state = TB_TUNNEL_ACTIVE; + for (i = 0; i < tunnel->npaths; i++) + tunnel->paths[i]->activated = true; + list_add_tail(&tunnel->list, tb_test_cm_tunnels(cm->tb)); + cm->regs[0][in->port][ADP_DP_CS_0] = ADP_DP_CS_0_VE | ADP_DP_CS_0_AE; + cm->regs[1][out->port][ADP_DP_CS_0] = ADP_DP_CS_0_VE | ADP_DP_CS_0_AE; + return tunnel; +} + +static void tb_test_cm_dp_tunnel_put(struct tb_tunnel *tunnel) +{ + int i; + + list_del(&tunnel->list); + for (i = 0; i < tunnel->npaths; i++) + tunnel->paths[i]->activated = false; + tunnel->state = TB_TUNNEL_INACTIVE; + tb_tunnel_put(tunnel); +} + +/* + * The DP bandwidth the connection manager counts on the host router's lane + * adapter for a new tunnel from host DP IN 6 to device DP OUT 14. + */ +static int tb_test_cm_counted(struct kunit *test, struct tb_test_cm *cm) +{ + int up, down; + + KUNIT_EXPECT_EQ(test, tb_consumed_dp_bandwidth(cm->tb, &cm->host->ports[6], + &cm->dev->ports[14], &cm->host->ports[1], + &up, &down), 0); + KUNIT_EXPECT_EQ(test, up, 0); + return down; +} + +/* + * TB_HOST_DP_SHARED_RATE, through the connection manager's DP bandwidth + * accounting: a tunnel capped at HBR2 whose DP IN adapter reports HBR3 x4 + * once its DPRX is done is counted at HBR2 x4, from its caps and from its + * allocated bandwidth; capped at HBR3, at HBR3 x4. + */ +static void tb_test_dp_shared_rate_counted(struct kunit *test) +{ + struct tb_test_cm *cm = tb_test_cm_alloc(test); + struct tb_port *in = &cm->host->ports[5]; + struct tb_tunnel *a; + + a = tb_test_cm_dp_tunnel(test, cm, in, &cm->dev->ports[13], 5400, 5400); + KUNIT_ASSERT_TRUE(test, tb_tunnel_port_on_path(a, &cm->host->ports[1])); + + /* Caps: HBR2 x4, not the HBR3 x4 reported */ + tb_test_cm_reports(cm, in, false); + KUNIT_EXPECT_EQ(test, tb_test_cm_counted(test, cm), 17280); + a->max_rate = 8100; + KUNIT_EXPECT_EQ(test, tb_test_cm_counted(test, cm), 25920); + + /* Allocated bandwidth: HBR2 x4, not the 26000 Mb/s reported */ + tb_test_cm_reports(cm, in, true); + a->bw_mode = true; + a->max_rate = 5400; + KUNIT_EXPECT_EQ(test, tb_test_cm_counted(test, cm), 17280); + a->max_rate = 8100; + KUNIT_EXPECT_EQ(test, tb_test_cm_counted(test, cm), 25920); + + tb_test_cm_dp_tunnel_put(a); + put_device(&cm->tb->dev); +} + +/* + * TB_HOST_DP_SHARED_RATE, through the connection manager: A (host DP IN 5 to + * device DP OUT 13) is left alone at HBR2 when B (DP IN 6 to DP OUT 14) goes, + * and C is plugged into DP OUT 14 during the debounce. Pairing drops A's + * re-link up and holds A at HBR2 before C's bandwidth is worked out, and the + * debounce timer is stopped unless deferred pairing (@deferred) needs the + * re-link work. A's DP IN adapter reports HBR3 x4, in its caps or (@bw_mode) + * as its allocated bandwidth; it is counted at HBR2 x4 and C gets HBR2 x4. + */ +static void tb_test_dp_shared_rate_arrival(struct kunit *test, bool bw_mode, bool deferred) +{ + struct tb_test_cm *cm = tb_test_cm_alloc(test); + struct list_head *resources = tb_test_cm_dp_resources(cm->tb); + struct tb_port *in = &cm->host->ports[5]; + struct tb_tunnel *a; + u32 rate, lanes; + int counted; + + /* + * The debounce timer is real: should it expire before the end, its + * work runs on this queue and stops at the missing root switch. + */ + cm->tb->wq = alloc_ordered_workqueue("tb-cm-test", 0); + KUNIT_ASSERT_NOT_NULL(test, cm->tb->wq); + + a = tb_test_cm_dp_tunnel(test, cm, in, &cm->dev->ports[13], 5400, 5400); + a->host_max_rate = 8100; + a->bw_mode = bw_mode; + tb_test_cm_reports(cm, in, bw_mode); + list_add_tail(&cm->host->ports[5].list, resources); + list_add_tail(&cm->host->ports[6].list, resources); + list_add_tail(&cm->dev->ports[13].list, resources); + + mutex_lock(&cm->tb->lock); + /* B went (two were up): A is marked, the debounce armed */ + tb_dp_host_tunnel_gone(cm->tb, 2); + KUNIT_EXPECT_TRUE(test, a->host_relink); + KUNIT_EXPECT_TRUE(test, tb_test_cm_relink_armed(cm->tb)); + KUNIT_EXPECT_EQ(test, kref_read(&cm->tb->dev.kobj.kref), 2U); + tb_test_cm_set_relink_deferred(cm->tb, deferred); + + /* C comes on DP OUT 14: pairing takes HBR2 for it, A's re-link dropped */ + list_add_tail(&cm->dev->ports[14].list, resources); + KUNIT_EXPECT_EQ(test, tb_dp_host_rate_limit(cm->tb), 5400U); + KUNIT_EXPECT_FALSE(test, a->host_relink); + KUNIT_EXPECT_EQ(test, a->host_rate_limit, 5400U); + KUNIT_EXPECT_EQ(test, a->max_rate, 5400U); + KUNIT_EXPECT_EQ(test, tb_test_cm_relink_armed(cm->tb), deferred); + KUNIT_EXPECT_EQ(test, kref_read(&cm->tb->dev.kobj.kref), deferred ? 2U : 1U); + + /* C's bandwidth: A counted at HBR2 x4, C at A's rate gets x4 */ + counted = tb_test_cm_counted(test, cm); + KUNIT_EXPECT_EQ(test, counted, 17280); + KUNIT_EXPECT_EQ(test, tb_dp_host_reduce_bandwidth(TB_TEST_DOCK_DP_BW - counted, 5400, 4, + 5400, 4, a->host_rate, 5400, + &rate, &lanes), 0); + KUNIT_EXPECT_EQ(test, rate, 5400); + KUNIT_EXPECT_EQ(test, lanes, 4); + mutex_unlock(&cm->tb->lock); + + /* Outside the domain lock, which the work takes: drain it, then the queue */ + tb_test_cm_stop(cm->tb); + destroy_workqueue(cm->tb->wq); + KUNIT_EXPECT_EQ(test, kref_read(&cm->tb->dev.kobj.kref), 1U); + INIT_LIST_HEAD(resources); + tb_test_cm_dp_tunnel_put(a); + put_device(&cm->tb->dev); +} + +/* + * TB_HOST_DP_SHARED_RATE, a second display comes during the debounce: it + * gets HBR2 x4 and the debounce timer is stopped, whether the tunnel left + * alone is counted from its caps or from its allocated bandwidth. + */ +static void tb_test_dp_shared_rate_debounce(struct kunit *test) +{ + tb_test_dp_shared_rate_arrival(test, false, false); + tb_test_dp_shared_rate_arrival(test, true, false); +} + +/* + * As tb_test_dp_shared_rate_debounce(), with pairing deferred for a system + * sleep transition: the re-link up is dropped, the re-link work kept armed. + */ +static void tb_test_dp_shared_rate_debounce_deferred(struct kunit *test) +{ + tb_test_dp_shared_rate_arrival(test, false, true); +} + static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_apple_right_dpin_wiring), KUNIT_CASE(tb_test_dp_reduce_bandwidth_shared), + KUNIT_CASE(tb_test_dp_shared_rate_single), + KUNIT_CASE(tb_test_dp_shared_rate_second), + KUNIT_CASE(tb_test_dp_shared_rate_relink_up), + KUNIT_CASE(tb_test_dp_shared_rate_recover), + KUNIT_CASE(tb_test_dp_shared_rate_lone), + KUNIT_CASE(tb_test_dp_shared_rate_two), + KUNIT_CASE(tb_test_dp_shared_rate_leave), + KUNIT_CASE(tb_test_dp_shared_rate_debounce), + KUNIT_CASE(tb_test_dp_shared_rate_debounce_deferred), + KUNIT_CASE(tb_test_dp_shared_rate_flap), + KUNIT_CASE(tb_test_dp_shared_rate_sleep), + KUNIT_CASE(tb_test_dp_shared_rate_retry), + KUNIT_CASE(tb_test_dp_shared_rate_counted), KUNIT_CASE(tb_test_dp_generation), KUNIT_CASE(tb_test_dp_dprx_deferred_first), KUNIT_CASE(tb_test_dp_dprx_cancel), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index f20b9d4d84a5e..947a297f4d09e 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -919,7 +919,7 @@ static int tb_dp_reduce_bandwidth(int max_bw, u32 in_rate, u32 in_lanes, /* * As tb_dp_reduce_bandwidth(), for a host whose DP IN streams of a PHY share - * one link rate (TB_HOST_DP_HBR2_MAX): the rate is kept and the lanes are + * one link rate (TB_HOST_DP_SHARED_RATE): the rate is kept and the lanes are * reduced (x4, x2, x1). @shared_rate, the rate of a DP tunnel already up on * the same host router, is the only rate allowed (0: none). Without one, the * highest rate both sides take comes first, then the usual combinations. @@ -948,17 +948,38 @@ int tb_dp_reduce_bandwidth_shared(int max_bw, u32 in_rate, u32 in_lanes, u32 out new_rate, new_lanes); } +/* + * A DP tunnel under TB_HOST_DP_SHARED_RATE that has to be reduced, to fit + * @max_bw or to take @shared_rate. Next to another DP tunnel (@shared_rate), + * or alone but held at HBR2 for one on its way (@rate_limit), the rate is + * kept and the lanes go first, see tb_dp_reduce_bandwidth_shared(). Alone at + * up to HBR3, the combination with the most bandwidth is taken, as without + * the policy: HBR2 x4 rather than HBR3 x2. + */ +int tb_dp_host_reduce_bandwidth(int max_bw, u32 in_rate, u32 in_lanes, u32 out_rate, + u32 out_lanes, u32 shared_rate, u32 rate_limit, + u32 *new_rate, u32 *new_lanes) +{ + if (!shared_rate && rate_limit > TB_DP_HOST_RATE_SHARED) + return tb_dp_reduce_bandwidth(max_bw, in_rate, in_lanes, out_rate, + out_lanes, new_rate, new_lanes); + return tb_dp_reduce_bandwidth_shared(max_bw, in_rate, in_lanes, out_rate, + out_lanes, shared_rate, new_rate, new_lanes); +} + /* * The highest DP link rate (Mb/s per lane) a tunnel from @in offers its - * DPTX, 0 for no cap: HBR2 from the DP IN adapters of a host that needs - * one link rate for both DP IN streams of a PHY (TB_HOST_DP_HBR2_MAX). + * DPTX, 0 for no cap. From the DP IN adapters of a host that needs one link + * rate for both DP IN streams of a PHY (TB_HOST_DP_SHARED_RATE), the one the + * connection manager set: HBR3 alone on the host router, HBR2 next to + * another DP tunnel; HBR2 if it set none. */ static u32 tb_dp_host_rate_cap(struct tb_tunnel *tunnel, struct tb_port *in) { if (!tb_port_is_apple_host_dpin(in) || - !tb_host_dp_policy(tunnel->tb->nhi, TB_HOST_DP_HBR2_MAX)) + !tb_host_dp_policy(tunnel->tb->nhi, TB_HOST_DP_SHARED_RATE)) return 0; - return 5400; + return tunnel->host_rate_limit ?: TB_DP_HOST_RATE_SHARED; } /* @val (a DP_LOCAL_CAP) with its rate capped at the tunnel's cap, if any. */ @@ -982,6 +1003,21 @@ int tb_dp_cap_bandwidth(struct tb_tunnel *tunnel, int bw) return tunnel->max_bw ? min(bw, tunnel->max_bw) : bw; } +/* + * The bandwidth (Mb/s) a DP tunnel capped at @max_rate (Mb/s per lane, 0 + * for no cap) is counted at, from the @rate and @lanes read from its DP IN + * adapter. Under TB_HOST_DP_SHARED_RATE the tunnel never runs above its + * cap, but once its DPRX is done what the adapter reports is not bounded by + * it: a tunnel held at HBR2 between adapters that take HBR3 is counted at + * HBR2 all the same, so that a second one next to it gets what HBR2 leaves. + */ +unsigned int tb_dp_host_counted_bandwidth(u32 max_rate, u32 rate, u32 lanes) +{ + if (max_rate && rate > max_rate) + rate = max_rate; + return tb_dp_bandwidth(rate, lanes); +} + static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) { u32 out_dp_cap, out_rate, out_lanes, in_dp_cap, in_rate, in_lanes, bw; @@ -994,6 +1030,7 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) tunnel->max_rate = tb_dp_host_rate_cap(tunnel, in); tunnel->max_bw = 0; tunnel->host_rate = 0; + tunnel->host_max_rate = 0; /* * Copy DP_LOCAL_CAP register to DP_REMOTE_CAP register for @@ -1021,6 +1058,11 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) if (ret) return ret; + /* What the tunnel could get alone on the host router: HBR3 at most. */ + if (tunnel->max_rate) + tunnel->host_max_rate = min(tb_dp_cap_get_rate(in_dp_cap), + tb_dp_cap_get_rate(out_dp_cap)); + /* * Both sides see the capped rate: the DP OUT through its remote caps, * the DPTX through the DP IN's. @@ -1065,7 +1107,7 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) max_bw = tunnel->max_up; /* - * Under TB_HOST_DP_HBR2_MAX a second DP tunnel from the host router + * Under TB_HOST_DP_SHARED_RATE a second DP tunnel from the host router * takes the link rate of the one already up, whatever bandwidth there * is: both DP IN streams of the PHY run at one rate. */ @@ -1077,9 +1119,10 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) u32 new_rate, new_lanes, new_bw; if (tunnel->max_rate) - ret = tb_dp_reduce_bandwidth_shared(max_bw, in_rate, in_lanes, - out_rate, out_lanes, shared_rate, - &new_rate, &new_lanes); + ret = tb_dp_host_reduce_bandwidth(max_bw, in_rate, in_lanes, + out_rate, out_lanes, shared_rate, + tunnel->max_rate, &new_rate, + &new_lanes); else ret = tb_dp_reduce_bandwidth(max_bw, in_rate, in_lanes, out_rate, out_lanes, &new_rate, @@ -1684,6 +1727,9 @@ static int tb_dp_bandwidth_mode_consumed_bandwidth(struct tb_tunnel *tunnel, return ret; if (allocated_bw == max_bw_rounded) allocated_bw = ret; + /* Capped: no more than the capped rate takes, see tb_dp_host_counted_bandwidth(). */ + if (tunnel->max_rate && allocated_bw > ret) + allocated_bw = ret; if (tb_tunnel_direction_downstream(tunnel)) { *consumed_up = 0; @@ -1878,9 +1924,9 @@ static int tb_dp_consumed_bandwidth(struct tb_tunnel *tunnel, int *consumed_up, if (tb_tunnel_direction_downstream(tunnel)) { *consumed_up = 0; - *consumed_down = tb_dp_bandwidth(rate, lanes); + *consumed_down = tb_dp_host_counted_bandwidth(tunnel->max_rate, rate, lanes); } else { - *consumed_up = tb_dp_bandwidth(rate, lanes); + *consumed_up = tb_dp_host_counted_bandwidth(tunnel->max_rate, rate, lanes); *consumed_down = 0; } diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 4cf4e12b77e74..fb5114bbc7563 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -109,8 +109,19 @@ struct tb_tunnel { /* host cap of a DP tunnel's link rate (Mb/s per lane, 0 none), and its bandwidth */ u32 max_rate; int max_bw; - /* under TB_HOST_DP_HBR2_MAX: the link rate the DPTX was offered (Mb/s per lane) */ + /* + * Under TB_HOST_DP_SHARED_RATE (Mb/s per lane): the highest link rate + * the connection manager allows, the one the DPTX was offered, and the + * one both adapters take without the policy. @host_relink: to be set + * up again at a higher rate once the debounce is over (alone now); + * @host_relinked_up: set up so, and set up at HBR2 again if it does + * not come up. + */ + u32 host_rate_limit; u32 host_rate; + u32 host_max_rate; + bool host_relink; + bool host_relinked_up; bool dprx_started; bool dprx_canceled; u64 host_dp_generation; @@ -140,9 +151,74 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, void tb_dp_tunnel_cancel_dprx(struct tb_tunnel *tunnel); bool tb_apple_nhi_needs_bw_grant(struct tb_nhi *nhi); int tb_dp_cap_bandwidth(struct tb_tunnel *tunnel, int bw); +unsigned int tb_dp_host_counted_bandwidth(u32 max_rate, u32 rate, u32 lanes); int tb_dp_reduce_bandwidth_shared(int max_bw, u32 in_rate, u32 in_lanes, u32 out_rate, u32 out_lanes, u32 shared_rate, u32 *new_rate, u32 *new_lanes); +int tb_dp_host_reduce_bandwidth(int max_bw, u32 in_rate, u32 in_lanes, u32 out_rate, + u32 out_lanes, u32 shared_rate, u32 rate_limit, + u32 *new_rate, u32 *new_lanes); u32 tb_dp_host_shared_rate(const struct tb_tunnel *tunnel); + +/* TB_HOST_DP_SHARED_RATE link rates (Mb/s per lane) */ +#define TB_DP_HOST_RATE_ALONE 8100 /* HBR3: one DP tunnel on the host router */ +#define TB_DP_HOST_RATE_SHARED 5400 /* HBR2: more than one */ + +/** + * enum tb_dp_relink - Whether a host DP tunnel is set up again at another rate + * @TB_DP_RELINK_NONE: It is not + * @TB_DP_RELINK_WAIT: Not now, it is decided again later + * @TB_DP_RELINK_NOW: It is, now + */ +enum tb_dp_relink { + TB_DP_RELINK_NONE, + TB_DP_RELINK_WAIT, + TB_DP_RELINK_NOW, +}; + +/* + * TB_HOST_DP_SHARED_RATE re-link work (ms): how long it waits to be decided + * again (also while system sleep is in progress), and the first of its + * TB_DP_RELINK_RETRIES retries, doubled each time, when the domain does not + * runtime resume. + */ +#define TB_DP_RELINK_DELAY_MS 2500 +#define TB_DP_RELINK_POLL_MS 500 +#define TB_DP_RELINK_RETRIES 5 + +/* + * A re-link to a higher rate waits TB_DP_RELINK_DELAY_MS (the debounce), + * doubled up to TB_DP_RELINK_DELAY_MAX_MS while a second DP tunnel takes + * the rate back down within TB_DP_RELINK_CALM_MS of such a re-link. + */ +#define TB_DP_RELINK_DELAY_MAX_MS 40000 +#define TB_DP_RELINK_CALM_MS 60000 + +/** + * struct tb_dp_pm_state - System sleep transitions, for host DP re-links + * @sleeping: From the start of a transition (its PREPARE event, before any + * other notifier) until it is over, its POST callbacks and any + * rollback included (tb_dp_pm_reopen()) + * @reopen: The POST event came: reopen once the transition is over + */ +struct tb_dp_pm_state { + bool sleeping; + bool reopen; +}; + +u32 tb_dp_host_rate_for(unsigned int tunnels); +enum tb_dp_relink tb_dp_relink_down(u32 rate, unsigned int tunnels, bool sleeping); +bool tb_dp_pm_event(struct tb_dp_pm_state *s, unsigned long event); +void tb_dp_pm_reopen(struct tb_dp_pm_state *s); +unsigned int tb_dp_relink_retry_delay(unsigned int tries); +u32 tb_dp_host_rate_after(unsigned int tunnels, bool recovering); +bool tb_dp_relink_up_wanted(unsigned int tunnels, u32 rate, u32 max_rate, bool sleeping); +enum tb_dp_relink tb_dp_relink_up(bool marked, unsigned int tunnels, u32 rate, u32 max_rate, + bool pairable, bool sleeping, unsigned long now, + unsigned long due); +unsigned int tb_dp_relink_backoff(unsigned int delay_ms, bool flapping); +enum tb_dp_relink tb_dp_relink_activation(bool relinked_up, bool active); +bool tb_dp_host_drop_relink_up(struct tb_tunnel *tunnel); + struct tb_tunnel *tb_tunnel_alloc_dp(struct tb *tb, struct tb_port *in, struct tb_port *out, int link_nr, int max_up, int max_down, diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index bce7cfa4ef7fc..64eae8cb4e207 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -32,6 +32,13 @@ # warning. We don't need to peek into it anyway. --opaque-type spinlock +# With CONFIG_SCHEDSTATS=n, `struct sched_statistics` is an empty +# `____cacheline_aligned` struct. bindgen (0.73.2 and earlier) drops the +# explicit alignment of a zero-sized non-opaque struct, which moves +# `task_struct::stats` and every later field, including `mm`. Opaque types take +# their alignment from Clang's layout, and Rust does not access these fields. +--opaque-type sched_statistics + # enums that appear in indirect function calls should specify a cfi type --newtype-enum lru_status --with-attribute-custom-enum=lru_status='#[cfi_encoding="10lru_status"]' diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 51e51790cb4f6..6a13017bacb6c 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -240,7 +240,7 @@ if [ -n "$bindgen_alignment_code" ] || echo >&2 "*** Rust bindings generator '$BINDGEN'" echo >&2 "*** does not preserve empty struct alignment." fi - echo >&2 "*** Default Asahi builds can fail the sched_statistics layout assertion." + echo >&2 "*** Types containing an aligned empty struct can fail their layout assertions." echo >&2 "*** Prepare the corrected generator before configuring the kernel:" echo >&2 "*** python3 tools/aurora-bindgen/build.py" echo >&2 "*** See Documentation/rust/bindgen-layout-checks.rst for details." diff --git a/tools/asahi/tb-dp-repair-tests.py b/tools/asahi/tb-dp-repair-tests.py index 396fdefb5ac04..be775d7618095 100644 --- a/tools/asahi/tb-dp-repair-tests.py +++ b/tools/asahi/tb-dp-repair-tests.py @@ -45,14 +45,14 @@ def function(text, name): #define BIT(n) (1UL<<(n)) #define QUIRK_NO_SYSTEM_SLEEP BIT(5) struct tb {int lock;struct device dev;void*wq;struct tb_switch*root_switch;struct tb_nhi*nhi;}; -struct tb_cm {bool hotplug_active;struct work_struct dp_tunnel_work;bool dp_tunnel_pending;}; +struct tb_cm {bool hotplug_active;struct work_struct dp_tunnel_work;bool dp_tunnel_pending;struct work_struct dp_relink_run;bool dp_host_hold_shared;}; struct domain {struct tb tb;struct tb_cm cm;}; _Static_assert(offsetof(struct domain,cm)==sizeof(struct tb),"mock private storage adjacency"); #define INIT_WORK(w,handler) ((w)->fn=(handler)) struct kref {int count;}; struct tb_path {bool reserved;}; enum {TB_TUNNEL_DP,TB_TUNNEL_USB3}; -struct tb_tunnel {struct kref kref;struct tb*tb;struct tb_port*src_port,*dst_port;int npaths;void(*destroy)(struct tb_tunnel*);struct list_head list;int type;bool dprx_started,dprx_canceled;void(*callback)(struct tb_tunnel*,void*);void*callback_data;struct tb_path*paths[3];}; +struct tb_tunnel {struct kref kref;struct tb*tb;struct tb_port*src_port,*dst_port;int npaths;void(*destroy)(struct tb_tunnel*);struct list_head list;int type;bool dprx_started,dprx_canceled,host_relinked_up;void(*callback)(struct tb_tunnel*,void*);void*callback_data;struct tb_path*paths[3];}; static int tb_tunnel_lock,occupied,alloc_calls,alloc_failures,alloc_successes,freed; static int domain_refs,pm_refs,pm_calls,pm_error,stop_during_pm,requeue_once,lock_depth; static int restore_calls,scan_calls,unplugged_xdomains; @@ -114,6 +114,13 @@ def function(text, name): static void tb_domain_put(struct tb*t){CHECK(domain_refs>1);domain_refs--;} static bool queue_work(void*wq,struct work_struct*w){if(w->pending)return false;w->pending=true;return true;} static void tb_queue_dp_tunnels(struct tb*tb); +/* The TB_HOST_DP_SHARED_RATE re-link work is not queued in these cases. */ +static void tb_dp_relink_run(struct work_struct*w){CHECK(0);} +/* No tunnel here was re-linked at a higher rate, and none waits for it. */ +#define TB_DP_HOST_RATE_SHARED 5400 +enum tb_dp_relink {TB_DP_RELINK_NONE,TB_DP_RELINK_WAIT,TB_DP_RELINK_NOW}; +static enum tb_dp_relink tb_dp_relink_activation(bool relinked_up,bool active){CHECK(!relinked_up);return TB_DP_RELINK_NONE;} +static void tb_dp_relink_replay(struct tb*t){CHECK(lock_depth==1);} static void tb_tunnel_dp(struct tb*t){ CHECK(lock_depth==1 && domain.cm.hotplug_active && t->root_switch); alloc_calls++;if(occupied)alloc_failures++;else alloc_successes++; @@ -298,7 +305,7 @@ def main(): raise ValueError('DPRX reference-release order changed; review extraction') production = '\n'.join(parts.values()) probe = (ROOT / 'drivers/thunderbolt/tb.c').read_text() - probe = probe[probe.index('struct tb *tb_probe(struct tb_nhi *nhi)'):] + probe = probe[probe.index('static void tb_cm_init(struct tb_cm *tcm)'):] initializers = re.findall(r'INIT_WORK\(&tcm->\w+,\s*\w+\);', probe) production += '\nstatic void init_fixture(struct tb_cm*tcm){\n' + '\n'.join(initializers) + '\n}\n' results = {}