diff --git a/Documentation/ABI/testing/sysfs-driver-macsmc-reboot b/Documentation/ABI/testing/sysfs-driver-macsmc-reboot new file mode 100644 index 0000000000000..8a485592fedba --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-macsmc-reboot @@ -0,0 +1,16 @@ +What: /sys/bus/platform/drivers/macsmc-reboot//ac_power_mode +Date: November 2026 +KernelVersion: 7.4 +Description: + What the machine does when external power arrives, as kept in + the PMU (present only when the device tree gives the reboot + node a pm_setting cell). Reading lists the modes with the + current one in brackets; writing a mode name sets it: + + off: + stay off + restore: + power on after a power loss if the machine was on + always: + power on whenever power is connected (the 2024 and later + desktops; other machines may ignore it) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 3db18e058dce3..459f91fceb7c8 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -97,17 +97,39 @@ properties: clocks: minItems: 1 - maxItems: 2 + maxItems: 64 clock-names: minItems: 1 - items: - - const: clock-154 - - const: clock-194 + maxItems: 64 + + apple,clock-ids: + oneOf: + - type: boolean + - $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 64 + description: | + Ordered firmware provider clock identifiers. Callback clock indices + select the corresponding entry in clocks. An explicitly empty table + is allowed when the provider has no clocks. + + apple,firmware-physical-regs: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 1 + maxItems: 10 + description: | + Coprocessor physical base addresses in display register-table order, + including the bandwidth scratch and optional doorbell windows. + The number of entries must match the complete register table. + Omit on identity-mapped systems; reg supplies the physical view. memory-region: minItems: 1 maxItems: 64 + description: + Boot framebuffer and reserved mappings. A T6050 external processor + without a boot framebuffer may omit these regions and their names. memory-region-names: minItems: 1 @@ -295,17 +317,18 @@ properties: mux-controls: $ref: /schemas/types.yaml#/definitions/phandle-array minItems: 1 - maxItems: 10 + maxItems: 21 description: | T6030 and T8132 external DCPs may also name the two DP IN tunnel states of each Type-C crossbar, in addition to the fixed output and Type-C - routes. + routes. T6050 has four DP IN states per crossbar and up to four + Type-C routes, plus a fixed output. mux-control-names: minItems: 1 - maxItems: 10 + maxItems: 21 items: - pattern: '^(dp-xbar|typec[0-9]+(-dpin[01])?)$' + pattern: '^(dp-xbar|typec[0-9]+(-dpin[0-3])?)$' mux-index: $ref: /schemas/types.yaml#/definitions/uint32 @@ -347,7 +370,9 @@ properties: properties: compatible: oneOf: - - const: apple,panel + - enum: + - apple,panel + - apple,panel-mini-led - items: - pattern: '^apple,panel-j[0-9]+$' - const: apple,panel @@ -411,7 +436,24 @@ allOf: properties: compatible: contains: - enum: ['apple,t6030-dcpext', 'apple,t8132-dcpext'] + enum: ['apple,t6050-dcp', 'apple,t6050-dcpext'] + required: + - compatible + then: + properties: + mux-controls: + maxItems: 10 + mux-control-names: + maxItems: 10 + items: + pattern: '^(dp-xbar|typec[0-9]+(-dpin[01])?)$' + - if: + not: + properties: + compatible: + contains: + enum: ['apple,t6030-dcpext', 'apple,t8132-dcpext', + 'apple,t6050-dcp', 'apple,t6050-dcpext'] required: - compatible then: @@ -438,12 +480,50 @@ allOf: then: required: - apple,bw-scratch + - if: + properties: + compatible: + contains: + enum: ['apple,t6050-dcp', 'apple,t6050-dcpext'] + then: + required: + - apple,clock-ids + - apple,firmware-compat + dependencies: + memory-region: [memory-region-names] + memory-region-names: [memory-region] + allOf: + - if: + properties: + compatible: + contains: + const: apple,t6050-dcp + then: + required: + - memory-region + - memory-region-names + properties: + apple,firmware-compat: + items: + - const: 27 + - const: 0 + - const: 0 + else: + properties: + clocks: + maxItems: 2 + clock-names: + items: + - const: clock-154 + - const: clock-194 + apple,clock-ids: false + apple,firmware-physical-regs: false - if: not: properties: compatible: contains: - enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext', 'apple,t6050-dcp', 'apple,t6050-dcpext'] then: properties: apple,power-lut-data-x: false @@ -455,7 +535,8 @@ allOf: properties: compatible: contains: - enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext', + 'apple,t6050-dcp', 'apple,t6050-dcpext'] then: properties: resets: false diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml index 0d3e74d6a804c..fab8c397fe94a 100644 --- a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -39,6 +39,11 @@ properties: maxItems: 64 description: Reserved memory retaining the inherited display mappings. + memory-region-names: + items: + - const: framebuffer + description: Identifies the boot framebuffer reservation supplied by the loader. + apple,j713-inherited-mappings: type: boolean description: The loader validated the inherited J713 SID0 scanout mapping. diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 202544c07b0bb..ea339c4623b44 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -22,6 +22,9 @@ properties: - apple,agx-t6050 - apple,agx-t8132 - apple,agx-t8140 + - items: + - const: apple,agx-t6050 + - const: apple,agx-g17s - items: - const: apple,agx-t8103 - const: apple,agx-g13g @@ -51,8 +54,8 @@ properties: interrupts: maxItems: 8 description: - The GPU block's interrupt outputs, as the ADT lists them. Only - described on T6050, whose GPU is not driven from Linux yet. + Interrupt outputs of the raw T6050 GPU block. The coprocessor-driven + GPU uses its mailbox interrupts instead. dma-noncoherent: true @@ -110,6 +113,21 @@ properties: operating-points-v2: true + apple,csafr-min-sram-microvolt: + maxItems: 1 + description: Loader-filled SRAM voltage floor for the CS and AFR domains. + + apple,aon-ptd: + $ref: /schemas/types.yaml#/definitions/uint32-array + oneOf: + - minItems: 9 + maxItems: 9 + - minItems: 18 + maxItems: 18 + description: + Loader-filled power telemetry records, three per die. Each record is + an identifier followed by two configuration words. + apple,firmware-version: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -218,6 +236,7 @@ allOf: compatible: contains: const: apple,agx-t6050 + maxItems: 1 then: description: The T6050 GPU block, described from the ADT /arm-io/sgx node. Its @@ -242,6 +261,48 @@ allOf: required: - mboxes - memory-region + - if: + properties: + compatible: + contains: + const: apple,agx-g17s + then: + properties: + reg: + items: + - description: GFX coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + items: + - description: GFX coprocessor mailbox + - description: GFX1 coprocessor mailbox + memory-region: + minItems: 6 + memory-region-names: + items: + - const: ttbs + - const: pagetables + - const: handoff + - const: fw-data + - const: l2-pagetables + - const: fw-text + required: + - reg-names + - power-domains + - memory-region-names + - operating-points-v2 + - apple,cs-opp + - apple,afr-opp + - apple,csafr-min-sram-microvolt + - apple,aon-ptd + else: + properties: + apple,csafr-min-sram-microvolt: false + apple,aon-ptd: false - if: properties: compatible: @@ -407,6 +468,7 @@ allOf: enum: - apple,agx-t6040 - apple,agx-t6041 + - apple,agx-t6050 - apple,agx-t8132 - apple,agx-t8140 then: diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index fcfe475d4dcf2..1c4249cfc5b98 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -53,7 +53,9 @@ properties: follow the temperature-* children in the listed order and are labelled "SoC die ". Keys absent from the running firmware or with an unsupported format are skipped. These channels do not change the indices - used by thermal-sensors references to temperature-* children. + used by thermal-sensors references to temperature-* children. With + apple,discover-temperatures, a listed key adds no channel; its + discovered channel (labelled with the key) supplies the zone. apple,thermal-pressure-key: $ref: /schemas/types.yaml#/definitions/string diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 9ba41ab9806d2..0c91127ba8505 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -151,9 +151,25 @@ properties: items: maximum: 255 description: - Reserved streams of a USB4 PCIe tunnel DART that its PCIe-C port issues + Streams set to bypass after every reset of the DART, outside any domain. + On a USB4 PCIe tunnel DART, the reserved streams its PCIe-C port issues its own writes on, such as the interrupt controller writes its MSIs - turn into. They are set to bypass after every reset of the DART. + turn into. On a T8110 DART that nothing initialises before Linux, such + as a USB4 router's, the streams its co-processor uses without + translation. + + apple,full-error-reporting: + type: boolean + description: + Unmask every error and enable the error interrupt for every stream, + with the registers of the DART's revision, and leave the DART unlocked + on error. For a T8110 DART that nothing initialises before Linux. + + apple,ctc-flush: + type: boolean + description: + Flush the DART's context cache after a stream flush, where the DART + has one. For a T8110 DART that nothing initialises before Linux. apple,remap-streams: $ref: /schemas/types.yaml#/definitions/uint32-matrix diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index 493e48de06a92..19ffe1bf167f7 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -26,6 +26,9 @@ description: SART3, first seen in M1 Pro/Max, extends both the address space and filter entry size to 42 bit. + SART4, seen on M5 Ultra (T6050), keeps SART3's entries but moves the + address and size arrays, so it has no SART3 fallback. + properties: compatible: oneOf: diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index 289c29ba752fb..a08c3224e7b1e 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -14,7 +14,9 @@ description: wrapper resource contains the coprocessor control registers and mailbox FIFOs. A separate resource provides the coprocessor vector-base register. The firmware role identifies which GPU firmware instance uses the mailbox; - the T6040 runs a single one. + the T6040 runs a single one. The T6050 J775D profile describes the physical + GPU aperture shared by J775D and J716c; the other T6050 profiles describe + alias windows. properties: compatible: @@ -25,6 +27,7 @@ properties: - apple,t6050-apif-ascwrap-v6 - apple,t6050-ascwrap-v6 - apple,t6050-x1n1-ascwrap-v6 + - apple,t6050-j775d-ascwrap-v6 reg: items: diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index 050ac9c6388ca..c35e7d0e10faa 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) AND GPL-2.0-only %YAML 1.2 --- $id: http://devicetree.org/schemas/mfd/apple,smc.yaml# diff --git a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml index 245f520044e64..1e81f153d2f4f 100644 --- a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml +++ b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) AND GPL-2.0-only %YAML 1.2 --- $id: http://devicetree.org/schemas/mux/apple,display-crossbar.yaml# @@ -11,8 +11,8 @@ maintainers: description: Routes display pipelines to the DisplayPort transmitters and inputs - of an Apple Type-C PHY. Each crossbar exposes three mux controls, - selected by the consumer's mux specifier. + of an Apple Type-C PHY. Older layouts expose three mux controls; the T8142/T6050 layout + exposes five. The consumer selects an output using the mux specifier. allOf: - $ref: mux-controller.yaml# @@ -28,6 +28,7 @@ properties: - apple,t6041-display-crossbar - apple,t8132-display-crossbar - apple,t8142-display-crossbar + - apple,t6050-display-crossbar - items: - const: apple,t6050-display-crossbar - const: apple,t8142-display-crossbar diff --git a/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml new file mode 100644 index 0000000000000..6b1a33cac7aa4 --- /dev/null +++ b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/bluetooth/apple,centauri.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple N1 Centauri Beta Bluetooth PCI function + +maintainers: + - Dj + +description: | + PCI function two is the Beta Bluetooth HCI/ACL transport. It depends on + the Control function in the same PCI slot, which boots the shared N1 + firmware using the protocol profile described on its own DT node. + The factory address is supplied in network byte order for the firmware + address-provisioning command; HCI returns the standard Bluetooth order. + +properties: + compatible: + const: pci106b,1903 + + reg: + maxItems: 1 + description: Standard five-cell PCI function address + + local-bd-address: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 6 + maxItems: 6 + description: Factory Bluetooth public address, least significant byte + first (the standard Bluetooth property a boot loader fills) + + local-mac-address: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 6 + maxItems: 6 + description: Factory Bluetooth public address in network byte order + +oneOf: + - required: [local-bd-address] + - required: [local-mac-address] + +required: + - compatible + - reg + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml b/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml index 4e46c3573edbf..f73dd9afceca1 100644 --- a/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml +++ b/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml @@ -13,8 +13,8 @@ description: | PCI function zero owns the shared N1 firmware bootstrap. The boot producer supplies the personalized FTAB filename and packed firmware platform ID. The endpoint must be left in its stopped ROM personality with DMA and MSI - quiesced. A supported APIF PCI host performs the isolated port reset after - ROM bootstrap. This binding does not authorize adopting an existing live + quiesced. The host's root-port reset retrains the link after the ROM + bootstrap. This binding does not authorize adopting an existing live firmware session or releasing another owner's retained DART streams. properties: diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 3d08734f0ee74..7f289a02d4b21 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -117,6 +117,7 @@ allOf: not: contains: enum: + - apple,t6050-nvme-ans2 - apple,t8132-nvme-ans2 - apple,t8140-nvme-ans2 then: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 245231ecfc3a9..36cf94442e064 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -75,6 +75,7 @@ properties: - const: apple,t8112-pciec - const: apple,t8103-pciec - const: apple,t6000-pciec + - const: apple,t6050-pciec - items: - const: apple,t8132-pciec - const: apple,t8122-pciec @@ -98,11 +99,11 @@ properties: reg: minItems: 3 - maxItems: 11 + maxItems: 14 reg-names: minItems: 3 - maxItems: 11 + maxItems: 14 ranges: minItems: 2 @@ -128,8 +129,7 @@ properties: apple,msi-vector-base: $ref: /schemas/types.yaml#/definitions/uint32 description: - First vector of this PCIe-C controller's slice of the MSI window it - shares with the internal PCIe controller. + First global vector targeted by this controller's local MSI map. iommu-map: true iommu-map-mask: true @@ -234,6 +234,52 @@ allOf: - apple,msi-vector-base - $ref: /schemas/pci/pci-host-bridge.yaml# - $ref: /schemas/interrupt-controller/msi-controller.yaml# + - if: + properties: + compatible: + contains: + const: apple,t6050-pcie + then: + properties: + reg: + minItems: 5 + maxItems: 14 + reg-names: + description: + config and rc, then port, phy and intr2axi for each + bootloader-trained root port 00:.0. + minItems: 5 + maxItems: 14 + items: + pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" + interrupts: + maxItems: 4 + interrupt-names: + items: + pattern: "^port[0-3]$" + apple,msi-vector-base: + const: 64 + power-domains: false + required: + - interrupt-names + - apple,msi-vector-base + - iommu-map + patternProperties: + "^pci@": + properties: + reset-gpios: + description: + The PERST# line of a port whose endpoint needs a root-port + reset after the bootloader trained the link. Left as the + bootloader set it until that reset. + pwren-gpios: false + required: + - apple,link-trained-by-bootloader + else: + patternProperties: + "^pci@": + properties: + apple,link-trained-by-bootloader: false - if: properties: compatible: @@ -273,6 +319,7 @@ allOf: - apple,t8103-pciec - apple,t6000-pciec - apple,t6040-pciec + - apple,t6050-pciec then: properties: reg: @@ -390,10 +437,10 @@ allOf: properties: reg: minItems: 5 - maxItems: 11 + maxItems: 14 reg-names: minItems: 5 - maxItems: 11 + maxItems: 14 items: enum: [config, rc, port0, phy0, intr2axi0, port1, phy1, intr2axi1, port2, phy2, intr2axi2, port3, phy3, @@ -420,6 +467,8 @@ allOf: maxItems: 3 else: properties: + reg: + maxItems: 10 reg-names: minItems: 3 items: diff --git a/Documentation/devicetree/bindings/phy/apple,combo-phy.yaml b/Documentation/devicetree/bindings/phy/apple,combo-phy.yaml index 43b179b4dd879..90e5dbf812a0e 100644 --- a/Documentation/devicetree/bindings/phy/apple,combo-phy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,combo-phy.yaml @@ -24,11 +24,11 @@ properties: reg: minItems: 6 - maxItems: 11 + maxItems: 12 reg-names: minItems: 6 - maxItems: 11 + maxItems: 12 '#phy-cells': enum: [0, 1] @@ -38,8 +38,8 @@ properties: apple,port-index: $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2, 3, 4, 5] - description: Type-C port; the debug port (0) is excluded + enum: [0, 1, 2, 3, 4, 5] + description: Type-C port, including port zero when handed over to the host apple,guest-port-reserved: type: boolean @@ -78,9 +78,11 @@ properties: description: A combo PHY may name the USB controller's domain fourth ("usb"). Its USB2 PHY and digital banks are in that domain, and the PHY then holds it - while it runs, also on a port with no USB controller bound. + while it runs, also on a port with no USB controller bound. USB4 + DisplayPort tunnel pixel clocks add four clock gates ("dp0" to "dp3"), + held while the corresponding clock runs. minItems: 3 - maxItems: 4 + maxItems: 8 power-domain-names: minItems: 3 @@ -89,6 +91,10 @@ properties: - const: common - const: wrapper - const: usb + - const: dp0 + - const: dp1 + - const: dp2 + - const: dp3 apple,tunable-axi2af: $ref: /schemas/types.yaml#/definitions/uint32-matrix @@ -174,8 +180,9 @@ allOf: const: 1 reg: minItems: 11 - maxItems: 11 + maxItems: 12 reg-names: + minItems: 11 items: - const: core - const: usb2phy @@ -188,6 +195,8 @@ allOf: - const: usb-control - const: firmware-data - const: pipehandler + - description: Source select words for the four USB4 DP pixel clocks + const: dp-clock-source required: - '#reset-cells' - apple,port-index diff --git a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml index a7bc6bf4885c9..5d7e40b564f91 100644 --- a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +++ b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml @@ -22,6 +22,9 @@ description: | On t8142, a standalone register page contains a shared request register with alternating request and busy bits for four ports. + On t6050 the same request register sits at offset 0x60 of a page in the + die's PMGR block. + In all cases the reset id is the CIO port index on the die. properties: diff --git a/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml b/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml index 5891ba9a6cf13..8b9a5eaba6e49 100644 --- a/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml +++ b/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml @@ -54,12 +54,6 @@ properties: bits 2n (TX) and 2n + 1 (RX) of the shared interrupt mask and flag registers. - apple,tx-never-wait: - type: boolean - description: - Transmit completes without waiting for a reader to drain the channel, - so a console with nobody attached does not stall. - apple,sw-drain: type: boolean description: @@ -68,6 +62,16 @@ properties: property of the channel's Apple device tree node. The interrupt, if described, is not used. + apple,tx-never-wait: + type: boolean + description: + Output never waits for the host to read the channel. The channel drains + only while a debug USB host reads it, so with none attached its TX FIFO + stays full. Console writes send what fits and drop the rest, a closing + port does not wait for its output, and output queued with no room for + two drain ticks is dropped. For boards that boot unattended with nothing + reading the channel. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index d9dd4cd1aa756..8980de2e7474a 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -52,7 +52,8 @@ properties: PCIe adapter configuration space of the USB4 router. Present where the root complex exposes the adapter separately, so that the tunnel can be configured before a PCIe-C link is brought up. - - description: MxWrap co-processor control registers + - description: MxWrap co-processor control registers, on T8132 and T6050 + - description: PCIe router AP firmware registers, on T6050 reg-names: oneOf: @@ -62,24 +63,37 @@ properties: - const: sram - const: pcie-adapter - const: cpu + - const: router-ap-fw - description: T6050, which has no separate PCIe adapter window items: - const: rc - const: sram - const: cpu - interrupts: - maxItems: 1 - description: Type 7 SRAM parity interrupt - - interrupt-names: - items: - - const: sram-parity - mboxes: maxItems: 1 description: Mailbox used for RTKit communication with the co-processor + interrupts: + minItems: 1 + maxItems: 4 + description: | + Type 7 SRAM parity interrupt, or interrupts of the router's DP IN + adapter blocks, one per DP IN adapter: + a change of the HPD level the adapter receives over the tunnel, and + IRQ_HPD from the sink. + + interrupt-names: + oneOf: + - items: + - const: sram-parity + - minItems: 1 + items: + - const: dpin0 + - const: dpin1 + - const: dpin2 + - const: dpin3 + resets: maxItems: 1 description: Reset used to start the ACIO block before booting the @@ -87,12 +101,12 @@ properties: power-domains: minItems: 3 - maxItems: 4 + maxItems: 5 description: | - The first three domains power the ACIO co-processor complex. When - apple,pcie-tunnel is present, the fourth domain powers its PCIe-C - controller and is enabled only after the USB4 PCIe tunnel is active. - On T6050 the fourth domain is the routers' shared audio (AUSS) domain. + The first three domains (four on T6050) power the ACIO co-processor + complex. When apple,pcie-tunnel is present, the domain after them powers + its PCIe-C controller and is enabled only after the USB4 PCIe tunnel is + active. thunderbolt-switch: type: boolean @@ -135,6 +149,18 @@ properties: named by the pcie-adapter reg entry. Only meaningful where that entry is present. + apple,tunable-router-ap-fw: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: | + As apple,tunable-rc, but applied to the PCIe router AP firmware + registers named by the router-ap-fw reg entry, with the other router + tunables once the co-processor firmware is ready. + '#address-cells': const: 1 @@ -209,16 +235,31 @@ allOf: contains: enum: - apple,t6040-usb4-acio + - apple,t6050-usb4-acio - apple,t8132-usb4-acio then: properties: - reg: - minItems: 4 - reg-names: - minItems: 4 pinctrl-names: items: - const: lstx + allOf: + - if: + properties: + compatible: + contains: + const: apple,t6050-usb4-acio + then: + properties: + reg: + minItems: 3 + reg-names: + minItems: 3 + else: + properties: + reg: + minItems: 4 + reg-names: + minItems: 4 else: properties: lstx-gpios: false @@ -228,6 +269,12 @@ allOf: contains: const: apple,t6040-usb4-acio then: + properties: + interrupts: + maxItems: 1 + interrupt-names: + items: + - const: sram-parity required: - iommu@1a80000 - nhi@1f00000 @@ -237,6 +284,24 @@ allOf: required: - iommu@a80000 - nhi@f00000 + - if: + properties: + compatible: + contains: + const: apple,t6050-usb4-acio + then: + properties: + reg: + minItems: 3 + reg-names: + minItems: 3 + power-domains: + minItems: 4 + else: + properties: + power-domains: + maxItems: 4 + apple,tunable-router-ap-fw: false additionalProperties: false diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml index ede4cea33eb2d..28718fed167c5 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml @@ -52,10 +52,12 @@ properties: interrupts: description: One interrupt for each of the 12 transmit rings followed by one for each - of the 12 receive rings. - maxItems: 24 + of the 12 receive rings. On T6050 an SRAM parity interrupt follows them. + minItems: 24 + maxItems: 25 interrupt-names: + minItems: 24 items: - const: txring0 - const: txring1 @@ -81,6 +83,7 @@ properties: - const: rxring9 - const: rxring10 - const: rxring11 + - const: parity iommus: minItems: 1 @@ -177,4 +180,17 @@ allOf: iommus: maxItems: 1 + - if: + properties: + compatible: + not: + contains: + const: apple,t6050-usb4-nhi + then: + properties: + interrupts: + maxItems: 24 + interrupt-names: + maxItems: 24 + additionalProperties: false diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 706d2866f61ba..7745fef5b686f 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -35,6 +35,23 @@ allOf: required: - resets - usb-role-switch + - if: + properties: + compatible: + contains: + const: apple,t6050-dwc3 + then: + properties: + reg: + minItems: 3 + reg-names: + minItems: 3 + else: + properties: + reg: + maxItems: 3 + reg-names: + maxItems: 3 $defs: apple,tunable: @@ -48,7 +65,7 @@ $defs: - description: Bits to be set after applying the mask description: > Controller table from the device's Apple device tree, applied after - every core reset (T6040). + every core reset (T6040, T6050). properties: compatible: @@ -74,7 +91,9 @@ properties: items: - description: Core DWC3 region - description: Apple-specific DWC3 region - - description: Controller configuration block (T6040) + - description: Controller configuration block (T6040, T6050) + - description: Bulk fabric block (T6050) + - description: AUSBC debug block (T6050) reg-names: minItems: 2 @@ -82,6 +101,8 @@ properties: - const: dwc3-core - const: dwc3-apple - const: drd-cfg + - const: drd-bulk + - const: drd-ausbc interrupts: maxItems: 1 @@ -118,7 +139,16 @@ properties: apple,tunable-cio: $ref: "#/$defs/apple,tunable" description: - Tunables applied to the CIO (USB4 tunnel) registers of the Apple region. + Tunables applied to the CIO registers of the Apple region, from offset + 0x20 of that region, after the driver's fixed CIO values. + + apple,tunable-bulk: + $ref: "#/$defs/apple,tunable" + description: Tunables applied to the drd-bulk region. + + apple,tunable-ausbc: + $ref: "#/$defs/apple,tunable" + description: Tunables applied to the drd-ausbc region. required: - compatible diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index c2d2eb1068ea8..282d26ae0c6d2 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -44,6 +44,13 @@ properties: connector: $ref: /schemas/connector/usb-connector.yaml# + apple,reset-inherited-usb2-connection: + type: boolean + description: + The boot USB port retains a USB2 repeater connection that cannot be + recovered by resetting the host PHY. Renegotiate a normal application + connection before starting the host controllers. Debug routes are preserved. + required: - compatible - reg diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index b0c78c3412088..46a6d504b7f5c 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -36,6 +36,13 @@ The driver leaves fan policy to the SMC until userspace writes a nonzero shutdown and removal release only fans taken over by such a request; resume does not reapply it. A new target write is required after resume. +A kernel thermal policy (the M5 GPU governor) sets the fans through +macsmc_hwmon_fans_request(), as a share of each fan's range, on every fan +whose node names its mode, target, minimum and maximum keys, with or without +apple,fan-manual-control. It uses the same manual mode, leaves fans userspace +holds alone, and its fans return to the SMC at suspend, shutdown and removal +like userspace's. + SoC die temperatures are described by the ``apple,soc-die-keys`` string list on the hwmon device node. The driver appends these temperature channels after the ``temperature-*`` children, in list order, with labels ``SoC die ``. @@ -43,6 +50,8 @@ Only readable four-byte floating-point keys are used. When thermal support is enabled, their hottest reading supplies the ``macsmc_soc_die`` thermal zone. A failed or implausible reading fails the zone read rather than reporting a partial result. Keys absent from the firmware are skipped. +With ``apple,discover-temperatures`` the listed keys add no channels: the +discovered channel of each listed key, labelled with the key, joins the zone. The optional ``apple,thermal-pressure-key`` on the same node describes a readable signed 32-bit firmware thermal pressure level. Neither feature needs diff --git a/arch/arm64/boot/dts/apple/t6050-j716c-gpu-panel.dtsi b/arch/arm64/boot/dts/apple/t6050-j716c-gpu-panel.dtsi new file mode 100644 index 0000000000000..cd640d3f8d7f1 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j716c-gpu-panel.dtsi @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J716c GPU and internal panel. The loader validates the running hardware, + * fills the memory reservations and operating points, then enables the + * dependent devices. A source DT alone must not start either coprocessor. + */ + +/* Replace the disabled inventory with the GPU driver's resource layout. */ +/delete-node/ &sgx; +/delete-node/ &gfx_asc; +/delete-node/ &gfx1_asc; + +/ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + gpu_fw_text: gpu-fw-text { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + gpu_fw_data: gpu-fw-data { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + }; + + /* Four clusters on one die: four main voltage lanes and one CS/AFR lane. */ + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp01 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp02 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp03 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp04 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp05 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp06 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp07 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp08 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp09 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp10 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp11 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp12 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + + opp13 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microvolt-sram = <0 0 0 0>; + }; + }; + + gpu_afr_opp: opp-table-gpu-afr { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp01 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp02 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp03 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp04 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp05 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp06 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp07 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp08 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp09 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp10 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp11 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp12 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp13 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + }; + + gpu_cs_opp: opp-table-gpu-cs { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp01 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp02 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp03 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp04 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp05 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp06 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp07 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp08 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp09 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp10 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp11 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp12 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + + opp13 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + }; + + soc@2300000000 { + + gpu: gpu@2302600000 { + compatible = "apple,agx-t6050", "apple,agx-g17s"; + reg = <0x23 0x02600000 0x0 0x88000>, + <0x23 0x00000000 0x0 0x4000000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&gpu_fw_data>, <&uat_l2_pagetables>, <&gpu_fw_text>; + memory-region-names = "ttbs", "pagetables", "handoff", "fw-data", + "l2-pagetables", "fw-text"; + operating-points-v2 = <&gpu_opp>; + apple,afr-opp = <&gpu_afr_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,csafr-min-sram-microvolt = <0>; + /* Three power telemetry records, each an identifier and two words. */ + apple,aon-ptd = <0 0 0 0 0 0 0 0 0>; + status = "disabled"; + }; + + gfx_mbox: mailbox@2302600000 { + compatible = "apple,t6050-j775d-ascwrap-v6"; + reg = <0x23 0x02600000 0x0 0x88000>, + <0x23 0x02050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + + gfx1_mbox: mailbox@2302e00000 { + compatible = "apple,t6050-j775d-ascwrap-v6"; + reg = <0x23 0x02e00000 0x0 0x88000>, + <0x23 0x02850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + }; + + /* The loader verifies these clock identifiers and fills their rates. */ + dcp_fw_clk361: clock-dcp-firmware-361 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <810000000>; + clock-output-names = "dcp_firmware_361"; + }; + + dcp_fw_clk458: clock-dcp-firmware-458 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <521064992>; + clock-output-names = "dcp_firmware_458"; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + status = "disabled"; + }; +}; + +/* The power states stay disabled until the loader accepts the GPU handoff. */ +&pmgr_gpx { + status = "okay"; +}; + +&ps_gpx { + apple,always-on; +}; + +&ps_afr { + apple,always-on; +}; + +&dcp_dart { + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&disp0_dart { + reg = <0x4 0x8a300000 0x0 0xc000>; + apple,dma-range = <0x100 0x0 0x300 0x0>; +}; + +&dcp { + apple,bw-scratch = <&dcpext0_pmc 0 5 0x20000>; + apple,clock-ids = <361 458>; + clocks = <&dcp_fw_clk361>, <&dcp_fw_clk458>; + clock-names = "firmware-0", "firmware-1"; + + panel { + compatible = "apple,panel-mini-led"; + apple,max-brightness = <600>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j716c.dts b/arch/arm64/boot/dts/apple/t6050-j716c.dts index 33fd4e1c9b03c..cb00c91eed4f1 100644 --- a/arch/arm64/boot/dts/apple/t6050-j716c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j716c.dts @@ -27,6 +27,16 @@ bluetooth0 = &n1_beta; }; + /* GPU aperture; its PMGR and coprocessors are described below. */ + j716c_gpu_bus: soc@2300000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x23 0x0 0x23 0x0 0x0 0x4000000>; + interrupt-parent = <&aic>; + nonposted-mmio; + }; + chosen { #address-cells = <2>; #size-cells = <2>; @@ -222,3 +232,403 @@ #define T6050_J716C #include "t6050-peripherals.dtsi" +#include "t6050-j716c-gpu-panel.dtsi" + +/* + * ==== The right USB-C port (ATC2) ==== + * + * ADT /arm-io/nub-spmi-a1/hpm2 (SN201202x, SPMI address 0xc, port-number 3, port-location + * "right", acio-parent acio2), atc-phy2, usb-drd2 and dart-usb2. The HPM is on its own bus, + * nub-spmi-a1, and nothing hands the port off, so the chain is enabled outright. Aura's + * kboot_setup_atc() fills the PHY's tunables through the atcphy2 alias. + */ +/ { + aliases { + atcphy2 = &atc2_phy; + }; +}; + +&ps_atc2_common { + status = "okay"; +}; + +&ps_atc2_usb_aon { + status = "okay"; +}; + +&ps_atc2_phymxwrap { + status = "okay"; +}; + +&ps_atc2_usb { + status = "okay"; +}; + +&atc2_phy_mbox { + status = "okay"; +}; + +&atc2_phy { + status = "okay"; + + port { + atc2_phy_ep: endpoint { + remote-endpoint = <&hpm2_ss>; + }; + }; +}; + +&usb_drd2 { + status = "okay"; + + port { + usb_drd2_hs: endpoint { + remote-endpoint = <&hpm2_hs>; + }; + }; +}; + +&usb_drd2_dart_0 { + status = "okay"; +}; + +&usb_drd2_dart_1 { + status = "okay"; +}; + +/* + * dcpext0's Type-C route stays off with dcpext0: a mode-switch endpoint whose mux is never + * registered would defer the PD controller forever. + */ +&dcpext0 { + typec-routes { + route@0 { + status = "disabled"; + + port { + dcpext0_typec: endpoint { + remote-endpoint = <&hpm2_dp>; + }; + }; + }; + }; +}; + +&nub_spmi_a1 { + hpm2: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + hpm2_hs: endpoint { + remote-endpoint = <&usb_drd2_hs>; + }; + }; + + port@1 { + reg = <1>; + hpm2_ss: endpoint { + remote-endpoint = <&atc2_phy_ep>; + }; + }; + + port@2 { + reg = <2>; + hpm2_dp: endpoint { + remote-endpoint = <&dcpext0_typec>; + }; + }; + }; + }; + }; +}; + +/* + * ==== The right USB-C port's USB4/Thunderbolt router (ATC2) ==== + * + * Until the boot loader fills them, the router, NHI, PHY CIO and DART tunables and the host + * DROM are this board's, from its ADT (the DROM's router UUID from an earlier boot). + */ +/* ATC2_CIO_AUSS's other parent; kept as the boot loader leaves it */ +&ps_auss { + status = "okay"; +}; + +/* The CIO reconfig request, port 2 (clock gate CIO2_RECONFIG_V) */ +&cio_reconfig0 { + status = "okay"; +}; + +/* The PHY's CIO80 RX_TOP tunables (ADT tunable_LN0/LN1_RX_TOP_CIO_DFLT); the other three are empty */ +&atc2_phy { + apple,tunable-lane0-rx-top-cio = <0x35c 0xe00 0x0>; + apple,tunable-lane1-rx-top-cio = <0x35c 0xe00 0x0>; +}; + +/* Only the router's LSTX pin (81) of the AOP GPIO controller is exposed. */ +&pinctrl_aop { + gpio-reserved-ranges = <0 81>, <82 3>; + status = "okay"; +}; + +&hpm2 { + connector { + ports { + port@3 { + reg = <3>; + hpm2_usb4: endpoint { + remote-endpoint = <&acio2_tbt>; + }; + }; + }; + }; +}; + +&acio2 { + apple,tunable-rc = <0x0 0xfff0 0x4020>, + <0x2c 0x4 0x0>, + <0x4004 0x1ff01ff 0x1000100>, + <0x4008 0x801f 0x4>, + <0x4010 0x7f 0x40>, + <0x4014 0x801f 0x4>, + <0x401c 0x7f 0x40>, + <0x4024 0x801f 0x4>, + <0x402c 0x7f 0x40>, + <0x4030 0x801f 0x4>, + <0x4038 0xffffffff 0x1010101>, + <0x403c 0xffff 0x101>, + <0x4040 0x1 0x1>, + <0x4044 0xffffffff 0x1010101>, + <0x4048 0x1 0x1>, + <0x404c 0x801f 0x4>, + <0x8004 0x70007 0x40004>, + <0x8008 0x801f 0x4>, + <0x8010 0x70007 0x40004>, + <0x8018 0x801f 0x4>, + <0x8020 0x70007 0x40004>, + <0x8028 0x801f 0x4>, + <0x8030 0x801f 0x4>, + <0x8038 0x801f 0x4>, + <0x8040 0x801f 0x4>, + <0x8048 0x801f 0x4>, + <0x8050 0x801f 0x4>, + <0x8058 0x801f 0x4>, + <0x8060 0x801f 0x4>; + apple,tunable-pcie-adapter = <0x1008 0x6 0x6>; + apple,tunable-router-ap-fw = <0x8 0xff2 0x482>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + acio2_tbt: endpoint { + remote-endpoint = <&hpm2_usb4>; + }; + }; + }; +}; + +&acio2_dart { + apple,tunable = <0x20c 0x80000121 0x80000121>, + <0x220 0xf0f 0xf0f>, + <0x224 0xffff 0x909>, + <0x300 0x1f31 0x1>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>, + <0x800 0xf003f 0x10000>, + <0x804 0xf003f 0x10002>, + <0x808 0xf003f 0x10004>, + <0x80c 0xf003f 0x20006>, + <0x810 0xf003f 0x2000a>, + <0x814 0xf003f 0x2000e>, + <0x818 0xf003f 0x20012>, + <0x81c 0xf003f 0x20016>, + <0x820 0xf003f 0x2001a>, + <0x824 0xf003f 0x2001e>, + <0x828 0xf003f 0x20022>, + <0x82c 0xf003f 0x20026>, + <0x830 0xf003f 0x2002a>, + <0x834 0xf003f 0x2002e>, + <0x838 0xf003f 0x20032>, + <0x83c 0xf003f 0x10036>; +}; + +&acio2_nhi { + apple,tunable-nhi = <0xf8004 0xff 0x80>, + <0xf8008 0x801f 0x4>, + <0xf8010 0x3f 0x20>, + <0xf8014 0x801f 0x4>, + <0xf801c 0x1f 0x10>, + <0xf8020 0x801f 0x4>, + <0xf8028 0xff 0x80>, + <0xf802c 0x801f 0x4>, + <0xf8034 0xff 0x80>, + <0xf8038 0x801f 0x4>, + <0xf8040 0x3f 0x20>, + <0xf8044 0x801f 0x4>, + <0xf804c 0x8007 0x1>, + <0xfc004 0x3 0x2>, + <0xfc008 0x801f 0x4>, + <0xfc010 0x7 0x4>, + <0xfc014 0x801f 0x4>, + <0xfc01c 0x7 0x4>, + <0xfc020 0x801f 0x4>, + <0xfc028 0x7 0x4>, + <0xfc02c 0x801f 0x4>, + <0xfc034 0x7 0x4>, + <0xfc038 0x801f 0x4>, + <0xfc040 0x3 0x2>, + <0xfc044 0x801f 0x4>, + <0xfc04c 0x8007 0x1>, + <0x4000 0x1000000f 0x10000002>, + <0x4004 0x1000000f 0x10000002>, + <0x4008 0x1000000f 0x10000002>, + <0x400c 0x1000000f 0x10000002>, + <0x4010 0x1000000f 0x10000002>, + <0x4014 0x1000000f 0x10000002>, + <0x4018 0x1000000f 0x10000002>, + <0x401c 0x1000000f 0x10000002>, + <0x4020 0x1000000f 0x10000002>, + <0x4024 0x1000000f 0x10000002>, + <0x4028 0x1000000f 0x10000002>, + <0x402c 0x1000000f 0x10000002>, + <0x4040 0x1000000f 0x10000002>, + <0x4044 0x1000000f 0x10000002>, + <0x4048 0x1000000f 0x10000002>, + <0x404c 0x1000000f 0x10000002>, + <0x4050 0x1000000f 0x10000002>, + <0x4054 0x1000000f 0x10000002>, + <0x4058 0x1000000f 0x10000002>, + <0x405c 0x1000000f 0x10000002>, + <0x4060 0x1000000f 0x10000002>, + <0x4064 0x1000000f 0x10000002>, + <0x4068 0x1000000f 0x10000002>, + <0x406c 0x1000000f 0x10000002>, + <0x4080 0x1000000f 0x10000002>, + <0x4084 0x1000000f 0x10000002>, + <0x4088 0x1000000f 0x10000002>, + <0x408c 0x1000000f 0x10000002>, + <0x4090 0x1000000f 0x10000002>, + <0x4094 0x1000000f 0x10000002>, + <0x4098 0x1000000f 0x10000002>, + <0x409c 0x1000000f 0x10000002>, + <0x40a0 0x1000000f 0x10000002>, + <0x40a4 0x1000000f 0x10000002>, + <0x40a8 0x1000000f 0x10000002>, + <0x40ac 0x1000000f 0x10000002>, + <0x40c0 0x1000000f 0x10000002>, + <0x40c4 0x1000000f 0x10000002>, + <0x40c8 0x1000000f 0x10000002>, + <0x40cc 0x1000000f 0x10000002>, + <0x40d0 0x1000000f 0x10000002>, + <0x40d4 0x1000000f 0x10000002>, + <0x40d8 0x1000000f 0x10000002>, + <0x40dc 0x1000000f 0x10000002>, + <0x40e0 0x1000000f 0x10000002>, + <0x40e4 0x1000000f 0x10000002>, + <0x40e8 0x1000000f 0x10000002>, + <0x40ec 0x1000000f 0x10000002>, + <0x4100 0x1000000f 0x10000002>, + <0x4104 0x1000000f 0x10000002>, + <0x4108 0x1000000f 0x10000002>, + <0x410c 0x1000000f 0x10000002>, + <0x4110 0x1000000f 0x10000002>, + <0x4114 0x1000000f 0x10000002>, + <0x4118 0x1000000f 0x10000002>, + <0x411c 0x1000000f 0x10000002>, + <0x4120 0x1000000f 0x10000002>, + <0x4124 0x1000000f 0x10000002>, + <0x4128 0x1000000f 0x10000002>, + <0x412c 0x1000000f 0x10000002>, + <0x4140 0x1000000f 0x10000002>, + <0x4144 0x1000000f 0x10000002>, + <0x4148 0x1000000f 0x10000002>, + <0x414c 0x1000000f 0x10000002>, + <0x4150 0x1000000f 0x10000002>, + <0x4154 0x1000000f 0x10000002>, + <0x4158 0x1000000f 0x10000002>, + <0x415c 0x1000000f 0x10000002>, + <0x4160 0x1000000f 0x10000002>, + <0x4164 0x1000000f 0x10000002>, + <0x4168 0x1000000f 0x10000002>, + <0x416c 0x1000000f 0x10000002>, + <0x43e4 0x7 0x7>; + apple,thunderbolt-drom = [ff 92 e6 6e ee 18 7c ac 05 90 19 c3 8c 00 5c 00 + 00 00 00 00 00 00 08 81 82 02 82 00 00 00 08 82 + 92 01 82 00 00 00 03 83 20 08 84 00 00 00 00 00 + 00 08 85 00 00 00 00 00 00 05 86 09 01 00 08 87 + 00 00 00 00 00 00 05 88 09 01 00 05 89 09 01 00 + 05 8a 09 01 00 0e 01 41 70 70 6c 65 2c 20 49 6e + 63 2e 00 06 02 69 4f 53 00 00 00 00]; +}; + +/* ==== The right USB-C port's PCIe tunnel (ATC2): this board's tunables ==== */ +&apciec2_dart { + apple,tunable = <0x20c 0x80000121 0x80000121>, + <0x220 0xf0f 0xf0f>, + <0x224 0xffff 0x909>, + <0x300 0x1f31 0x1>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>; +}; + +&apciec2 { + apple,tunable-debug = <0x1000 0x1 0x0>; + apple,tunable-rc = <0x78 0x7000 0x0>, + <0x718 0x7c000 0x0>, + <0x814 0xffffffff 0x2020f060>, + <0x8bc 0x1 0x0>; + apple,tunable-fabric = <0x4 0x3f003f 0x240024>, + <0x8 0x1 0x1>, + <0xc 0x1 0x1>, + <0x10 0x801f 0x4>, + <0x18 0x1ff 0x1e0>, + <0x1c 0x1 0x1>, + <0x20 0x801f 0x4>, + <0x28 0x1f 0x10>, + <0x2c 0x801f 0x4>, + <0x34 0x3f 0x20>, + <0x38 0x801f 0x4>, + <0x40 0x3f 0x20>, + <0x44 0x801f 0x4>, + <0x4c 0x10001 0x10001>, + <0x50 0x1 0x1>, + <0x54 0x1 0x1>, + <0x58 0x801f 0x4>, + <0x60 0x7f007f 0x450045>, + <0x64 0x801f 0x4>, + <0x6c 0x1f001f 0x100010>, + <0x70 0xf1 0x1>, + <0x74 0x801f 0x4>, + <0x7c 0x801f 0x4>, + <0x84 0x1 0x1>, + <0x88 0x1 0x1>, + <0x8c 0x801f 0x4>, + <0x94 0x1 0x1>, + <0x98 0x801f 0x4>, + <0xa0 0x1 0x1>, + <0xa4 0x801f 0x4>, + <0xac 0x1 0x1>, + <0xb0 0x1 0x1>, + <0xb4 0x801f 0x4>, + <0xbc 0x801f 0x4>, + <0xc4 0x801f 0x4>; +}; + +&apciec2_port { + apple,tunable = <0x140 0x1 0x1>, + <0x2a0 0x7e 0xa>; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-max.dtsi b/arch/arm64/boot/dts/apple/t6050-max.dtsi index 760094fce8097..970bcfd6239f8 100644 --- a/arch/arm64/boot/dts/apple/t6050-max.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-max.dtsi @@ -115,4 +115,292 @@ interrupt-controller; #interrupt-cells = <2>; }; + + /* + * /arm-io/nub-spmi-a1: reg[0] 0x8a1a8000, generation 4, a bus of USB-C PD controllers (the + * board's). They are polled, as on nub-spmi-a0, so the bus takes no interrupt. + */ + nub_spmi_a1: spmi@28a1a8000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a1a8000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + }; +}; + +/* + * ATC2's USB4/Thunderbolt router (ADT /arm-io/acio2, acio-cpu2 and dart-acio2), powered per + * connection: the PHY enters CIO mode, then the router's domains come on (ATC2_CIO, _PCIE, _USB, + * _AUSS), the CIO reset (reconfig request) clocks the CPU wrapper, the CPU runs from the image the + * boot loader left in its SRAM, and once its firmware is ready the DART and the NHI are populated. + * Nothing here is touched at boot. The router is named acio@: the boot loader disables a cio@ node + * unless its first register is the third register of the matching /arm-io/acioN, which on T6050 + * is not the router's rc block. The board gives the tunables and the host DROM until the boot + * loader fills them. + */ +&pmgr0 { + ps_atc2_cio: power-controller@2e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + label = "atc2_cio"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_common>; + }; + + ps_atc2_cio_pcie: power-controller@3f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + label = "atc2_cio_pcie"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_cio>; + }; + + ps_atc2_cio_usb: power-controller@400 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + label = "atc2_cio_usb"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_cio>; + }; + + ps_atc2_cio_auss: power-controller@408 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + label = "atc2_cio_auss"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_cio>, <&ps_auss>; + }; +}; + +/* LSTX: /arm-io/acio2 gpio-lstx, AOP GPIO pin 81 (0x51), function 1 for the router */ +&pinctrl_aop { + acio2_lstx_pins: acio2-lstx-pins { + pinmux = <0x10051>; /* APPLE_PINMUX(81, 1) */ + }; +}; + +&die0 { + /* /arm-io/acio-cpu2 reg[0]: the router CPU's mailbox, inside its wrapper */ + acio2_mbox: mailbox@381108000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x81108000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + /* + * /arm-io/acio2: rc (reg[1]), the RTKit data segment in its SRAM, the PCIe adapter, + * acio-cpu2 reg[1] (the CPU wrapper) and the PCIe router AP firmware block. + */ + acio2: acio@381ac0000 { + compatible = "apple,t6050-usb4-acio"; + reg = <0x3 0x81ac0000 0x0 0xc000>, + <0x3 0x81080000 0x0 0x26000>, + <0x3 0x81e44000 0x0 0x4000>, + <0x3 0x81100000 0x0 0x4000>, + <0x3 0x81c80000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter", "cpu", "router-ap-fw"; + mboxes = <&acio2_mbox>; + resets = <&cio_reconfig0 2>; + /* ATC2_PCIE last: the PCIe tunnel root port, set up before it is populated */ + power-domains = <&ps_atc2_cio>, <&ps_atc2_cio_pcie>, + <&ps_atc2_cio_usb>, <&ps_atc2_cio_auss>, <&ps_atc2_pcie>; + thunderbolt-switch; + apple,pcie-tunnel = <&acio2_pcie_tunnel>; + pinctrl-0 = <&acio2_lstx_pins>; + pinctrl-names = "lstx"; + lstx-gpios = <&pinctrl_aop 81 GPIO_ACTIVE_HIGH>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x3 0x81000000 0x1000000>; + nonposted-mmio; + + /* + * /arm-io/dart-acio2, populated with the NHI once the router runs, so that it is + * first read with the router up. Nothing initialises it before Linux: its tunables + * (dart-tunables-instance-0), stream 15 (bypass-15) in bypass, every error + * reported and the context cache flushed. vm-base 0x100_0000_0000. + */ + acio2_dart: iommu@a80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0xa80000 0xc000>; + interrupts = ; + power-domains = <&ps_atc2_cio>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; + apple,bypass-sids = <15>; + apple,full-error-reporting; + apple,ctc-flush; + }; + + /* Interrupts 0x66b.. are the TX rings and 0x677.. the RX rings */ + acio2_nhi: nhi@f00000 { + compatible = "apple,t6050-usb4-nhi"; + reg = <0xf00000 0x100000>; + reg-names = "nhi"; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "txring0", "txring1", "txring2", "txring3", "txring4", "txring5", + "txring6", "txring7", "txring8", "txring9", "txring10", "txring11", + "rxring0", "rxring1", "rxring2", "rxring3", "rxring4", "rxring5", + "rxring6", "rxring7", "rxring8", "rxring9", "rxring10", "rxring11", + "parity"; + iommus = <&acio2_dart 0>, + <&acio2_dart 1>, + <&acio2_dart 2>, + <&acio2_dart 3>, + <&acio2_dart 4>, + <&acio2_dart 5>, + <&acio2_dart 6>, + <&acio2_dart 7>, + <&acio2_dart 8>, + <&acio2_dart 9>, + <&acio2_dart 10>, + <&acio2_dart 11>, + <&acio2_dart 16>, + <&acio2_dart 17>, + <&acio2_dart 18>, + <&acio2_dart 19>, + <&acio2_dart 20>, + <&acio2_dart 21>, + <&acio2_dart 22>, + <&acio2_dart 23>, + <&acio2_dart 24>, + <&acio2_dart 25>, + <&acio2_dart 26>, + <&acio2_dart 27>; + }; + }; +}; + +/* + * ATC2's PCIe tunnel: /arm-io/apciec2 and dart-apciec2, the PCIe-C root port behind the router's + * PCIe down adapter, and its DART. The router populates both once it has enabled a PCIe tunnel, + * after the kernel has cold-initialized the port: no boot loader sets it up, and it has no reset + * line of its own. Both sit in ATC2_PCIE, which the router powers with its own domains and the + * host keeps on once it is bound. + * + * The container is not a usb4-pcie-tunnel-N node, whose DART would get the fixed 64-stream + * topology of the earlier tunnel DARTs: this one is read like any T8110 DART. Nothing + * initialises it before Linux, so like the router's it carries every error reported and its + * bypass streams, 253 and 255, which carry the port's own writes (MSIs); the board gives its + * tunables, and the port's. + * + * MSIs use the bridge's 32-vector slice (192-223) of the window shared with the internal host. + */ +&pmgr0 { + /* With the ATC2 USB always-on clock gate, which has no state register of its own */ + ps_atc2_pcie: power-controller@2f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + label = "atc2_pcie"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_common>, <&ps_atc2_usb_aon>; + }; +}; + +/ { + acio2_pcie_tunnel: pcie-tunnel-2 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + nonposted-mmio; + + apciec2_dart: iommu@384000000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x3 0x84000000 0x0 0x20000>; + interrupts = ; + power-domains = <&ps_atc2_pcie>; + #iommu-cells = <1>; + apple,bypass-sids = <253 255>; + apple,full-error-reporting; + apple,ctc-flush; + }; + + apciec2: pcie@1c70000000 { + compatible = "apple,t6050-pciec"; + device_type = "pci"; + reg = <0x1c 0x70000000 0x0 0x10000000>, + <0x3 0x84034000 0x0 0x4000>, + <0x3 0x84020000 0x0 0x8000>, + <0x3 0x84038000 0x0 0x4000>, + <0x3 0x84060000 0x0 0x4000>, + <0x3 0x8402c000 0x0 0x4000>, + <0x3 0x84028000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "fabric", "debug", "ltssm", "intr2axi"; + /* The port's: link events and INTx */ + interrupts = ; + interrupt-names = "port0"; + power-domains = <&ps_atc2_pcie>; + /* No boot loader hands the port over: the kernel cold-initializes it. */ + apple,pciec-kernel-init; + + msi-controller; + msi-parent = <&apciec2>; + msi-ranges = <&aic AIC_IRQ 0 2016 IRQ_TYPE_EDGE_RISING 32>; + apple,msi-vector-base = <192>; + + iommu-map = <0x0 &apciec2_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + linux,pci-domain = <3>; + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x10 0x0 0x10 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 0x12 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 0x12 0x40000000 0x0 0x40000000>; + + apciec2_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec2_port 0 0 0 0>, + <0 0 0 2 &apciec2_port 0 0 0 1>, + <0 0 0 3 &apciec2_port 0 0 0 2>, + <0 0 0 4 &apciec2_port 0 0 0 3>; + }; + }; + }; }; diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index eef667d647d3d..5799735e8b295 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -54,6 +54,7 @@ }; }; #endif +#ifndef T6050_J716C / { soc { compatible = "simple-bus"; @@ -64,6 +65,8 @@ }; }; +#endif + &pmgr0 { ps_pms: power-controller@100 { compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; @@ -883,7 +886,11 @@ }; }; +#ifdef T6050_J716C +&j716c_gpu_bus { +#else &{/soc} { +#endif pmgr_gpx: power-management@2300e80000 { compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; reg = <0x23 0xe80000 0x0 0x4000>; @@ -1112,7 +1119,11 @@ }; }; +#ifdef T6050_J716C +&die0 { +#else &{/soc} { +#endif peripheral_clocks: clock-controller@280600000 { compatible = "apple,t6050-pmgr-clocks", "apple,pmgr-clocks"; reg = <0x2 0x80600000 0x0 0x1fc000>, @@ -1181,7 +1192,11 @@ }; }; +#ifdef T6050_J716C +&die0 { +#else &{/soc} { +#endif #if defined(T6050_J714C) || defined(T6050_J716C) || defined(T6050_J775C) || defined(T6050_J775D) || defined(T6050_K114C) || defined(T6050_K116C) nub_spmi1: spmi@28a054000 { compatible = "apple,t6050-spmi", "apple,t8142-spmi"; @@ -1320,6 +1335,7 @@ #endif #if defined(T6050_J714C) || defined(T6050_J714S) || defined(T6050_J716C) || defined(T6050_K114C) || defined(T6050_K114S) || defined(T6050_K116C) || defined(T6050_K116S) +#ifndef T6050_J716C nub_spmi_a1: spmi@28a1a8000 { compatible = "apple,t6050-spmi", "apple,t8142-spmi"; reg = <0x2 0x8a1a8000 0x0 0x4000>, @@ -1331,6 +1347,7 @@ status = "disabled"; }; #endif +#endif #if defined(T6050_J714C) || defined(T6050_J714S) || defined(T6050_J716C) || defined(T6050_J716S) || defined(T6050_K114C) || defined(T6050_K114S) || defined(T6050_K116C) || defined(T6050_K116S) isp0: isp@580000000 { @@ -1431,6 +1448,7 @@ +#ifndef T6050_J716C apciec0_dart: iommu@2a4000000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x2 0xa4000000 0x0 0x20000>; @@ -1438,6 +1456,7 @@ #iommu-cells = <1>; status = "disabled"; }; +#endif apciec0_piodma: dma-controller@2a4068000 { compatible = "apple,t6050-pciec-piodma", "apple,pciec-piodma"; @@ -1450,6 +1469,7 @@ status = "disabled"; }; +#ifndef T6050_J716C apciec1_dart: iommu@2ac000000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x2 0xac000000 0x0 0x20000>; @@ -1457,6 +1477,7 @@ #iommu-cells = <1>; status = "disabled"; }; +#endif apciec1_piodma: dma-controller@2ac068000 { compatible = "apple,t6050-pciec-piodma", "apple,pciec-piodma"; @@ -1469,6 +1490,7 @@ status = "disabled"; }; +#ifndef T6050_J716C apciec2_dart: iommu@384000000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x3 0x84000000 0x0 0x20000>; @@ -1476,6 +1498,7 @@ #iommu-cells = <1>; status = "disabled"; }; +#endif apciec2_piodma: dma-controller@384068000 { compatible = "apple,t6050-pciec-piodma", "apple,pciec-piodma"; diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 832e2d3019ecc..002a8985ca04f 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -520,4 +520,12 @@ config BT_CENTAURI Say M to build the btcentauri module for bring-up testing. +config BT_HCICENTAURI + tristate "Apple N1 Centauri T6050 Bluetooth" + depends on PCI && APPLE_CENTAURI_T6050 + help + Protocol-27 Beta HCI and ACL transport using the T6050 Control + supplier and a device-tree factory address. This can coexist with + the J813/J873 Bluetooth transport. SCO and ISO are not implemented. + endmenu diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index eb3b4de799a3b..4f2f47e39eb43 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -59,3 +59,4 @@ obj-$(CONFIG_BT_CENTAURI) += btcentauri.o obj-$(CONFIG_BT_MTK7932_PCIE) += mt7932_bt_pcie.o CONTEXT_ANALYSIS := y +obj-$(CONFIG_BT_HCICENTAURI) += btcentauri_t6050.o diff --git a/drivers/bluetooth/btcentauri_t6050.c b/drivers/bluetooth/btcentauri_t6050.c new file mode 100644 index 0000000000000..1d701d218a512 --- /dev/null +++ b/drivers/bluetooth/btcentauri_t6050.c @@ -0,0 +1,961 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CEN_ARENA_SIZE SZ_1M +#define CEN_MTR 0x2000 +#define CEN_MCR 0x2200 +#define CEN_MESSAGES 0x4000 +#define CEN_RX 0x8000 +#define CEN_TX 0xa000 +#define CEN_FOOTER 260 +#define CEN_STRIDE (16 + CEN_FOOTER) +#define CEN_RX_SLOTS 17 +#define CEN_TX_SLOTS 5 +#define CEN_LE_RX 0xc000 +#define CEN_LE_TX 0xe000 +#define CEN_LE_FOOTER 256 +#define CEN_LE_STRIDE (16 + CEN_LE_FOOTER) +#define CEN_LE_SLOTS 17 +#define CEN_MESSAGE_SLOTS 16 +#define CEN_AUX_BASE 0x20000 +#define CEN_CAPTURE_SIZE SZ_1M + +#define CEN_OP_READ_PROP_FEATURES 0xffa3 +#define CEN_OP_CREATE_CONN 0xff7b +#define CEN_OP_FLUSH_DUPLICATES 0xff3f +#define CEN_PROP_CREATE_CONN BIT(9) + +struct cen_aux_spec { + u16 id, slots, footer; + bool receive; +}; + +static const struct cen_aux_spec cen_aux_specs[] = { + { 2, 9, 2592, false }, { 3, 9, 724, false }, { 5, 9, 260, false }, + { 6, 5, 1024, false }, { 8, 9, 2592, true }, { 9, 9, 724, true }, + { 11, 9, 268, true }, { 12, 128, 1024, true }, { 13, 17, 260, true }, + { 14, 9, 1304, false }, { 15, 9, 1304, true }, +}; + +struct cen_aux_state { + u32 offset; + u16 head, post; + bool opened; +}; + +struct cen_bt { + struct pci_dev *pdev; + struct hci_dev *hdev; + void __iomem *bar; + u8 *arena; + dma_addr_t dma; + struct mutex lock; + struct work_struct io_work; + unsigned int irq_count; + bool irq_vectors; + struct sk_buff_head txq; + struct sk_buff_head acl_txq, le_txq; + u16 acl_tx_head, le_tx_head; + u16 message_head, message_tag, tx_head, rx_head, rx_post; + u16 le_rx_head, le_rx_post; + struct cen_aux_state aux[ARRAY_SIZE(cen_aux_specs)]; + bool ipc_requested, running; + u8 native_startup_steps; + bool connection_command_supported, connection_status_pending; +}; + +static u16 cen_tail(struct cen_bt *bt, u16 ring) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(bt->arena + 0x7a + ring * 2))); +} + +static void cen_post(struct cen_bt *bt, u16 ring, u16 value) +{ + put_unaligned_le16(value, bt->arena + 0xa8 + ring * 2); + dma_wmb(); + writel(value, bt->bar + 0x1000 + ring * 4); +} + +static int cen_message(struct cen_bt *bt, const void *data, size_t length) +{ + u16 head = bt->message_head, tail, + next = (head + 1) % CEN_MESSAGE_SLOTS; + u16 tag = bt->message_tag + 1; + u8 *descriptor = bt->arena + CEN_MTR + head * 16; + u8 *completion = bt->arena + CEN_MCR + head * 32; + int ret; + + if (head >= CEN_MESSAGE_SLOTS || !length || length > 512) + return -EINVAL; + memset(descriptor, 0, 16); + memset(completion, 0, 32); + memcpy(bt->arena + CEN_MESSAGES + head * 512, data, length); + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(bt->dma + CEN_MESSAGES + head * 512, descriptor + 4); + put_unaligned_le16(tag, descriptor + 12); + cen_post(bt, 0, next); + ret = read_poll_timeout(cen_tail, tail, tail == next, 100, 1500000, + false, bt, 0); + if (ret) + return ret; + + ret = read_poll_timeout(le16_to_cpu, tail, tail == next, 100, 1500000, + false, + READ_ONCE(*(__le16 *)(bt->arena + 0x78))); + if (ret) + return ret; + dma_rmb(); + if (get_unaligned_le16(completion + 4) != tag || + get_unaligned_le16(completion + 6) != length || + get_unaligned_le32(completion + 8)) + return -EPROTO; + bt->message_head = next; + bt->message_tag = tag; + put_unaligned_le16(bt->message_head, bt->arena + 0xa6); + dma_wmb(); + return 0; +} + +static int cen_open_ring(struct cen_bt *bt, u16 id, u16 slots, u32 offset, + u16 footer) +{ + u8 message[0x34] = { 1 }; + u32 words = footer / 4, shift = 0; + + if (!words || footer % 4) + return -EINVAL; + if (words > 255) + shift = min_t(u32, __ffs(words), 4); + if ((words >> shift) > 255) + return -EINVAL; + message[2] = words >> shift; + message[3] = shift << 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le64(bt->dma + offset, message + 8); + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(slots, message + 0x18); + put_unaligned_le16(0xffff, message + 0x1a); + put_unaligned_le16(id, message + 0x1c); + put_unaligned_le16(0x40, message + 0x1e); + return cen_message(bt, message, sizeof(message)); +} + +static void cen_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_RX + slot * CEN_STRIDE; + + memset(descriptor, 0, CEN_STRIDE); + put_unaligned_le32(1 | (CEN_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_le_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_LE_RX + slot * CEN_LE_STRIDE; + + memset(descriptor, 0, CEN_LE_STRIDE); + put_unaligned_le32(1 | (CEN_LE_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static int cen_receive_acl(struct cen_bt *bt, const u8 *data, u32 length) +{ + struct sk_buff *skb; + u32 packet_length, pos; + + if (length < HCI_ACL_HDR_SIZE) + return -EPROTO; + packet_length = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + if (packet_length > length) + return -EPROTO; + + for (pos = packet_length; pos < length;) { + if (!data[pos]) + break; + if (data[pos] == 1) + pos++; + else if (data[pos] == 2 && length - pos >= 9) + pos += 9; + else + return -EPROTO; + } + skb = bt_skb_alloc(packet_length, GFP_KERNEL); + if (!skb) { + bt->hdev->stat.err_rx++; + return 0; + } + skb_put_data(skb, data, packet_length); + hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT; + bt->hdev->stat.byte_rx += packet_length; + hci_recv_frame(bt->hdev, skb); + return 0; +} + +static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) +{ + const struct cen_aux_spec *spec = &cen_aux_specs[index]; + u8 *descriptor = + bt->arena + bt->aux[index].offset + slot * (16 + spec->footer); + + memset(descriptor, 0, 16 + spec->footer); + put_unaligned_le32(1 | (spec->footer << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static bool cen_aux_required(u16 id) +{ + return id == 2 || id == 8; +} + +static int cen_aux_start(struct cen_bt *bt) +{ + u32 offset = CEN_AUX_BASE; + unsigned int i, slot; + int ret; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u32 length = spec->slots * (16U + spec->footer); + + if (offset > CEN_ARENA_SIZE || length > CEN_ARENA_SIZE - offset) + return -ENOSPC; + state->offset = offset; + offset = ALIGN(offset + length, SZ_4K); + if (!cen_aux_required(spec->id)) + continue; + ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, + spec->footer); + if (ret) { + dev_err(&bt->pdev->dev, + "Auxiliary ring %u open failed: %d\n", spec->id, + ret); + return ret; + } + state->opened = true; + if (!spec->receive) + continue; + for (slot = 0; slot < spec->slots - 1; slot++) + cen_aux_rx_buffer(bt, i, slot); + state->post = spec->slots - 1; + cen_post(bt, spec->id, state->post); + } + return 0; +} + +static int cen_aux_drain(struct cen_bt *bt, bool repost) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u16 tail; + int budget = spec->slots; + + if (!state->opened || !spec->receive) + continue; + tail = cen_tail(bt, spec->id); + if (tail >= spec->slots) + return -EPROTO; + while (state->head != tail && budget--) { + u8 *descriptor = bt->arena + state->offset + + state->head * (16 + spec->footer); + u32 length; + + dma_rmb(); + length = get_unaligned_le32(descriptor) >> 8; + if (length > spec->footer) + return -EPROTO; + if (repost && spec->id == 8 && + cen_receive_acl(bt, descriptor + 16, length)) + return -EPROTO; + state->head = (state->head + 1) % spec->slots; + if (repost) { + cen_aux_rx_buffer(bt, i, state->post); + state->post = (state->post + 1) % spec->slots; + cen_post(bt, spec->id, state->post); + } + } + } + return 0; +} + +static int cen_start(struct cen_bt *bt) +{ + static const u16 indices[] = { 0x78, 0x7a, 0xa6, 0xa8 }; + u8 config[4] = { 9, 0, 0xff, 0xff }; + u32 status; + int i, ret; + + put_unaligned_le16(0x300, bt->arena); + put_unaligned_le16(0x68, bt->arena + 2); + put_unaligned_le64(bt->dma + 0x68, bt->arena + 8); + for (i = 0; i < ARRAY_SIZE(indices); i++) + put_unaligned_le64(bt->dma + indices[i], + bt->arena + 0x10 + i * 8); + put_unaligned_le16(1, bt->arena + 0x30); + put_unaligned_le16(22, bt->arena + 0x32); + put_unaligned_le64(bt->dma + CEN_MCR, bt->arena + 0x34); + put_unaligned_le64(bt->dma + CEN_MTR, bt->arena + 0x3c); + put_unaligned_le16(16, bt->arena + 0x44); + put_unaligned_le16(16, bt->arena + 0x46); + + bt->arena[0x53] = 4; + put_unaligned_le64(bt->dma + 0x10000, bt->arena + 0x58); + pci_set_master(bt->pdev); + writel(lower_32_bits(bt->dma), bt->bar + 0xc); + writel(upper_32_bits(bt->dma), bt->bar + 0x10); + writel(CEN_ARENA_SIZE, bt->bar + 0x14); + bt->ipc_requested = true; + writel(1, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 1, 100, 2000000); + if (ret) + return ret; + dma_wmb(); + writel(lower_32_bits(bt->dma), bt->bar + 4); + writel(upper_32_bits(bt->dma), bt->bar + 8); + writel(2, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 2, 100, 2000000); + if (ret) + return ret; + ret = cen_message(bt, config, sizeof(config)); + if (!ret) + ret = cen_open_ring(bt, 7, CEN_RX_SLOTS, CEN_RX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 10, CEN_LE_SLOTS, CEN_LE_RX, + CEN_LE_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER); + if (ret) + return ret; + for (i = 0; i < CEN_RX_SLOTS - 1; i++) + cen_rx_buffer(bt, i); + bt->rx_post = CEN_RX_SLOTS - 1; + cen_post(bt, 7, bt->rx_post); + { + for (i = 0; i < CEN_LE_SLOTS - 1; i++) + cen_le_rx_buffer(bt, i); + bt->le_rx_post = CEN_LE_SLOTS - 1; + cen_post(bt, 10, bt->le_rx_post); + } + return cen_aux_start(bt); +} + +static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, u16 ring, + u16 slots, u32 offset, u16 footer, u16 *head) +{ + struct sk_buff *skb; + u16 tail, next; + u8 *descriptor; + u32 wire_length; + + while (!skb_queue_empty(queue)) { + tail = cen_tail(bt, ring); + if (tail >= slots) + return -EPROTO; + next = (*head + 1) % slots; + if (next == tail) + break; + dma_rmb(); + skb = skb_dequeue(queue); + if (!skb) + break; + descriptor = bt->arena + offset + *head * (16 + footer); + memset(descriptor, 0, 16 + footer); + put_unaligned_le16(*head + 1, descriptor + 12); + memcpy(descriptor + 16, skb->data, skb->len); + wire_length = skb->len; + if (ring == 1 && READ_ONCE(bt->connection_command_supported) && + get_unaligned_le16(skb->data) == HCI_OP_CREATE_CONN) { + u8 *command = descriptor + 16; + + put_unaligned_le16(CEN_OP_CREATE_CONN, command); + command[2] = 15; + command[HCI_COMMAND_HDR_SIZE + 9] = 0; + command[HCI_COMMAND_HDR_SIZE + 12] = 0; + wire_length = HCI_COMMAND_HDR_SIZE + 15; + bt->connection_status_pending = true; + } + put_unaligned_le32(1 | (wire_length << 8), descriptor); + *head = next; + cen_post(bt, ring, next); + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) + bt->hdev->stat.cmd_tx++; + else + bt->hdev->stat.acl_tx++; + bt->hdev->stat.byte_tx += wire_length; + kfree_skb(skb); + } + return 0; +} + +static void cen_fixup_scan_response(u8 *event, size_t length) +{ + struct hci_ev_le_ext_adv_info *info; + size_t offset = HCI_EVENT_HDR_SIZE + 2; + u8 count; + + if (length < offset || event[0] != HCI_EV_LE_META || + event[HCI_EVENT_HDR_SIZE] != HCI_EV_LE_EXT_ADV_REPORT) + return; + count = event[HCI_EVENT_HDR_SIZE + 1]; + while (count-- && length - offset >= sizeof(*info)) { + u16 type; + + info = (void *)(event + offset); + if (info->length > length - offset - sizeof(*info)) + return; + type = get_unaligned_le16(&info->type); + if ((type & LE_EXT_ADV_EVT_TYPE_MASK) == + (LE_EXT_ADV_LEGACY_PDU | LE_EXT_ADV_SCAN_RSP)) + put_unaligned_le16(type | LE_EXT_ADV_SCAN_IND, + &info->type); + offset += sizeof(*info) + info->length; + } +} + +static void cen_io_work(struct work_struct *work) +{ + struct cen_bt *bt = container_of(work, struct cen_bt, io_work); + struct hci_dev *hdev = bt->hdev; + struct sk_buff *skb; + u16 tail; + u8 *descriptor; + u32 length; + int budget = CEN_RX_SLOTS; + + mutex_lock(&bt->lock); + if (!bt->running) + goto unlock; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 2) { + if (readl(bt->bar) == 2) + cen_aux_drain(bt, false); + goto protocol_error; + } + if (cen_aux_drain(bt, true)) + goto protocol_error; + { + tail = cen_tail(bt, 10); + if (tail >= CEN_LE_SLOTS) + goto protocol_error; + while (bt->le_rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_LE_RX + + bt->le_rx_head * CEN_LE_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length > CEN_LE_FOOTER) + goto protocol_error; + if (cen_receive_acl(bt, descriptor + 16, length)) + goto protocol_error; + bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; + cen_le_rx_buffer(bt, bt->le_rx_post); + bt->le_rx_post = (bt->le_rx_post + 1) % CEN_LE_SLOTS; + cen_post(bt, 10, bt->le_rx_post); + } + budget = CEN_RX_SLOTS; + } + tail = cen_tail(bt, 7); + if (tail >= CEN_RX_SLOTS) + goto protocol_error; + while (bt->rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_RX + bt->rx_head * CEN_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length < HCI_EVENT_HDR_SIZE || length > CEN_FOOTER || + length != descriptor[17] + HCI_EVENT_HDR_SIZE) + goto protocol_error; + skb = bt_skb_alloc(length, GFP_KERNEL); + if (skb) { + skb_put_data(skb, descriptor + 16, length); + cen_fixup_scan_response(skb->data, skb->len); + + if (bt->connection_status_pending && length == 6 && + skb->data[0] == HCI_EV_CMD_STATUS && + get_unaligned_le16(skb->data + 4) == + CEN_OP_CREATE_CONN) { + put_unaligned_le16(HCI_OP_CREATE_CONN, + skb->data + 4); + bt->connection_status_pending = false; + } + + if (READ_ONCE(bt->connection_command_supported) && + length == + HCI_EVENT_HDR_SIZE + + sizeof(struct hci_ev_conn_complete) && + skb->data[0] == HCI_EV_CONN_COMPLETE && + (skb->data[11] & 0xfe) == 0xf0) { + skb->data[11] = ACL_LINK; + } + hci_skb_pkt_type(skb) = HCI_EVENT_PKT; + hdev->stat.byte_rx += length; + hci_recv_frame(hdev, skb); + } else { + hdev->stat.err_rx++; + } + bt->rx_head = (bt->rx_head + 1) % CEN_RX_SLOTS; + cen_rx_buffer(bt, bt->rx_post); + bt->rx_post = (bt->rx_post + 1) % CEN_RX_SLOTS; + cen_post(bt, 7, bt->rx_post); + } + if (cen_transmit(bt, &bt->txq, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER, + &bt->tx_head)) + goto protocol_error; + + if ((cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, + bt->aux[0].offset, cen_aux_specs[0].footer, + &bt->acl_tx_head) || + cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER, &bt->le_tx_head))) + goto protocol_error; + goto unlock; +protocol_error: + bt->running = false; + hdev->stat.err_rx++; + dev_err(&bt->pdev->dev, + "Beta transport stopped (exec=%u ipc=%u); interrupt processing stopped\n", + readl(bt->bar + 0x18), readl(bt->bar)); +unlock: + mutex_unlock(&bt->lock); +} + +static int cen_hci_open(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = true; + schedule_work(&bt->io_work); + mutex_unlock(&bt->lock); + return 0; +} + +static int cen_hci_close(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = false; + bt->connection_status_pending = false; + mutex_unlock(&bt->lock); + cancel_work_sync(&bt->io_work); + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, + const void *data, u8 length) +{ + struct sk_buff *skb; + int ret; + + skb = __hci_cmd_sync(hdev, opcode, length, data, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len < 1 ? -EPROTO : -bt_to_errno(skb->data[0]); + kfree_skb(skb); + return ret; +} + +static int cen_hci_reset_dup_filter(struct hci_dev *hdev) +{ + static const u8 flags; + int ret; + + ret = __hci_cmd_sync_status(hdev, CEN_OP_FLUSH_DUPLICATES, + sizeof(flags), &flags, HCI_CMD_TIMEOUT); + if (ret) { + bt_dev_warn(hdev, "Duplicate filter reset failed: %d", ret); + return ret; + } + return 0; +} + +static int cen_hci_post_init(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff *skb; + u32 features; + int ret; + + WRITE_ONCE(bt->connection_command_supported, false); + skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len == 5 ? -bt_to_errno(skb->data[0]) : -EPROTO; + features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; + kfree_skb(skb); + if (!ret && !(features & CEN_PROP_CREATE_CONN)) + ret = -EOPNOTSUPP; + if (ret) { + bt_dev_warn(hdev, "Firmware connection command unavailable: %d", + ret); + return 0; + } + WRITE_ONCE(bt->connection_command_supported, true); + return 0; +} + +/* + * The factory address in network byte order, for the firmware's provisioning command: from + * the standard local-bd-address (least significant byte first, as boot loaders fill it from + * the bluetooth0 alias), else from local-mac-address (already in network order). + */ +static int cen_bt_address(struct device *dev, u8 address[6]) +{ + u8 bd[6]; + int i, ret; + + if (device_property_count_u8(dev, "local-bd-address") == 6) { + ret = device_property_read_u8_array(dev, "local-bd-address", bd, sizeof(bd)); + if (ret) + return ret; + for (i = 0; i < 6; i++) + address[i] = bd[5 - i]; + return 0; + } + if (device_property_count_u8(dev, "local-mac-address") != 6) + return -EINVAL; + return device_property_read_u8_array(dev, "local-mac-address", address, 6); +} + +static int cen_hci_setup(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + u8 address[6], init_done[16], seed[24]; + int ret; + + ret = cen_bt_address(&bt->pdev->dev, address); + if (ret) + return ret; + if (is_zero_ether_addr(address) || is_broadcast_ether_addr(address)) + return -EINVAL; + ret = get_random_bytes_wait(seed, sizeof(seed)); + if (ret) + return ret; + memset(init_done, 0xff, sizeof(init_done)); + ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 1); + ret = cen_vendor_command(hdev, 0xff0b, init_done, sizeof(init_done)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 2); + ret = cen_vendor_command(hdev, 0xfe44, seed, sizeof(seed)); + if (!ret) + WRITE_ONCE(bt->native_startup_steps, 3); +out: + memzero_explicit(seed, sizeof(seed)); + if (ret) + bt_dev_err( + hdev, + "Firmware initialization stopped after %u commands: %d", + bt->native_startup_steps, ret); + return ret; +} + +static int cen_hci_flush(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff_head *queue = &bt->txq; + struct hci_conn *conn; + u16 handle; + u8 link_type; + int ret = 0; + + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && + (skb->len < HCI_COMMAND_HDR_SIZE || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2])) + return -EMSGSIZE; + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT) { + if (skb->len < HCI_ACL_HDR_SIZE || + skb->len != HCI_ACL_HDR_SIZE + + get_unaligned_le16(skb->data + 2)) { + ret = -EMSGSIZE; + goto out; + } + handle = get_unaligned_le16(skb->data); + rcu_read_lock(); + conn = hci_conn_hash_lookup_handle(hdev, hci_handle(handle)); + link_type = conn ? conn->type : 0xff; + rcu_read_unlock(); + if (link_type == LE_LINK && (handle & 0x3000) != 0x3000) { + queue = &bt->le_txq; + if (skb->len > CEN_LE_FOOTER) + ret = -EMSGSIZE; + } else if (link_type == ACL_LINK) { + queue = &bt->acl_txq; + if (skb->len > cen_aux_specs[0].footer) + ret = -EMSGSIZE; + } else { + ret = -EOPNOTSUPP; + } + if (ret) + goto out; + } else if (hci_skb_pkt_type(skb) != HCI_COMMAND_PKT) { + ret = -EOPNOTSUPP; + goto out; + } else if (skb->len < HCI_COMMAND_HDR_SIZE || skb->len > CEN_FOOTER || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2]) { + ret = -EMSGSIZE; + goto out; + } + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) { + u16 opcode = get_unaligned_le16(skb->data); + + if (opcode == CEN_OP_CREATE_CONN) { + ret = -EOPNOTSUPP; + goto out; + } + if (opcode == HCI_OP_CREATE_CONN && + skb->len != HCI_COMMAND_HDR_SIZE + + sizeof(struct hci_cp_create_conn)) { + ret = -EMSGSIZE; + goto out; + } + } + mutex_lock(&bt->lock); + if (!bt->running) + ret = -ENETDOWN; + else if (skb_queue_len(queue) >= 64) + ret = -ENOBUFS; + else { + skb_queue_tail(queue, skb); + schedule_work(&bt->io_work); + } + mutex_unlock(&bt->lock); + if (!ret) + return 0; +out: + + return ret; +} + +static irqreturn_t cen_irq(int irq, void *data) +{ + struct cen_bt *bt = data; + + if (READ_ONCE(bt->running)) + schedule_work(&bt->io_work); + return IRQ_HANDLED; +} + +static void cen_free_irqs(struct cen_bt *bt) +{ + while (bt->irq_count) + free_irq(pci_irq_vector(bt->pdev, --bt->irq_count), bt); + if (bt->irq_vectors) { + pci_free_irq_vectors(bt->pdev); + bt->irq_vectors = false; + } +} + +static int cen_request_irqs(struct cen_bt *bt) +{ + int ret = pci_alloc_irq_vectors(bt->pdev, 2, 2, PCI_IRQ_MSI); + + if (ret < 0) + return ret; + bt->irq_vectors = true; + for (bt->irq_count = 0; bt->irq_count < 2; bt->irq_count++) { + ret = request_irq(pci_irq_vector(bt->pdev, bt->irq_count), + cen_irq, 0, "btcentauri_t6050", bt); + if (ret) { + cen_free_irqs(bt); + return ret; + } + } + return 0; +} + +static void cen_stop(struct cen_bt *bt) +{ + u16 command; + int ret; + + pci_clear_master(bt->pdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->io_work); + + ret = pci_read_config_word(bt->pdev, PCI_COMMAND, &command); + if (bt->ipc_requested || ret || (command & PCI_COMMAND_MASTER) || + !pci_wait_for_pending_transaction(bt->pdev)) + dev_warn( + &bt->pdev->dev, + "Beta detach requires fresh firmware boot; retaining DMA arena\n"); + else + dma_free_coherent(&bt->pdev->dev, CEN_ARENA_SIZE, bt->arena, + bt->dma); +} + +static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cen_bt *bt; + struct hci_dev *hdev; + int ret; + struct pci_dev *control; + struct device_link *link; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + if (!bridge || !bridge->dev.parent || + !of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie") || + PCI_FUNC(pdev->devfn) != 2) + return -ENODEV; + control = pci_get_domain_bus_and_slot( + pci_domain_nr(pdev->bus), pdev->bus->number, + PCI_DEVFN(PCI_SLOT(pdev->devfn), 0)); + if (!control) + return -EPROBE_DEFER; + link = device_link_add(&pdev->dev, &control->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + pci_dev_put(control); + return -ENOMEM; + } + ret = apple_centauri_control_ready(control) ? 0 : -EPROBE_DEFER; + pci_dev_put(control); + if (ret) + return ret; + bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + mutex_init(&bt->lock); + skb_queue_head_init(&bt->txq); + skb_queue_head_init(&bt->acl_txq); + skb_queue_head_init(&bt->le_txq); + INIT_WORK(&bt->io_work, cen_io_work); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + if (pci_resource_len(pdev, 0) != SZ_64K) + return -ENODEV; + ret = pcim_iomap_regions(pdev, BIT(0), "btcentauri_t6050"); + if (ret) + return ret; + bt->bar = pcim_iomap_table(pdev)[0]; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 0) + return dev_err_probe( + &pdev->dev, -EBUSY, + "Beta requires running firmware and unowned IPC (exec=%u ipc=%u)\n", + readl(bt->bar + 0x18), readl(bt->bar)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + bt->arena = dma_alloc_coherent(&pdev->dev, CEN_ARENA_SIZE, &bt->dma, + GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + memset(bt->arena, 0, CEN_ARENA_SIZE); + ret = cen_request_irqs(bt); + if (ret) + goto stop; + ret = cen_start(bt); + if (ret) { + dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", + ret); + goto stop; + } + hdev = hci_alloc_dev(); + if (!hdev) { + ret = -ENOMEM; + goto stop; + } + bt->hdev = hdev; + hdev->bus = HCI_PCI; + hdev->open = cen_hci_open; + hdev->close = cen_hci_close; + hdev->flush = cen_hci_flush; + hdev->send = cen_hci_send; + hdev->setup = cen_hci_setup; + hdev->post_init = cen_hci_post_init; + hdev->reset_dup_filter = cen_hci_reset_dup_filter; + + hci_set_quirk(hdev, HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY); + SET_HCIDEV_DEV(hdev, &pdev->dev); + hci_set_drvdata(hdev, bt); + pci_set_drvdata(pdev, bt); + ret = hci_register_dev(hdev); + if (ret) { + hci_free_dev(hdev); + goto stop; + } + return 0; +stop: + cen_stop(bt); + return ret; +} + +static void cen_remove(struct pci_dev *pdev) +{ + struct cen_bt *bt = pci_get_drvdata(pdev); + hci_unregister_dev(bt->hdev); + cen_hci_close(bt->hdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->io_work); + hci_free_dev(bt->hdev); + cen_stop(bt); +} + +static int cen_suspend(struct device *dev) +{ + return pci_save_state(to_pci_dev(dev)); +} + +static int cen_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cen_pm_ops, cen_suspend, cen_resume); + +static const struct pci_device_id cen_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, + {} +}; +MODULE_DEVICE_TABLE(pci, cen_ids); + +static struct pci_driver cen_driver = { + .name = "btcentauri_t6050", + .id_table = cen_ids, + .probe = cen_probe, + .remove = cen_remove, + .driver.pm = pm_sleep_ptr(&cen_pm_ops), +}; +module_pci_driver(cen_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 protocol-27 Bluetooth transport"); diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index 5f43b79b6013c..fe5b6910b4146 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple SMC GPIO driver * Copyright The Asahi Linux Contributors @@ -16,6 +16,7 @@ #include #include #include +#include #define MAX_GPIO 64 @@ -103,6 +104,7 @@ struct macsmc_gpio { struct apple_smc *smc; struct gpio_chip gc; + u64 *output_commands; int first_index; smc_key base_key; @@ -383,6 +385,72 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, return 0; } +/* Each DT template identifies one output; its low 16 bits carry the value. */ +static int macsmc_pcio_set(struct gpio_chip *gc, unsigned int offset, int value) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + u8 command[8]; + int ret; + + put_unaligned_le64(smcgp->output_commands[offset] | !!value, command); + ret = apple_smc_write(smcgp->smc, SMC_KEY(pcIO), command, sizeof(command)); + /* SMC writes return status, not a byte count. */ + return ret < 0 ? ret : 0; +} + +static int macsmc_pcio_get(struct gpio_chip *gc, unsigned int offset) +{ + /* The native output command has no established input/readback operation. */ + return -EOPNOTSUPP; +} + +static int macsmc_pcio_get_direction(struct gpio_chip *gc, unsigned int offset) +{ + return GPIO_LINE_DIRECTION_OUT; +} + +static int macsmc_pcio_probe(struct macsmc_gpio *smcgp) +{ + struct device *dev = smcgp->dev; + struct apple_smc_key_info info; + int count, ret; + + ret = apple_smc_get_key_info(smcgp->smc, SMC_KEY(pcIO), &info); + if (ret < 0) + return ret; + if (info.size != 8 || !(info.flags & APPLE_SMC_WRITABLE)) + return dev_err_probe(dev, -EINVAL, "pcIO is not an eight-byte output key\n"); + count = of_property_count_u64_elems(dev->of_node, "apple,output-commands"); + if (count <= 0 || count > MAX_GPIO) + return -EINVAL; + smcgp->output_commands = devm_kcalloc(dev, count, sizeof(u64), GFP_KERNEL); + if (!smcgp->output_commands) + return -ENOMEM; + ret = of_property_read_u64_array(dev->of_node, "apple,output-commands", + smcgp->output_commands, count); + if (ret) + return ret; + for (int i = 0; i < count; i++) { + if (smcgp->output_commands[i] & 0xffff) + return -EINVAL; + for (int j = 0; j < i; j++) + if (smcgp->output_commands[i] == smcgp->output_commands[j]) + return -EINVAL; + } + + smcgp->gc.label = "macsmc-pcio-output"; + smcgp->gc.owner = THIS_MODULE; + smcgp->gc.get = macsmc_pcio_get; + smcgp->gc.set = macsmc_pcio_set; + smcgp->gc.direction_output = macsmc_pcio_set; + smcgp->gc.get_direction = macsmc_pcio_get_direction; + smcgp->gc.can_sleep = true; + smcgp->gc.ngpio = count; + smcgp->gc.base = -1; + smcgp->gc.parent = dev; + return devm_gpiochip_add_data(dev, &smcgp->gc, smcgp); +} + struct macsmc_gpio_of_match_data { smc_key base_key; }; @@ -449,6 +517,8 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->dev = &pdev->dev; smcgp->smc = smc; + if (device_is_compatible(&pdev->dev, "apple,smc-pcio-gpio")) + return macsmc_pcio_probe(smcgp); smcgp->base_key = data ? data->base_key : _SMC_KEY("gP\0\0"); smcgp->j700_pcio = of_machine_is_compatible("apple,j700") && device_is_compatible(&pdev->dev, "apple,smc-gpio"); @@ -498,6 +568,7 @@ static const struct macsmc_gpio_of_match_data macsmc_gpio_low_data = { }; static const struct of_device_id macsmc_gpio_of_table[] = { + { .compatible = "apple,smc-pcio-gpio" }, { .compatible = "apple,smc-gpio", .data = &macsmc_gpio_up_data, diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 7575fb03695ec..3f09ef36c656d 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -6,6 +6,8 @@ config DRM_APPLE depends on APPLE_RTKIT depends on TYPEC || !TYPEC depends on OF_ADDRESS + # The firmware-27 remote buffer path calls into the DART driver. + depends on APPLE_DART || !APPLE_DART select APPLE_PMP_REPORT select OF_DYNAMIC select DRM_CLIENT_SELECTION diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 964da993e8f28..ccdb10a97bec3 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -29,6 +29,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o j613_dcp_activate.o +appledrm-y += iomfb_v27.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9ac50fb8b6576..3a902c239dd3e 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -2,6 +2,8 @@ /* Copyright 2022 Sven Peter */ #include +#include +#include #include #include #include @@ -15,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -106,6 +110,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, afkep->ops = ops; afkep->dcp = dcp; + afkep->wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0; afkep->endpoint = endpoint; afkep->wq = alloc_ordered_workqueue("apple-dcp-afkep%02x", WQ_MEM_RECLAIM, endpoint); @@ -1510,7 +1515,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, - ep->dcp->fw_compat == DCP_FIRMWARE_H17P); + (ep->dcp->fw_compat == DCP_FIRMWARE_H17P || ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0), + ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0); if (ret == -ENOENT && ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) { /* @@ -1613,6 +1619,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, } if (ops->prepare) afk_service_publish(service); + if (ops->init_props) + ops->init_props(service, epic_unit, props, props_size); if (ops->init) ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", @@ -1898,6 +1906,61 @@ static void afk_v2_status(struct apple_dcp_afkep *ep, dev_warn(ep->dcp->dev, "AFKV2 error reply failed: %d\n", ret); } +/* + * A buffer the DCP firmware names that is none of the host's pending command + * buffers. Only the firmware's own retained mappings are followed, with the + * permission they grant: a reply is written only where the firmware's leaf + * allows writes, and a DVA that Linux mapped itself, or that names Linux's RAM, + * a claimed resource or anything below DRAM (a device's registers, claimed or + * not), is refused. The copy goes through a write-combined mapping, as the + * RTKit buffers in the same firmware segments are mapped (dcp_rtk_shmem_setup()): + * the DCP is not described as DMA-coherent, so a cacheable alias could see + * stale data. + */ +static int afk_v27_access_remote(struct apple_dcp_afkep *ep, u64 dva, void *buffer, + size_t size, bool to_remote, bool copy) +{ + size_t off = 0; + + if (!dva || size > SZ_1M || size > U64_MAX - dva) + return -EINVAL; + while (off < size) { + phys_addr_t pa = 0; + size_t chunk = min_t(size_t, size - off, + PAGE_SIZE - ((dva + off) & ~PAGE_MASK)); + int ret = apple_dart_fw_remote(ep->dcp->dev, dva + off, to_remote, &pa); + void *cpu; + + if (!ret && (!dcp_fw_phys_in_dram(pa) || + region_intersects(pa, chunk, IORESOURCE_MEM, + IORES_DESC_NONE) != REGION_DISJOINT)) + ret = -EACCES; + if (ret) { + dev_err_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: refused remote buffer %#llx + %#zx (%s): %d\n", + ep->endpoint, dva, off, to_remote ? "reply" : "request", ret); + return ret; + } + if (!copy) { + off += chunk; + continue; + } + cpu = memremap(pa, chunk, MEMREMAP_WC); + if (!cpu) + return -ENOMEM; + if (to_remote) { + memcpy(cpu, buffer + off, chunk); + dma_wmb(); + } else { + dma_rmb(); + memcpy(buffer + off, cpu, chunk); + } + memunmap(cpu); + off += chunk; + } + return 0; +} + static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, void *buffer, size_t size, bool to_peer, bool copy) { @@ -1941,6 +2004,9 @@ static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, break; } spin_unlock_irqrestore(&service->lock, flags); + /* T6050 callbacks may refer to firmware-owned buffers in the retained DART. */ + if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + ret = afk_v27_access_remote(service->ep, dva, buffer, size, to_peer, copy); if (ret) dev_warn_ratelimited(service->ep->dcp->dev, "AFKV2 unowned buffer: ep=%u interface=%u size=%zu write=%u\n", @@ -2159,7 +2225,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, hdr_len = sizeof(*ehdr) + sizeof(*c); if (data_size < hdr_len) goto too_small; - if (ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && + ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !afk_validate_h17p_header(data, data_size)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: invalid compact message length\n", @@ -2177,7 +2244,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, * queue channel at face value binds "system" to 0 and makes * every later announce and report look like a duplicate. */ - channel = le16_to_cpup((__le16 *)(data + 2)); + channel = get_unaligned_le16(data + 2); /* * Normalise into the wide sub-header the rest of this file @@ -2199,6 +2266,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload = data + hdr_len; payload_size = data_size - hdr_len; + trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -2543,7 +2611,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, } spin_lock_irqsave(&ep->lock, flags); - if (ep->stopping) { + if (ep->stopping || !ep->txbfr.ready) { ret = -ESHUTDOWN; goto out; } @@ -2557,6 +2625,12 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, dma_rmb(); rptr = le32_to_cpu(*ep->txbfr.rptr); wptr = le32_to_cpu(*ep->txbfr.wptr); + if (rptr >= ep->txbfr.bufsz || wptr >= ep->txbfr.bufsz || + !IS_ALIGNED(rptr, ep->txbfr.stride) || + !IS_ALIGNED(wptr, ep->txbfr.stride)) { + ret = -EPROTO; + goto out; + } trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + (ep->wire_v2 ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 1e0a3fae1138c..6273b19a6f0a5 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -119,6 +119,9 @@ struct apple_epic_service_ops { const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); + /* Consumed synchronously before init publishes service readiness. */ + void (*init_props)(struct apple_epic_service *service, s64 unit, + const void *props, size_t size); int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 41b9c672ae398..848ac65435a2d 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1002,23 +1002,27 @@ static int apple_probe_per_dcp(struct device *dev, * Create a connector per physical Type-C port and attach it to every * pipeline that can drive it. The M2 Pro/Max laptops append a second * connector per port for a dock's second DP tunnel, preserving primary - * connector names. + * connector names. T6050 instead exposes one connector per port/pipeline + * pair: tunnel allocation chooses which connector is connected, without + * changing any encoder's advertised CRTC mask after registration. */ static int apple_probe_typec_ports(struct drm_device *drm, struct platform_device **dcp, u32 *crtc_mask, int num_dcp) { unsigned int idx, nr_ports = dcp_typec_nr_ports(); + bool fixed = dcp_typec_fixed_crtcs(); + unsigned int slots = fixed ? num_dcp : (dcp_typec_dual_stream() ? 2 : 1); int i, ret; /* Keep the existing physical-port connector numbers stable. On the * M2 Pro/Max laptops, append one more connector per port for a second * USB4 DP tunnel. */ - for (idx = 0; idx < nr_ports * (dcp_typec_dual_stream() ? 2 : 1); - idx++) { + for (idx = 0; idx < nr_ports * slots; idx++) { struct apple_connector *connector; struct apple_encoder *enc; + struct platform_device *pipeline = fixed ? dcp[idx / nr_ports] : NULL; unsigned int port_idx = idx % nr_ports; bool secondary = idx >= nr_ports; u32 mask = 0; @@ -1030,6 +1034,29 @@ static int apple_probe_typec_ports(struct drm_device *drm, continue; } + if (fixed) { + if (!dcp_typec_port_has_candidate(port_idx, pipeline)) + continue; + mask = crtc_mask[idx / nr_ports]; + if (!mask) + continue; + } else { + for (i = 0; i < num_dcp; i++) { + if (!dcp_typec_port_has_candidate(port_idx, dcp[i])) + continue; + /* Ordered dual-stream docks reserve the Type-C-only pipe. */ + mask = dcp_fabric_port_crtc_mask(secondary, + dcp_typec_ordered_dual_stream(), + dcp_is_typec_only(dcp[i]), + mask, crtc_mask[i]); + } + if (!mask) { + drm_warn(drm, "Type-C port %u has no display pipeline\n", + port_idx); + return -ENODEV; + } + } + connector = kzalloc_obj(*connector); if (!connector) return -ENOMEM; @@ -1061,32 +1088,10 @@ static int apple_probe_typec_ports(struct drm_device *drm, connector->connected = false; connector->dcp = NULL; - for (i = 0; i < num_dcp; i++) { - if (!dcp_typec_port_has_candidate(port_idx, dcp[i])) - continue; - /* - * A dock's second stream (DPIN1) always runs on the - * lowest-indexed Type-C-only pipeline (dcpext1). Fix the - * connector to it: compositors read possible_crtcs once - * and would otherwise pair it with a pipeline the stream - * can never be routed to. T6030 instead narrows both - * connectors to the native processor holding their route. - */ - mask = dcp_fabric_port_crtc_mask(secondary, - dcp_typec_ordered_dual_stream(), - dcp_is_typec_only(dcp[i]), - mask, crtc_mask[i]); - } - - if (!mask) { - drm_warn(drm, "Type-C port %u has no display pipeline\n", - port_idx); - return -ENODEV; - } - /* - * One encoder for the port, spanning every pipeline that can - * drive it. Attaching one encoder per pipeline would describe + * One encoder per connector, covering its allowed pipelines. + * T6050 connectors each cover exactly one. Attaching several + * encoders to the same connector would describe * the same hardware, but userspace takes the CRTCs a connector * can use to be what all of its encoders have in common, and * single-pipeline encoders have nothing in common -- the port @@ -1104,7 +1109,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, connector->port_encoder = &enc->base; connector->candidate_crtcs = mask; - dcp_typec_port_set_connector(port_idx, secondary, connector); + dcp_typec_port_set_connector(port_idx, secondary, connector, pipeline); } return 0; @@ -1475,8 +1480,13 @@ static int apple_drm_init(struct device *dev) * the boot framebuffer owns the screen; past the eviction there is no * way back. On refusal leave the DCP running (its shutdown sequence * can crash it and blank the panel), pin the module and keep the boot - * framebuffer. + * framebuffer. The pin is taken before the trials, while a refusal can + * still unwind; it is given back if every trial passes. */ + if (!try_module_get(THIS_MODULE)) { + ret = -ENODEV; + goto err_unbind; + } drm_for_each_crtc(crtc, &apple->drm) { const char *reason = "none"; @@ -1490,9 +1500,9 @@ static int apple_drm_init(struct device *dev) /* * The DCPs keep running and keep their pipelines: pin the * module and the DRM device their CRTCs and connectors live - * in, and keep them out of shutdown and sleep. + * in, and keep them out of shutdown and sleep. The module + * reference taken before the trials is kept. */ - __module_get(THIS_MODULE); drm_dev_get(&apple->drm); drm_for_each_crtc(crtc, &apple->drm) if (!t8132_ext_crtc(crtc)) @@ -1502,6 +1512,7 @@ static int apple_drm_init(struct device *dev) t8132_ext_drm_fini(&apple->drm); return 0; } + module_put(THIS_MODULE); if (retained_modeset) drm_modeset_lock_all(&apple->drm); diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 6e7ec1badc321..b68ef2c8dcdb7 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -39,8 +39,9 @@ struct apple_connector { struct platform_device *dcp; /* - * A Type-C connector is a physical port rather than a fixed pipeline, - * and it has one encoder covering every pipeline that can drive it. + * A Type-C connector has one encoder covering its allowed pipelines. + * T6050 has a connector per port/pipeline pair with a single CRTC; + * other platforms have a connector per port or USB4 stream. * One encoder per pipeline would describe the same hardware, but * userspace takes the CRTCs a connector can use to be what all of its * encoders have in common, and single-pipeline encoders have nothing diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index f703ea061fb50..6a7fab89c1908 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -41,7 +41,7 @@ int dcp_fabric_tunnel_request(const struct dcp_fabric_port *ports, const struct dcp_fabric_port *port; *found = NULL; - if (!key || dpin > 1) + if (!key || dpin >= DCP_FABRIC_DPIN_MAX) return -EINVAL; for (port = ports; port; port = port->next) { if (port->key != key) @@ -292,7 +292,7 @@ EXPORT_SYMBOL_GPL(dcp_fabric_plan); int dcp_fabric_tunnel_slot(const struct dcp_fabric_port *port, unsigned int dpin) { - const struct dcp_fabric_route *slot = port->owner[dpin]; + const struct dcp_fabric_route *slot = port->owner[dcp_fabric_dpin_slot(dpin)]; if (slot) return slot->tunnel && slot->dpin == dpin ? 1 : -EADDRINUSE; @@ -313,7 +313,7 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, unsigned int best_score = UINT_MAX; bool waiting = false, clock_blocked = false, clock_capable = false; - if (dpin > 1) { + if (dpin >= DCP_FABRIC_DPIN_MAX) { *error = -EINVAL; return NULL; } @@ -331,7 +331,7 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, !dcp_fabric_available(pipeline, policy)) continue; if (!dcp_fabric_fits(pipeline, policy, - port->candidate_crtcs[dpin], + port->candidate_crtcs[dcp_fabric_dpin_slot(dpin)], connector_present)) continue; if (pipeline->terminal) { @@ -343,7 +343,8 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, continue; } score = dcp_fabric_score(pipeline, policy); - if (policy->dual_stream && !dpin && !pipeline->has_fixed && + if (policy->dual_stream && !dcp_fabric_dpin_slot(dpin) && + !pipeline->has_fixed && score < UINT_MAX - 100) score += 100; if (score < best_score) { @@ -384,10 +385,10 @@ dcp_fabric_follow(const struct dcp_fabric_route *from, return DCP_FABRIC_FOLLOW_REFUSE; if (from == to) return DCP_FABRIC_FOLLOW_STAY; - if ((from->tunnel && (from->dpin > 1 || + if ((from->tunnel && (from->dpin >= DCP_FABRIC_DPIN_MAX || (to->tunnel_clock_blocked & BIT(from->dpin)))) || (holder && holder->tunnel && holder_back && - (holder->dpin > 1 || + (holder->dpin >= DCP_FABRIC_DPIN_MAX || (holder_back->tunnel_clock_blocked & BIT(holder->dpin))))) return DCP_FABRIC_FOLLOW_REFUSE; pipeline = to->pipeline; diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index dc85dbc40c15b..602e7b2efb7f6 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -5,6 +5,19 @@ #include #include +/* + * A Thunderbolt DP tunnel enters through one of up to four DP IN adapters of + * the port's host router, each behind a crossbar output of its own. A port + * carries at most two streams, primary and secondary: DP IN n and n + 2 share + * a stream slot. + */ +#define DCP_FABRIC_DPIN_MAX 4 + +static inline unsigned int dcp_fabric_dpin_slot(unsigned int dpin) +{ + return dpin & 1; +} + enum dcp_fabric_presence_state { DCP_FABRIC_ABSENT, DCP_FABRIC_SETTLING, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index f2d2fd40593fc..ef6381b159073 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -478,7 +478,8 @@ static void fabric_scenario_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, found, NULL); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 3, 0, &found), -ENODEV); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 0, 0, &found), -EINVAL); - KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 1, 2, &found), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 1, DCP_FABRIC_DPIN_MAX, + &found), -EINVAL); break; } } diff --git a/drivers/gpu/drm/apple/dcp-fabric-topology-test.c b/drivers/gpu/drm/apple/dcp-fabric-topology-test.c new file mode 100644 index 0000000000000..a9e4c3d95e3a7 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric-topology-test.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by dcp-fabric.c to exercise its connector publication paths. */ +#include +#include + +struct dcp_topology_test { + struct apple_dcp_typec_port port[3]; + struct apple_dcp_typec_route route[3][2]; + struct apple_connector connector[3][2]; + struct drm_encoder encoder[3][2]; + struct apple_dcp dcp[2]; + struct apple_crtc crtc[2]; + struct platform_device *pdev[2]; + struct drm_device drm; +}; + +static bool topology_fixed_crtcs(void) +{ + return true; +} + +static void topology_noop(struct work_struct *work) +{ +} + +static void topology_cleanup(void *data) +{ + struct dcp_topology_test *t = data; + unsigned int p, i; + + for (i = 0; i < 2; i++) + cancel_work_sync(&t->dcp[i].tb_event_work); + for (p = 0; p < 3; p++) + for (i = 0; i < 2; i++) + cancel_work_sync(&t->connector[p][i].bl_sync_wq); +} + +static struct dcp_topology_test *topology_init(struct kunit *test) +{ + struct dcp_topology_test *t; + unsigned int p, i; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + kunit_activate_static_stub(test, dcp_typec_fixed_crtcs, topology_fixed_crtcs); + for (i = 0; i < 2; i++) { + struct apple_dcp *dcp = &t->dcp[i]; + + t->pdev[i] = kunit_platform_device_alloc(test, "dcp-topology", + PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev[i]); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev[i]), 0); + platform_set_drvdata(t->pdev[i], dcp); + dcp->dev = &t->pdev[i]->dev; + dcp->crtc = &t->crtc[i]; + t->crtc[i].base.index = 1 + 2 * i; + t->crtc[i].base.dev = &t->drm; + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; + mutex_init(&dcp->tb_event_lock); + mutex_init(&dcp->tb_lock); + mutex_init(&dcp->external_lock); + mutex_init(&dcp->modes_lock); + spin_lock_init(&dcp->metadata.lock); + INIT_WORK(&dcp->tb_event_work, topology_noop); + } + for (p = 0; p < 3; p++) { + INIT_LIST_HEAD(&t->port[p].routes); + for (i = 0; i < 2; i++) { + struct apple_connector *connector = &t->connector[p][i]; + struct apple_dcp_typec_route *route = &t->route[p][i]; + + route->dcp = &t->dcp[i]; + route->port = &t->port[p]; + list_add_tail(&route->port_link, &t->port[p].routes); + connector->base.dev = &t->drm; + connector->port_encoder = &t->encoder[p][i]; + connector->candidate_crtcs = drm_crtc_mask(&t->crtc[i].base); + t->encoder[p][i].possible_crtcs = connector->candidate_crtcs; + apple_connector_edid_init(connector); + INIT_WORK(&connector->bl_sync_wq, topology_noop); + } + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, topology_cleanup, t), 0); + return t; +} + +/* The DP IN ordinal must never choose the connector on fixed-CRTC ports. */ +static void topology_pipeline_identity_test(struct kunit *test) +{ + struct dcp_topology_test *t = topology_init(test); + unsigned int p, i, dpin; + + guard(mutex)(&dcp_typec_fabric_lock); + for (p = 0; p < 3; p++) { + for (i = 0; i < 2; i++) { + struct apple_dcp_typec_route *route = &t->route[p][i]; + struct apple_connector *connector = &t->connector[p][i]; + + dcp_typec_port_bind_connector(&t->port[p], false, connector, t->pdev[i]); + KUNIT_EXPECT_PTR_EQ(test, connector->dcp, NULL); + KUNIT_EXPECT_FALSE(test, dcp_typec_narrows(&t->dcp[i])); + for (dpin = 0; dpin < DCP_FABRIC_DPIN_MAX; dpin++) { + route->tunnel_dpin = dpin; + KUNIT_EXPECT_PTR_EQ(test, + dcp_typec_route_connector(route, true), + connector); + KUNIT_EXPECT_PTR_EQ(test, + dcp_typec_route_connector(route, false), + connector); + } + } + } +} + +/* Boot adoption and replug can put either tunnel slot on either processor. */ +static void topology_adopt_release_test(struct kunit *test) +{ + struct dcp_topology_test *t = topology_init(test); + unsigned int p, i; + + guard(mutex)(&dcp_typec_fabric_lock); + for (p = 0; p < 3; p++) { + for (i = 0; i < 2; i++) { + struct apple_dcp_typec_route *route = &t->route[p][i]; + struct apple_connector *connector = &t->connector[p][i]; + u32 mask = connector->port_encoder->possible_crtcs; + + /* Opposite of the connector creation order, as after replug. */ + t->port[p].owner = &t->route[p][1]; + t->port[p].secondary_owner = &t->route[p][0]; + t->dcp[i].active_typec_route = route; + dcp_typec_port_bind_connector(&t->port[p], false, connector, t->pdev[i]); + KUNIT_EXPECT_PTR_EQ(test, t->dcp[i].typec_connector, connector); + KUNIT_EXPECT_PTR_EQ(test, connector->dcp, t->pdev[i]); + KUNIT_EXPECT_EQ(test, connector->port_encoder->possible_crtcs, mask); + connector->connected = true; + /* The hardware has acknowledged release before this retry. */ + route->deactivate_xbar_done = true; + KUNIT_ASSERT_EQ(test, dcp_typec_route_deactivate(route), 0); + KUNIT_EXPECT_FALSE(test, connector->connected); + KUNIT_EXPECT_PTR_EQ(test, connector->dcp, NULL); + KUNIT_EXPECT_PTR_EQ(test, t->dcp[i].typec_connector, NULL); + KUNIT_EXPECT_EQ(test, connector->port_encoder->possible_crtcs, mask); + KUNIT_EXPECT_PTR_EQ(test, route->connector, connector); + } + t->port[p].owner = NULL; + t->port[p].secondary_owner = NULL; + } +} + +static void topology_unbind_test(struct kunit *test) +{ + struct dcp_topology_test *t = topology_init(test); + unsigned int p, i; + + guard(mutex)(&dcp_typec_fabric_lock); + for (p = 0; p < 3; p++) { + for (i = 0; i < 2; i++) + dcp_typec_port_bind_connector(&t->port[p], false, + &t->connector[p][i], t->pdev[i]); + dcp_typec_port_unlink(&t->port[p], &t->drm); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_PTR_EQ(test, t->route[p][i].connector, NULL); + } +} + +static struct kunit_case dcp_topology_cases[] = { + KUNIT_CASE(topology_pipeline_identity_test), + KUNIT_CASE(topology_adopt_release_test), + KUNIT_CASE(topology_unbind_test), + {} +}; + +static struct kunit_suite dcp_topology_suite = { + .name = "apple-dcp-topology", + .test_cases = dcp_topology_cases, +}; + +kunit_test_suite(dcp_topology_suite); diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 3dc3128b600ae..dda07f431c226 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -28,6 +28,7 @@ #include "dcpext_scanout.h" #include "ibootep.h" #include "dcp-internal.h" +#include "dptx-service.h" #include "iomfb_m4.h" #include "parser.h" #include "t8132_ext_drm.h" @@ -301,11 +302,16 @@ bool dcp_typec_dual_stream(void) return atomic_read(&dcp_dual_stream_routes) > 0; } -/* Keep native T6030 routes pinned to their owning processor, on either DPIN. */ +/* + * Keep native T6030 and T6050 routes pinned to their owning processor, on + * any DPIN: their external processors are interchangeable, so no ordering + * pass pairs streams with pipelines. + */ bool dcp_typec_ordered_dual_stream(void) { return dcp_fabric_ordered_dual_stream(dcp_typec_dual_stream(), - of_machine_is_compatible("apple,t6030")); + of_machine_is_compatible("apple,t6030") || + of_machine_is_compatible("apple,t6050")); } /* @@ -336,7 +342,28 @@ static bool dcp_typec_port_follows(struct apple_dcp_typec_port *port) static bool dcp_typec_narrows(struct apple_dcp *dcp) { /* This tree narrows on dual-stream machines too. */ - return !dcp_typec_follows_crtc(dcp); + return !dcp_typec_fixed_crtcs() && !dcp_typec_follows_crtc(dcp); +} + +/* + * T6050 establishes a tunnel on a processor before KMS sees its monitor. + * Give each port/pipeline pair a connector whose capabilities stay fixed, + * rather than publishing the current tunnel owner in possible_crtcs. + */ +bool dcp_typec_fixed_crtcs(void) +{ + KUNIT_STATIC_STUB_REDIRECT(dcp_typec_fixed_crtcs); + + return of_machine_is_compatible("apple,t6050"); +} + +static struct apple_connector * +dcp_typec_route_connector(struct apple_dcp_typec_route *route, bool tunnel) +{ + if (dcp_typec_fixed_crtcs()) + return route->connector; + return tunnel && dcp_fabric_dpin_slot(route->tunnel_dpin) ? + route->port->secondary_connector : route->port->connector; } bool dcp_is_typec_only(struct platform_device *pdev) @@ -352,6 +379,23 @@ static int dcp_route_clock_rate(struct apple_dcp_typec_route *route, bool t8132 = dcp_is_t8132_ext(route->dcp); int ret; + /* + * T6050: DP IN n has the PHY's tunnel clock n + 1, which feeds the + * crossbar's pixel clock n + 1. Each clock has its own source. + */ + if (of_machine_is_compatible("apple,t6050")) { + typeof(apple_t6050_dp_tunnel_clock) *fn = + symbol_get(apple_t6050_dp_tunnel_clock); + + if (!fn) + return rate ? -EOPNOTSUPP : 0; + ret = fn(phy, dpin + 1, rate); + symbol_put(apple_t6050_dp_tunnel_clock); + if (!ret) + route->tunnel_rate = rate; + return ret; + } + /* A T8132 dcpext names its source (2 or 4), as T6030 does. */ if (of_machine_is_compatible("apple,t6030") || t8132) { typeof(apple_atc_dp_tunnel_rate_source) *fn; @@ -381,10 +425,27 @@ static int dcp_route_clock_rate(struct apple_dcp_typec_route *route, return ret; } +/* + * The fabric lock excludes new event admission while a route is retired. + * Delivery holds tb_event_lock through its RPC, drops tb_lock before the RPC, + * and never takes the fabric lock. Drain it before changing the route or its binding. + */ +static void dcp_tb_events_retire(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp_typec_fabric_lock); + scoped_guard(mutex, &dcp->tb_event_lock) { + dcp->tb_events = 0; + dcp->tb_event_route = NULL; + dcp->tb_event_generation = 0; + } + cancel_work_sync(&dcp->tb_event_work); +} + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dpxbar_preselect(struct mux_control *mux, int state); +static bool dcp_xbar_t8142(struct mux_control *mux); static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) { @@ -466,10 +527,11 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, { struct apple_dcp *dcp = route->dcp; struct apple_connector *connector = - xbar != route->xbar && route->tunnel_dpin == 1 ? - route->port->secondary_connector : route->port->connector; + dcp_typec_route_connector(route, xbar != route->xbar); int ret; + dcp_tb_events_retire(dcp); + /* Never select a DP PHY output that has no DP PHY behind it. */ if (xbar == route->xbar && dcp_typec_route_no_dp_alt(route)) return -EOPNOTSUPP; @@ -598,7 +660,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, /* A newly selected output starts with its clocks off. */ dcp->direct_xbar_up = NULL; dcp->dptx_tunnel = route->tunnel; - /* crossbar controls are dpphy, dpin0, dpin1: same order as the DFP port */ + /* crossbar controls are dpphy, then dpin0..3: same order as the DFP port */ dcp->dptx_dfp_port = route->tunnel ? xbar - &route->xbar->chip->mux[0] : 0; dcp->tb_clock_ok = false; } @@ -617,6 +679,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) bool ended_attachment; int ret = 0; + dcp_tb_events_retire(dcp); + WRITE_ONCE(dcp->peer_starved, false); WRITE_ONCE(dcp->typec_narrow, false); /* A peer reserved for this route's display is not needed any more. */ @@ -639,18 +703,19 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return route->deactivate_error; if (!route->deactivate_xbar_done) { dcp_typec_route_phy_off(route); - if (!route->tunnel || route->xbar_up) + if (!route->tunnel || route->xbar_up) { ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else + } else { ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - /* - * A T602X crossbar has no preselection, and a T8132 - * tunnel output that never came up was never selected: - * it is idle. - */ - if (ret == -EOPNOTSUPP && route->tunnel && !route->xbar_up && - dcp_typec_route_no_dp_alt(route)) - ret = 0; + /* + * The T8142-layout crossbar writes an output only when it + * is selected: one never selected is idle already. + */ + if (ret == -EOPNOTSUPP && + (dcp_xbar_t8142(route->active_xbar) || + dcp_typec_route_no_dp_alt(route))) + ret = 0; + } /* Deselect always unlocks: never consume that selection twice. */ route->deactivate_xbar_done = true; route->xbar_up = false; @@ -711,7 +776,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) schedule_work(&connector->bl_sync_wq); /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) + if (connector->port_encoder && !dcp_typec_fixed_crtcs()) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } @@ -2006,11 +2071,27 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, return 0; } -/* Limited to the T6030 14.7 pipeline; other T602X tunnels keep their order. */ +/* The T8142-layout crossbar (one per ATC on T6050) */ +static bool dcp_xbar_t8142(struct mux_control *mux) +{ + return mux && of_device_is_compatible(mux->chip->dev.parent->of_node, + "apple,t8142-display-crossbar"); +} + +/* + * Tunnel routes whose crossbar must point the DP IN output at its pipeline + * before the firmware uses AUX through it, i.e. from DCP's Activate: the T6030 + * 14.7 pipeline on its T602X crossbar (other T602X tunnels keep their order) + * and firmware 27 on the T8142-layout crossbar. Selecting routes the output + * only; its clocks follow the link. + */ static bool dcp_t602x_tunnel_route(struct apple_dcp_typec_route *route) { - return route && route->tunnel && route->active_xbar && - route->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + if (!route || !route->tunnel || !route->active_xbar) + return false; + if (route->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_xbar_t8142(route->active_xbar); + return route->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && of_device_is_compatible(route->active_xbar->chip->dev.parent->of_node, "apple,t6020-display-crossbar"); } @@ -2206,6 +2287,27 @@ static int dcp_t8132_crossbar_up_locked(struct apple_dcp *dcp, u32 unit) return ret; } +/* + * The T8142-layout crossbar takes the pixel clock and coding of an output + * before its link comes up: the tunnel's own clock, DP IN n + 1. A running + * link keeps its configuration. + */ +static int dcp_tunnel_xbar_configure(struct apple_dcp_typec_route *route) +{ + typeof(&apple_dpxbar_link_configure) configure; + int ret; + + if (!dcp_xbar_t8142(route->active_xbar)) + return 0; + configure = symbol_get(apple_dpxbar_link_configure); + if (!configure) + return -ENODEV; + ret = configure(route->active_xbar, route->tunnel_dpin + 1, + route->tunnel_rate == 0x81 || route->tunnel_rate == 0x82); + symbol_put(apple_dpxbar_link_configure); + return ret == -EBUSY ? 0 : ret; +} + int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit) { struct apple_dcp_typec_route *route; @@ -2232,15 +2334,23 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit) if (!ret) route->xbar_up = true; /* T602X DP IN selection reserves and routes the mux only. */ - if (!ret && dcp_t602x_tunnel_route(route)) - ret = dcp_dpxbar_link(route->active_xbar, true); + if (!ret && dcp_t602x_tunnel_route(route)) { + ret = dcp_tunnel_xbar_configure(route); + if (!ret) + ret = dcp_dpxbar_link(route->active_xbar, true); + } } else { - ret = dcp_dpxbar_link(route->active_xbar, true); + ret = dcp_tunnel_xbar_configure(route); + if (!ret) + ret = dcp_dpxbar_link(route->active_xbar, true); } if (ret) { dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); return ret; } + /* T8142 layout: let the connection settle before the DP IN takes it. */ + if (dcp_xbar_t8142(route->active_xbar)) + usleep_range(5000, 6000); return dcp_tunnel_dpin_locked(dcp, true); } @@ -2252,6 +2362,7 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit) int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) { struct apple_dcp_typec_route *route; + int ret; guard(mutex)(&dcp->tb_lock); /* A T8132 dcpext has no second tile. */ @@ -2273,7 +2384,14 @@ int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) if (dcp_is_t8132_ext(dcp)) return dcp_t8132_crossbar_down_locked(dcp, route, "WillChangeLinkConfiguration"); dcp_tunnel_dpin_locked(dcp, false); - return dcp_dpxbar_link(route->active_xbar, false); + ret = dcp_dpxbar_link(route->active_xbar, false); + /* + * T8142 layout (T6050): the DP IN goes active again at once, so its + * AUX path is there while DCP trains the new link configuration. + */ + if (dcp_xbar_t8142(route->active_xbar)) + dcp_tunnel_dpin_locked(dcp, true); + return ret; } /* @@ -2284,12 +2402,51 @@ int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) * The output is the borrowed Type-C route's, or the fixed route's. Other * SoCs clock it at select. Runs inside a DCP apcall: only tb_lock. */ +static int dcp_direct_crossbar_link_t6050(struct apple_dcp *dcp, bool up) +{ + typeof(&apple_dp_phy_link_config) get_config; + typeof(&apple_dpxbar_link_configure) configure; + struct mux_control *xbar; + unsigned int pclk; + bool uhbr; + int ret; + + guard(mutex)(&dcp->tb_lock); + /* A borrowed Type-C route has its own mux; xbar remains the fixed + * output's mux (or NULL on a Type-C-only pipeline). */ + xbar = dcp->active_typec_route ? + dcp->active_typec_route->active_xbar : dcp->xbar; + if (dcp->dptx_tunnel || !dcp->phy || !xbar) + return 0; + get_config = symbol_get(apple_dp_phy_link_config); + if (!get_config) + return dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? -ENODEV : 0; + ret = get_config(dcp->phy, &pclk, &uhbr); + symbol_put(apple_dp_phy_link_config); + if (ret == -EOPNOTSUPP) + return 0; + if (!up && ret == -ENOLINK) + ret = 0; + if (ret) + return ret; + if (!up) + return dcp_dpxbar_link(xbar, false); + configure = symbol_get(apple_dpxbar_link_configure); + if (!configure) + return -ENODEV; + ret = configure(xbar, pclk, uhbr); + symbol_put(apple_dpxbar_link_configure); + return ret ? ret : dcp_dpxbar_link(xbar, true); +} + int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) { struct apple_dcp_typec_route *route; struct mux_control *xbar = NULL; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_direct_crossbar_link_t6050(dcp, up); if (!of_machine_is_compatible("apple,t6030")) return 0; guard(mutex)(&dcp->tb_lock); @@ -2392,7 +2549,10 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, dcp_dpxbar_link(route->active_xbar, false); ret = dcp_route_clock_rate(route, phy, route->tunnel_dpin, link_rate); dcp->tb_clock_ok = !ret && link_rate; - if (ret) + if (ret == -EOPNOTSUPP) + dev_warn(dcp->dev, "no DP tunnel pixel clock on dpin%u: link rate 0x%x refused\n", + route->tunnel_dpin, link_rate); + else if (ret) dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", link_rate, ret); return ret; @@ -2445,7 +2605,8 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, u32 unit, bool active) return dcp->split.set_active(dcp->split.binding, active); } /* Route the upstream engine before firmware begins AUX negotiation. - * T602X DP IN .set does not enable clocks; DidChange does that later. + * Selecting a T602X or T8142-layout DP IN output does not enable its + * clocks; DidChange does that later. */ route = dcp->active_typec_route; if (!route || !dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) @@ -2513,13 +2674,14 @@ struct dcp_tb_attach_context { static struct mux_control * dcp_typec_tunnel_ctl(struct apple_dcp_typec_route *route, unsigned int dpin) { - if (dpin > 1 || (dcp_route_tunnel_clock_blocked(route) & BIT(dpin))) + if (dpin >= DCP_FABRIC_DPIN_MAX || (dcp_route_tunnel_clock_blocked(route) & BIT(dpin))) return NULL; if (route->dpin[dpin]) return route->dpin[dpin]; /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ if (route->xbar != &route->xbar->chip->mux[0] || - route->xbar->chip->controllers < 3) + route->xbar->chip->controllers < 3 || + route->xbar->chip->controllers < 2 + dpin) return NULL; return &route->xbar->chip->mux[1 + dpin]; } @@ -2531,6 +2693,7 @@ static int dcp_tb_candidate(void *data) struct apple_dcp_typec_route *best, *planned = NULL; struct mux_control *ctl; unsigned int dpin = ctx->dpin; + unsigned int stream = dcp_fabric_dpin_slot(dpin); bool ordered; int ret; @@ -2541,7 +2704,7 @@ static int dcp_tb_candidate(void *data) */ ordered = dcp_typec_keep_order(); if (ordered) - planned = dcp_typec_rebalance_locked(port, dpin); + planned = dcp_typec_rebalance_locked(port, stream); ctx->planned = planned; { @@ -2551,7 +2714,8 @@ static int dcp_tb_candidate(void *data) .hybrid_first = dcp_typec_hybrid_first(), }; struct dcp_fabric_route *chosen; - bool connector_present = dpin ? !!port->secondary_connector : !!port->connector; + bool connector_present = stream ? !!port->secondary_connector : + !!port->connector; dcp_fabric_snapshot_port(port, true); chosen = dcp_fabric_tunnel_candidate(&port->core, @@ -2594,6 +2758,7 @@ static void dcp_tb_rollback(void *data) struct dcp_tb_attach_context *ctx = data; struct apple_dcp *dcp = ctx->best->dcp; + dcp_tb_events_retire(dcp); /* The failed attempt's binding is retired before the provider hears of it. */ guard(mutex)(&dcp->tb_lock); dcp->tb_dpin_set_active = NULL; @@ -2693,6 +2858,7 @@ static void dcp_tb_binding_retire(struct apple_dcp *dcp, struct apple_dcp_typec_route *route) { lockdep_assert_held(&dcp_typec_fabric_lock); + dcp_tb_events_retire(dcp); scoped_guard(mutex, &dcp->tb_lock) { dcp->tb_generation = 0; if (route) @@ -2821,13 +2987,20 @@ static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, * holds before any of its services are up. On T6030 every Type-C pipeline is * an external processor. A port with no routes is not judged here. */ +/* + * Ports whose pipelines cannot take a tile on their second DPTX port: the + * T6030 external processors, and firmware 27 (T6050), whose pipelines drive + * one DP IN each, so that two monitors behind a dock get two processors. + */ static bool dcp_typec_port_untileable(struct apple_dcp_typec_port *port) { struct apple_dcp_typec_route *route; /* No tiles on a T8132 dcpext. */ list_for_each_entry(route, &port->routes, port_link) - if (!route->dcp->external && !dcp_typec_route_no_dp_alt(route)) + if (!route->dcp->external && + !dcp_typec_route_no_dp_alt(route) && + route->dcp->fw_compat != DCP_FIRMWARE_V_27_0) return false; return !list_empty(&port->routes); } @@ -2916,7 +3089,8 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, static int dcp_tb_release_locked(struct apple_dcp_typec_port *port, unsigned int dpin) { - struct apple_dcp_typec_route **slot = dpin ? &port->secondary_owner : &port->owner; + struct apple_dcp_typec_route **slot = dcp_fabric_dpin_slot(dpin) ? + &port->secondary_owner : &port->owner; struct apple_dcp *dcp; int ret; @@ -2927,6 +3101,7 @@ static int dcp_tb_release_locked(struct apple_dcp_typec_port *port, return ret; } dcp = (*slot)->dcp; + dcp_tb_events_retire(dcp); /* * A refused display release keeps the route and its binding owned. * port->hpd is the port's, and stays set while the other DP IN @@ -2976,7 +3151,7 @@ static int dcp_tb_dispatch_request(void *data, const struct dcp_fabric_port *fou unsigned int dpin = ctx->dpin; int ret; - slot = dpin ? &port->secondary_owner : &port->owner; + slot = dcp_fabric_dpin_slot(dpin) ? &port->secondary_owner : &port->owner; if (dpin == 1 && (tiled_split || port->split_dcp)) { ret = dcp_tb_split_locked(port, ctx); @@ -3060,7 +3235,7 @@ static int dcp_tb_dispatch_request(void *data, const struct dcp_fabric_port *fou static void dcp_tb_finish_request(struct apple_dcp_typec_port *port, const struct dcp_tb_attach_context *request) { - struct apple_dcp_typec_route *route = request->dpin ? + struct apple_dcp_typec_route *route = dcp_fabric_dpin_slot(request->dpin) ? port->secondary_owner : port->owner; struct apple_dcp *dcp; bool matching; @@ -3117,7 +3292,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, .binding = binding, }; - if (!connector_np || dpin > 1 || !generation || (active && !set_active)) + if (!connector_np || dpin >= DCP_FABRIC_DPIN_MAX || !generation || + (active && !set_active)) return -EINVAL; guard(mutex)(&dcp_typec_fabric_lock); list_for_each_entry(pos, &dcp_typec_ports, link) { @@ -3131,6 +3307,110 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); +enum { + DCP_TB_EVENT_IRQ_HPD, + DCP_TB_EVENT_HPD, +}; + +/* + * A DP source learns of a sink's request through IRQ_HPD and of a sink that + * went away or came back through the HPD level. On a tunnel, both arrive at + * the DP IN adapter: pass them to the DPTX port as its HPD input would, so + * that DCP services the request (and retrains a link the sink lost). + */ +void dcp_tb_event_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, tb_event_work); + struct dptx_port *dptx = &dcp->dptxport[0]; + struct apple_epic_service *service; + struct apple_dcp_typec_route *route; + bool irq, changed, hpd, valid; + ktime_t now = ktime_get(); + int ret; + + /* Keep the identity check and RPC ordered before attachment retirement. */ + guard(mutex)(&dcp->tb_event_lock); + route = dcp->tb_event_route; + irq = test_and_clear_bit(DCP_TB_EVENT_IRQ_HPD, &dcp->tb_events); + changed = test_and_clear_bit(DCP_TB_EVENT_HPD, &dcp->tb_events); + hpd = dcp->tb_event_hpd; + if ((!irq && !changed) || !route) + return; + scoped_guard(mutex, &dcp->tb_lock) + valid = dcp->active_typec_route == route && route->tunnel && + dcp_fabric_callback_valid(route->tunnel_generation, + dcp->tb_event_generation) && + dcp->tb_generation == dcp->tb_event_generation; + if (!valid) + return; + service = READ_ONCE(dptx->service); + if (!service || !dptx_service_available(dptx) || !READ_ONCE(dptx->connected)) { + dev_info(dcp->dev, "DP tunnel:%s%s with no connected link: not passed\n", + irq ? " IRQ_HPD" : "", changed ? (hpd ? " HPD high" : " HPD low") : ""); + return; + } + if (changed) { + dcp->tb_hpd_count++; + ret = dptxport_set_hpd(service, hpd); + dev_info(dcp->dev, "DP tunnel: HPD %s (#%u) passed to the firmware: %d\n", + hpd ? "high" : "low", dcp->tb_hpd_count, ret); + } + if (irq && hpd) { + s64 since = dcp->tb_irq_hpd_count ? + ktime_ms_delta(now, dcp->tb_irq_hpd_last) : -1; + + dcp->tb_irq_hpd_count++; + dcp->tb_irq_hpd_last = now; + ret = dptxport_sink_irq(service); + dev_info(dcp->dev, "DP tunnel: sink IRQ_HPD (#%u, %lld ms after the last) passed to the firmware: %d\n", + dcp->tb_irq_hpd_count, since, ret); + } +} + +int apple_dcp_tb_dp_event(struct device_node *connector_np, unsigned int dpin, + u64 generation, unsigned int events, bool hpd) +{ + struct apple_dcp_typec_port *pos; + struct apple_dcp_typec_route *attachment = NULL; + struct apple_dcp *dcp = NULL; + + if (!connector_np || dpin >= DCP_FABRIC_DPIN_MAX || !generation) + return -EINVAL; + guard(mutex)(&dcp_typec_fabric_lock); + list_for_each_entry(pos, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *route; + + if (pos->connector_np != connector_np) + continue; + route = dcp_fabric_dpin_slot(dpin) ? pos->secondary_owner : pos->owner; + if (route && route->tunnel && route->tunnel_dpin == dpin && + route->tunnel_generation == generation) { + dcp = route->dcp; + attachment = route; + break; + } + } + if (!dcp) + return -ESTALE; + guard(mutex)(&dcp->tb_event_lock); + if (dcp->tb_event_route != attachment || dcp->tb_event_generation != generation) + dcp->tb_events = 0; + dcp->tb_event_route = attachment; + dcp->tb_event_generation = generation; + if (events & APPLE_DP_TUNNEL_HPD_CHANGED) { + WRITE_ONCE(dcp->tb_event_hpd, hpd); + set_bit(DCP_TB_EVENT_HPD, &dcp->tb_events); + } + if (events & APPLE_DP_TUNNEL_IRQ_HPD) { + if (!(events & APPLE_DP_TUNNEL_HPD_CHANGED)) + WRITE_ONCE(dcp->tb_event_hpd, true); + set_bit(DCP_TB_EVENT_IRQ_HPD, &dcp->tb_events); + } + queue_work(system_dfl_wq, &dcp->tb_event_work); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_event); + /* * Direct DP-alt routes and Thunderbolt tunnels follow the CRTC selected by * a modeset on 12.3/13.5. A tunnel keeps its Thunderbolt binding: only the @@ -3419,6 +3699,7 @@ static int dcp_follow_release(struct apple_dcp_typec_route *route) bool tunnel = route->tunnel; int ret; + dcp_tb_events_retire(dcp); /* Revoked sessions cannot touch the new route after a reconnect wait. */ ret = dcp_dptx_park(dcp); if (ret) @@ -3942,19 +4223,25 @@ bool dcp_typec_port_unusable(unsigned int idx) return true; } -void dcp_typec_port_set_connector(unsigned int idx, bool secondary, - struct apple_connector *connector) +static void dcp_typec_port_bind_connector(struct apple_dcp_typec_port *port, + bool secondary, + struct apple_connector *connector, + struct platform_device *pipeline) { - struct apple_dcp_typec_port *port; struct apple_dcp_typec_route *owner; - guard(mutex)(&dcp_typec_fabric_lock); + lockdep_assert_held(&dcp_typec_fabric_lock); - port = dcp_typec_port_by_index(idx); - if (!port) - return; + if (pipeline) { + struct apple_dcp *dcp = platform_get_drvdata(pipeline); + struct apple_dcp_typec_route *route = dcp_typec_port_route(port, dcp); - if (secondary) { + if (WARN_ON(!route)) + return; + route->connector = connector; + owner = route == port->owner || route == port->secondary_owner ? + route : NULL; + } else if (secondary) { port->secondary_connector = connector; owner = port->secondary_owner; } else { @@ -3985,6 +4272,18 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, } } +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, + struct apple_connector *connector, + struct platform_device *pipeline) +{ + struct apple_dcp_typec_port *port; + + guard(mutex)(&dcp_typec_fabric_lock); + port = dcp_typec_port_by_index(idx); + if (port) + dcp_typec_port_bind_connector(port, secondary, connector, pipeline); +} + static void dcp_typec_route_unregister(void *data) { struct apple_dcp_typec_route *route = data; @@ -4034,7 +4333,7 @@ static void dcp_typec_route_unregister(void *data) static unsigned int dcp_typec_route_endpoints(struct device_node *connector_np) { struct device_node *port __free(device_node) = - of_graph_get_port_by_id(connector_np, 3); + apple_dp_tunnel_routes_port(connector_np); unsigned int count = 0; if (!port) @@ -4053,9 +4352,10 @@ static int dcp_typec_route_wiring(struct apple_dcp_typec_route *route, enum dcp_fabric_wiring wiring; char name[24]; unsigned int i; - bool named[2]; + bool named[DCP_FABRIC_DPIN_MAX]; - for (i = 0; i < 2; i++) { + /* DP IN 0/1 decide the stream wiring; DP IN 2/3 only add outputs. */ + for (i = 0; i < DCP_FABRIC_DPIN_MAX; i++) { snprintf(name, sizeof(name), "typec%u-dpin%u", index, i); named[i] = of_property_match_string(dev->of_node, "mux-control-names", name) >= 0; if (!named[i]) @@ -4065,9 +4365,12 @@ static int dcp_typec_route_wiring(struct apple_dcp_typec_route *route, return PTR_ERR(route->dpin[i]); if (route->xbar != &route->xbar->chip->mux[0] || route->xbar->chip->controllers < 3 || + route->xbar->chip->controllers < 2 + i || route->dpin[i] != &route->xbar->chip->mux[1 + i]) return -EINVAL; } + if ((named[2] || named[3]) && !(named[0] && named[1])) + return -EINVAL; if (!named[0] && !named[1]) { snprintf(name, sizeof(name), "typec%u-usb4", index); if (of_property_match_string(dev->of_node, "mux-control-names", name) >= 0) { @@ -4866,6 +5169,21 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, WRITE_ONCE(connector->connected, !iomfb_m4_error(dcp) && dcp->active); } +static void dcp_typec_port_unlink(struct apple_dcp_typec_port *port, + struct drm_device *drm) +{ + struct apple_dcp_typec_route *route; + + lockdep_assert_held(&dcp_typec_fabric_lock); + list_for_each_entry(route, &port->routes, port_link) + if (route->connector && route->connector->base.dev == drm) + route->connector = NULL; + if (port->connector && port->connector->base.dev == drm) + port->connector = NULL; + if (port->secondary_connector && port->secondary_connector->base.dev == drm) + port->secondary_connector = NULL; +} + /* Called after component unbind has drained all firmware callbacks. The * platform devices and their Type-C muxes can outlive this DRM instance. */ @@ -4876,13 +5194,8 @@ void dcp_unlink(struct drm_device *drm) guard(mutex)(&dcp_typec_fabric_lock); - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->connector && port->connector->base.dev == drm) - port->connector = NULL; - if (port->secondary_connector && - port->secondary_connector->base.dev == drm) - port->secondary_connector = NULL; - } + list_for_each_entry(port, &dcp_typec_ports, link) + dcp_typec_port_unlink(port, drm); drm_for_each_crtc(crtc, drm) { struct apple_crtc *apple_crtc = to_apple_crtc(crtc); @@ -4926,3 +5239,14 @@ void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp) } } } + +int dcp_typec_test_link(struct platform_device *pdev, bool enabled) +{ + guard(mutex)(&dcp_typec_fabric_lock); + return enabled ? dcp_dptx_connect_oob(pdev, 0) : + dcp_dptx_disconnect_oob(pdev, 0); +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +#include "dcp-fabric-topology-test.c" +#endif diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index f68b91e4d18f8..0805c54d39051 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -30,10 +30,12 @@ struct apple_dcp_typec_route { struct dcp_fabric_route core; struct apple_dcp *dcp; struct apple_dcp_typec_port *port; + /* T6050: this port/pipeline pair has a connector with a fixed CRTC. */ + struct apple_connector *connector; struct list_head port_link; struct phy *phy; struct mux_control *xbar; - struct mux_control *dpin[2]; + struct mux_control *dpin[DCP_FABRIC_DPIN_MAX]; bool dual_stream; struct typec_mux_dev *typec_mux; u32 dptx_phy; @@ -43,6 +45,8 @@ struct apple_dcp_typec_route { bool phy_powered; /* crossbar output actually selected: xbar (dpphy) or a Thunderbolt dpin */ unsigned int tunnel_dpin; + /* T6050: DP link rate code the tunnel's pixel clock runs at, 0 if stopped */ + u8 tunnel_rate; u64 tunnel_generation; struct mux_control *active_xbar; bool tunnel; @@ -72,7 +76,9 @@ struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); bool dcp_typec_port_unusable(unsigned int idx); void dcp_typec_port_set_connector(unsigned int idx, bool secondary, - struct apple_connector *connector); + struct apple_connector *connector, + struct platform_device *pipeline); +bool dcp_typec_fixed_crtcs(void); bool dcp_typec_dual_stream(void); bool dcp_typec_ordered_dual_stream(void); void dcp_typec_reorder(void); @@ -127,4 +133,6 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); bool dcp_is_external_h17p(struct apple_dcp *dcp); +int dcp_typec_test_link(struct platform_device *pdev, bool enabled); + #endif /* __APPLE_DCP_FABRIC_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index e7d3d9891aa4f..9660a474607f8 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" +#include "iomfb_v27.h" #include "iomfb-state.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" @@ -37,6 +39,7 @@ #include "iomfb_v14_7_tpl.h" #include "iomfb_v26_6.h" #include "iomfb_v26_6_desktop.h" +#include "dcp-peer-frames-v27.h" #include "epic/dpavservep.h" #include "version_utils.h" @@ -232,6 +235,7 @@ enum dcp_iomfb_method_profile { }; struct apple_dcp_hw_data { + u8 firmware_dma_bits; u32 num_dptx_ports; enum dcp_iomfb_method_profile iomfb_method_profile; /* @@ -415,6 +419,10 @@ struct apple_dcp { struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; + /* Protected by dcp_peer_lock; never take the physical route lock in + * an AFK callback (a route operation may be waiting for that reply). + */ + struct dcp_peer_frames_v27 peer_frames; struct apple_dcp_hw_data hw; bool external_prepared; @@ -450,7 +458,8 @@ struct apple_dcp { /* A desktop 26.x external processor (apple,t6041-dcpext) */ bool desktop_external; /* - * T8122 dcpext: the panel's DRM device never waits on this processor. + * A failsafe dcpext (dcp_ext_is_failsafe(): T8122's, T6041's, T6050's): + * the DRM device of the other displays never waits on this processor. * A probe or bind failure leaves it in the device as an unusable pipe * (ext_dead): no CRTC, Type-C routes disabled or ignored, RTKit * messages dropped. See dcp_platform_probe(). @@ -505,6 +514,8 @@ struct apple_dcp { /* DCP has crashed */ bool crashed; + /* CoreAnalytics events dumped in full (dcpep_cb_core_analytics_v27). */ + u32 analytics_dumps; DECLARE_BITMAP(iomfb_surfaces, DCP_MAX_PLANES); @@ -516,6 +527,18 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; struct clk *clk_194; + struct clk_bulk_data *firmware_clocks; + unsigned int num_firmware_clocks; + u64 disp_firmware_base[MAX_DISP_REGISTERS]; + u32 dfb_format, dfb_compression; + u64 dfb_region_size; + bool dfb_format_valid, dfb_compression_valid; + struct list_head iomfb_boolean_properties; + u32 iomfb_boolean_property_count; + u32 external_pipes_v27, external_displays_v27, external_suppression_v27; + struct dcp_swap_submit_req_v27 present_v27; + /* Both paired wire records belong to the primary endpoint lock. */ + struct dcp_swap_submit_req_v27 peer_present_v27; /* DCP shared memory */ void *shmem; @@ -620,6 +643,8 @@ struct apple_dcp { /* completion for active turning true */ struct completion start_done; + /* IOMFB has published fresh modes and acknowledged a Type-C display. */ + struct completion typec_display_ready; /* Is the DCP booted? */ bool active; @@ -839,6 +864,10 @@ struct apple_dcp { struct iomfb_transaction *active; struct task_struct *owner; struct work_struct work; + spinlock_t callback_lock; + struct list_head callbacks; + struct work_struct callback_work; + unsigned int callback_count; struct delayed_work timeout; struct delayed_work backlight_retry; unsigned long deadline; @@ -876,6 +905,21 @@ struct apple_dcp { /* T6030 HDMI converter service requests, see dcp-hdmi.c */ struct dcp_hdmi *hdmi; + /* + * Interrupts of a Thunderbolt DP tunnel's DP IN adapter, to pass to + * the DPTX port; see apple_dcp_tb_dp_event(). + */ + struct work_struct tb_event_work; + /* Serializes event delivery against attachment retirement. */ + struct mutex tb_event_lock; + struct apple_dcp_typec_route *tb_event_route; + u64 tb_event_generation; + unsigned long tb_events; + bool tb_event_hpd; + unsigned int tb_irq_hpd_count; + unsigned int tb_hpd_count; + ktime_t tb_irq_hpd_last; + /* Kept last: the template instances share the layout above. */ struct dcp_desktop_state *desktop; /* Internal panel identity; NULL without an internal panel. */ @@ -892,7 +936,8 @@ static inline bool iomfb_is_m4(const struct apple_dcp *dcp) /* Presents carry nits; one serialized IOMFB owner. */ static inline bool dcp_presents_nits(const struct apple_dcp *dcp) { - return (dcp->fw_compat == DCP_FIRMWARE_H17P && + return ((dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || iomfb_is_m4(dcp); } @@ -901,6 +946,7 @@ static inline bool dcp_presents_nits(const struct apple_dcp *dcp) static inline bool dcp_fail_stop(const struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || iomfb_is_m4(dcp); } @@ -930,6 +976,8 @@ void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp); int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits); +int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, + const void *data, size_t size, void *cookie); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); @@ -1039,6 +1087,33 @@ static inline bool dcp_ext_dptx_607(const struct apple_dcp *dcp) dcp->hw.fw_14_7_endpoints; } +/* + * Whether this DCP reads a surface's element geometry as the surface record + * defines it: bytes per element (pix_size, a linear plane's tile_size), the + * element's width and height in pixels, and compression information only for + * a compressed plane. Only firmware 27 (DCP_FIRMWARE_V_27_0, which only + * T6050's DCP hw data selects) is tested with it. Earlier firmware keeps the + * fixed 1/1/1 and has_compr_info 1 it has run on: before widening this, + * check a linear XRGB8888 and an NV12 swap, and the default framebuffer, on + * 12.x-26.x firmware. + */ +static inline bool dcp_surface_element_geometry(const struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_27_0; +} + +/* + * Whether @pa, a physical address the DCP firmware placed a buffer at, can be + * DRAM. On Apple SoCs every device register window lies below DRAM, so an + * address below all of Linux's RAM may be a device's registers and is never + * mapped as memory. The firmware's own segments lie in DRAM beside Linux's RAM. + */ +static inline bool dcp_fw_phys_in_dram(phys_addr_t pa) +{ + return pa && region_intersects(0, pa, IORESOURCE_SYSTEM_RAM, + IORES_DESC_NONE) != REGION_DISJOINT; +} + #define DCP_AUDIO_MAX_CHANS 15 void dcp_trace_panel_property(struct apple_dcp *dcp, int tag, const char *key, diff --git a/drivers/gpu/drm/apple/dcp-peer-frames-v27.h b/drivers/gpu/drm/apple/dcp-peer-frames-v27.h new file mode 100644 index 0000000000000..abb7798a1764f --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-frames-v27.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_FRAMES_V27_H +#define APPLE_DCP_PEER_FRAMES_V27_H + +#include + +/* Protected by dcp_peer_lock. Sequence comparisons cover less than 2^31 + * outstanding swaps, as in the existing single-pipe retirement path. + */ +struct dcp_peer_frames_v27 { + u32 first, submitted, engine[2], retired; + bool active, armed, failed, have[2], have_retired; +}; + +static inline void dcp_peer_frame_submit_v27(struct dcp_peer_frames_v27 *s, u32 id) +{ + if (!s->armed) { + s->first = id; + s->armed = true; + } + s->submitted = id; +} + +static inline bool dcp_peer_frame_complete_v27(struct dcp_peer_frames_v27 *s, + unsigned int engine, u32 id, + u32 *watermark) +{ + u32 done; + + if (!s->active || s->failed || !s->armed || engine > 1 || + (s32)(id - s->first) < 0 || (s32)(s->submitted - id) < 0 || + (s->have[engine] && (s32)(id - s->engine[engine]) <= 0)) + return false; + s->engine[engine] = id; + s->have[engine] = true; + if (!s->have[0] || !s->have[1]) + return false; + done = (s32)(s->engine[0] - s->engine[1]) < 0 ? s->engine[0] : s->engine[1]; + if (s->have_retired && (s32)(done - s->retired) <= 0) + return false; + s->retired = *watermark = done; + s->have_retired = true; + return true; +} +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc new file mode 100644 index 0000000000000..a5aa449b446dd --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Paired modeset bring-up, opt-in only. Scanout is a separate step. */ +/* The power-state step starts one asynchronous AP request per engine. + * Submit both before waiting: either firmware may rendezvous with its peer. + */ +static int dcp_peer_probe_power(struct apple_dcp *dcp, struct apple_dcp *peer, + bool on) +{ + struct dcp_peer_video_wait *wait; + struct dcp_method_entry method = { .name = "peer_power_test", + .tag = {'A', '4', '8', '4'} }; + u8 power[16] = {}, response[8]; + int ret; + + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + kref_init(&wait->refs); + init_completion(&wait->done); + wait->size = sizeof(response); + put_unaligned_le64(on, power); + kref_get(&wait->refs); + dcp_push(dcp, false, &method, sizeof(power), sizeof(response), power, + dcp_peer_video_done, wait); + ret = dcp_peer_video_call(peer, "A484", power, sizeof(power), + response, sizeof(response)); + if (!ret && get_unaligned_le32(response + 4)) + ret = -EIO; + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000))) + ret = ret ?: -ETIMEDOUT; + else if (!ret && get_unaligned_le32(wait->reply + 4)) + ret = -EIO; + kref_put(&wait->refs, dcp_peer_video_free); + dev_info(dcp->dev, "peer power %u result=%d\n", on, ret); + return ret; +} + +static int dcp_peer_probe_quiesce(struct apple_dcp *dcp, struct apple_dcp *peer) +{ + __le32 manage = 0; + int ret; + + /* Setting the digital-out mode: pair API power0, 100ms, + * then manage_elements(false), A362, on each engine before A365. + * Only discard AP state after both engines have acknowledged power0. + */ + { + guard(mutex)(&dcp_peer_lock); + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + } + ret = dcp_peer_probe_power(dcp, peer, false); + if (ret) + return ret; + msleep(100); + ret = dcp_peer_video_call(dcp, "A362", &manage, sizeof(manage), NULL, 0); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A362", &manage, sizeof(manage), NULL, 0); + if (ret) + return ret; + { + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames = (struct dcp_peer_frames_v27){}; + dcp->peer_tiling.member = peer->peer_tiling.member = false; + dcp->peer_tiling.vftg = peer->peer_tiling.vftg = false; + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + dcp->peer_tiling.phase = peer->peer_tiling.phase = 0; + dcp->peer_tiling.modeset_state = peer->peer_tiling.modeset_state = 0; + dcp->peer_tiling.ready = peer->peer_tiling.ready = false; + } + return 0; +} + +static int dcp_peer_probe_prepare_vftg(struct apple_dcp *dcp, struct apple_dcp *peer, + const struct dcp_peer_insert_video_req_v27 *video) +{ + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_reserved != peer || peer->peer_owner != dcp) + return -ENODEV; + /* These are set BEFORE A366/A365, but membership is marked only after + * both merge-mode insertions have accepted the derived video data. + */ + dcp->peer_tiling.vftg = peer->peer_tiling.vftg = true; + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + dcp->peer_tiling.primary = true; + peer->peer_tiling.primary = false; + dcp->peer_tiling.width = peer->peer_tiling.width = + get_unaligned_le32(video->video.timing + 8); + dcp->peer_tiling.height = peer->peer_tiling.height = + get_unaligned_le32(video->video.timing + 0x28); + dcp->peer_tiling.x = dcp->peer_tiling.y = peer->peer_tiling.y = 0; + peer->peer_tiling.x = dcp->peer_tiling.width; + return 0; +} + +static int dcp_peer_probe_copy_dsc(struct apple_dcp *primary, struct apple_dcp *peer) +{ + u8 caps_req[36] = {}, caps[32], bpp_req[4] = {}, bpp[8]; + u8 hints_req[12] = {}, hints[12]; + u32 required; + int ret; + + ret = dcp_peer_video_call_stream(primary, true, "A369", NULL, 0, &required, 4); + if (ret || !(required & 1)) + return ret; + ret = dcp_peer_video_call_stream(primary, true, "A367", caps_req, 36, caps, 32); + if (ret) + return ret; + memcpy(caps_req, caps, 32); + ret = dcp_peer_video_call(peer, "A368", caps_req, 36, caps, 32); + if (ret) + return ret; + ret = dcp_peer_video_call_stream(primary, true, "A370", bpp_req, 4, bpp, 8); + if (ret || get_unaligned_le32(bpp + 2)) + return ret ?: -EIO; + memcpy(bpp_req, bpp, 2); + ret = dcp_peer_video_call(peer, "A371", bpp_req, 4, bpp, 8); + if (ret || get_unaligned_le32(bpp + 2)) + return ret ?: -EIO; + ret = dcp_peer_video_call_stream(primary, true, "A372", hints_req, 12, hints, 12); + if (ret || get_unaligned_le32(hints + 8)) + return ret ?: -EIO; + memcpy(hints_req, hints, 8); + ret = dcp_peer_video_call(peer, "A373", hints_req, 12, hints, 12); + if (ret || get_unaligned_le32(hints + 8)) + return ret ?: -EIO; + dev_info(primary->dev, "peer DSC copied: bpp=%u hints=%*ph\n", + get_unaligned_le16(bpp), 8, hints); + return 0; +} + +static int dcp_peer_probe_modeset(struct apple_dcp *dcp, struct apple_dcp *peer, + const struct dcp_peer_insert_video_req_v27 *video, + bool firmware_requires_merge) +{ + struct dcp_peer_video_wait *wait; + struct dcp_method_entry method = { .name = "peer_modeset_test", + .tag = {'A', '4', '1', '3'} }; + __le32 mode[2] = {video->color_id, video->timing_id}; + u32 primary_id, peer_id, phase, status; + unsigned long deadline; + int ret; + + { + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_reserved != peer || peer->peer_owner != dcp || + !dcp->peer_tiling.display_id_valid || !peer->peer_tiling.display_id_valid) + return -ENODEV; + dcp->peer_tiling.member = peer->peer_tiling.member = true; + /* The original A364 result is preserved even when the host + * allocation requires a pair for a lower-bandwidth timing. + */ + dcp->peer_tiling.mode_flag = peer->peer_tiling.mode_flag = + !firmware_requires_merge; + } + ret = dcp_peer_probe_power(dcp, peer, true); + if (ret) + return ret; + /* Wait 100ms after both API-power transitions, before swap IDs. */ + msleep(100); + ret = dcp_peer_video_call(dcp, "A360", NULL, 0, &primary_id, 4); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A360", NULL, 0, &peer_id, 4); + if (ret) + return ret; + if (primary_id != peer_id) { + status = max(primary_id, peer_id); + ret = dcp_peer_video_call(primary_id < peer_id ? dcp : peer, + "A361", &status, 4, NULL, 0); + if (ret) + return ret; + } + phase = 1; + dcp_peer_tiling_state_v27(dcp, true, 1, 10, &phase); + dcp_peer_tiling_state_v27(peer, true, 1, 10, &phase); + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + init_completion(&wait->done); + kref_init(&wait->refs); + kref_get(&wait->refs); + wait->size = 4; + dcp_push(dcp, false, &method, sizeof(mode), 4, mode, + dcp_peer_video_done, wait); + /* The primary negotiates the link, signals phase2, and then waits for + * its peer. Waiting for its final ACK first would deadlock the pair. + */ + deadline = jiffies + msecs_to_jiffies(8500); + for (;;) { + phase = 0; + if (!dcp_peer_tiling_state_v27(dcp, false, 1, 10, &phase)) { + ret = -ENODEV; + goto put; + } + if (phase >= 2) + break; + if (completion_done(&wait->done) || time_after_eq(jiffies, deadline)) { + ret = completion_done(&wait->done) ? -EIO : -ETIMEDOUT; + dev_err(dcp->dev, "peer primary ended before phase2: phase=%u result=%#x\n", + phase, get_unaligned_le32(wait->reply)); + goto put; + } + msleep(2); + } + dev_info(dcp->dev, "peer primary reached modeset phase%u\n", phase); + ret = dcp_peer_probe_copy_dsc(dcp, peer); + if (ret) + goto put; + ret = dcp_peer_video_call(peer, "A413", mode, sizeof(mode), &status, 4); + if (ret || status) { + ret = ret ?: -EIO; + goto put; + } + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(8500))) + ret = -ETIMEDOUT; + else if (get_unaligned_le32(wait->reply)) + ret = -EIO; +put: + kref_put(&wait->refs, dcp_peer_video_free); + if (!ret) { + guard(mutex)(&dcp_peer_lock); + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = true; + } + dev_info(dcp->dev, "peer paired modeset result=%d (scanout not submitted)\n", ret); + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc new file mode 100644 index 0000000000000..33def11c752ea --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Explicit, opt-in firmware bring-up command. This is not the final hotplug + * lifecycle: use only the dedicated, otherwise unrouted peer reservation. + */ +struct dcp_peer_probe_wait { + struct kref refs; + struct completion done; +}; + +static void dcp_peer_probe_free(struct kref *ref) +{ + kfree(container_of(ref, struct dcp_peer_probe_wait, refs)); +} + +static void dcp_peer_probe_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_probe_wait *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refs, dcp_peer_probe_free); +} + +static ssize_t peer_setup_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + static const struct dcp_method_entry method = { + .name = "setup_peer_dcp", .tag = {'A', '3', '6', '6'}, + }; + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct dcp_peer_probe_wait *wait; + __le32 enabled = cpu_to_le32(1); + bool value; + long ret; + + if (kstrtobool(buf, &value) || !value) + return -EINVAL; + if (!peer_allocation_probe || !dcp) + return -EOPNOTSUPP; + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + init_completion(&wait->done); + kref_init(&wait->refs); + { + guard(mutex)(&dcp_peer_lock); + if (!dcp->peer_owner || dcp->peer_owner->peer_reserved != dcp || + dcp->peer_unbinding || dcp->crashed || !dcp->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + dcp->fixed_phy || dcp->nr_typec_routes || + (dcp->connector && dcp->connector->connected) || + dcp->ch_cmd.depth || dcp->ch_cb.depth || dcp->ch_async.depth || + dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth || + dcp->ch_oobasync.depth) { + kref_put(&wait->refs, dcp_peer_probe_free); + return -EBUSY; + } + kref_get(&wait->refs); + /* Peer-DCP setup (one bool argument): four input + * bytes (boolean followed by zero padding), no output. + */ + dcp_push(dcp, false, &method, sizeof(enabled), 0, &enabled, + dcp_peer_probe_done, wait); + } + ret = wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000)); + kref_put(&wait->refs, dcp_peer_probe_free); + dev_info(dev, "peer A366 setup test %s\n", ret ? "acknowledged" : "timed out"); + return ret ? count : -ETIMEDOUT; +} +static DEVICE_ATTR_WO(peer_setup_test); + +#include "dcp-peer-video-probe-v27.inc" +#include "dcp-peer-swap-probe-v27.inc" +#include "dcp-peer-runtime-v27.inc" + +static struct attribute *dcp_peer_probe_attrs[] = { + &dev_attr_peer_setup_test.attr, + &dev_attr_peer_video_test.attr, + &dev_attr_peer_link_test.attr, + &dev_attr_peer_swap_test.attr, + &dev_attr_peer_stop_test.attr, + NULL, +}; + +static umode_t dcp_peer_probe_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + return peer_allocation_probe ? attr->mode : 0; +} + +static const struct attribute_group dcp_peer_probe_group = { + .attrs = dcp_peer_probe_attrs, + .is_visible = dcp_peer_probe_visible, +}; +static const struct attribute_group *dcp_peer_probe_groups[] = { + &dcp_peer_probe_group, + NULL, +}; diff --git a/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc b/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc new file mode 100644 index 0000000000000..67401e465fcca --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Paired DRM scanout of a mode one firmware-27 pipe cannot drive alone. */ +static bool peer_scanout = true; +module_param(peer_scanout, bool, 0444); +MODULE_PARM_DESC(peer_scanout, "Firmware27 paired DRM scanout (default: on)"); + +static bool dcp_peer_transport_idle(struct apple_dcp *dcp) +{ + return dcp->active && !dcp->crashed && !dcp->peer_unbinding && + !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_async.depth && + !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && !dcp->ch_oobasync.depth; +} + +int dcp_peer_modeset_v27(struct apple_dcp *dcp, u32 timing, u32 color, bool vrr) +{ + struct apple_dcp *peer; + int ret = -EOPNOTSUPP; + + if (!peer_scanout || !peer_allocation_probe || + dcp->fw_compat != DCP_FIRMWARE_V_27_0) + return ret; + if (timing > U32_MAX - 10000 || color > U32_MAX - 10000) + return -EINVAL; + if (!device_trylock(dcp->dev)) + return -EBUSY; + { + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved; + if (!peer) + goto unlock; + if (peer->peer_owner != dcp || !dcp_peer_transport_idle(dcp) || + !dcp_peer_transport_idle(peer) || peer->fixed_phy || + peer->nr_typec_routes || !peer->crtc || + !peer->crtc->base.state || peer->crtc->base.state->active || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + } + /* Do not silently select an untiled mode when paired VRR was requested. + * The internal and single-pipe external adaptive-sync paths are unchanged. + */ + if (vrr) { + struct dcp_peer_get_video_req_v27 req = { + .timing_id = cpu_to_le32(timing), .color_id = cpu_to_le32(color), + }; + struct dcp_peer_get_video_resp_v27 resp; + + ret = dcp_peer_video_call(dcp, "A364", &req, sizeof(req), &resp, sizeof(resp)); + if (ret || !(resp.success & 1) || (resp.requires_merge & 1)) { + ret = ret ?: -EINVAL; + goto peer_unlock; + } + } + ret = dcp_peer_install_video_v27(dcp, peer, timing, color, 4); + if (!ret) { + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames = (struct dcp_peer_frames_v27){ .active = true }; + dev_info(dcp->dev, "paired DRM mode ready: timing=%u color=%u\n", timing, color); + } +peer_unlock: + device_unlock(peer->dev); +unlock: + device_unlock(dcp->dev); + return ret; +} + +/* Return1 after a paired stop,0 for a normal CRTC, or an error. */ +int dcp_peer_stop_v27(struct apple_dcp *dcp) +{ + struct apple_dcp *peer; + int ret = 0; + + if (!peer_scanout || !READ_ONCE(dcp->peer_frames.active)) + return 0; + if (!device_trylock(dcp->dev)) + return -EBUSY; + { + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved; + if (!peer || peer->peer_owner != dcp || !dcp_peer_transport_idle(dcp) || + !dcp_peer_transport_idle(peer) || !device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + } + ret = dcp_peer_probe_quiesce(dcp, peer); + device_unlock(peer->dev); + if (!ret) + ret = 1; +unlock: + device_unlock(dcp->dev); + return ret; +} + +struct apple_dcp *dcp_peer_scanout_v27(struct apple_dcp *dcp) +{ + struct apple_dcp *peer; + + if (!peer_scanout || !READ_ONCE(dcp->peer_frames.active)) + return NULL; + guard(mutex)(&dcp_peer_lock); + if (!dcp->peer_frames.active) + return NULL; + peer = dcp->peer_reserved; + if (!peer || peer->peer_owner != dcp || dcp->peer_frames.failed || + !dcp->peer_tiling.modeset_complete || !peer->peer_tiling.modeset_complete || + dcp->crashed || peer->crashed || dcp->peer_unbinding || peer->peer_unbinding) + return ERR_PTR(-EIO); + return peer; +} + +void dcp_peer_swap_armed_v27(struct apple_dcp *dcp, u32 id) +{ + guard(mutex)(&dcp_peer_lock); + dcp_peer_frame_submit_v27(&dcp->peer_frames, id); +} + +void dcp_peer_swap_failed_v27(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames.failed = true; + dcp_mode_invalidate(&dcp->mode_state); +} + +bool dcp_peer_swap_complete_v27(struct apple_dcp *dcp, u32 id, ktime_t now, + void (*report)(struct apple_dcp *, u32, ktime_t)) +{ + struct apple_dcp *owner; + u32 watermark; + + /* Independent CRTCs must not serialize through the pair's mutex. */ + if (!peer_scanout || (!READ_ONCE(dcp->peer_frames.active) && + !READ_ONCE(dcp->peer_owner))) + return false; + guard(mutex)(&dcp_peer_lock); + owner = dcp->peer_owner ?: dcp; + if (!peer_scanout || !owner->peer_frames.active) + return false; + if (dcp_peer_frame_complete_v27(&owner->peer_frames, dcp != owner, id, &watermark)) + /* Queue the combined watermark to the primary IOMFB owner. */ + report(owner, watermark, now); + return true; +} + +int dcp_peer_color_v27(struct apple_dcp *dcp, struct iomfb_set_matrix_req *matrix) +{ + struct apple_dcp *peer = dcp_peer_scanout_v27(dcp); + u32 status, identity = 0x10000; + int ret; + + if (!peer) + return 0; + if (IS_ERR(peer)) + return PTR_ERR(peer); + ret = dcp_peer_video_call(peer, "A423", matrix, sizeof(*matrix), &status, 4); + if (ret || status) + return ret ?: -EIO; + ret = dcp_peer_video_call(peer, "A428", &identity, sizeof(identity), &status, 4); + return ret ?: (status ? -EIO : 0); +} diff --git a/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc new file mode 100644 index 0000000000000..f6c8ced4317ed --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Explicit background-only paired swap diagnostic. No framebuffer DMA or + * DRM events are owned here. Both DRM CRTCs must remain inactive. + */ +struct dcp_peer_background { + struct iomfb_transaction transaction; + struct dcp_peer_video_wait *wait; + struct dcp_swap_submit_req_v27 wire; + bool sent; +}; + +static void dcp_peer_background_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_background *background = cookie; + struct dcp_swap_submit_resp_v27 failed = { .ret = ~0U }; + struct dcp_swap_submit_resp_v27 *response = out ?: &failed; + + if (!dcp_present_submit_h17p(&dcp->present_state_h17p, + background->wire.swap.swap_id, !response->ret)) + WRITE_ONCE(dcp->crashed, true); + dcp_peer_video_done(dcp, response, background->wait); +} + +static void dcp_peer_background_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct dcp_peer_background *background = + container_of(transaction, struct dcp_peer_background, transaction); + struct dcp_method_entry method = { .name = "peer_background_test", + .tag = {'A', '4', '0', '8'} }; + struct dcp_swap_submit_resp_v27 failed = { .ret = ~0U }; + + background->sent = true; + if (!dcp_present_begin_h17p(&dcp->present_state_h17p, + background->wire.swap.swap_id)) { + dcp_peer_video_done(dcp, &failed, background->wait); + return; + } + dcp_push(dcp, false, &method, sizeof(background->wire), sizeof(failed), + &background->wire, dcp_peer_background_done, background); +} + +static void dcp_peer_background_release(struct iomfb_transaction *transaction) +{ + struct dcp_peer_background *background = + container_of(transaction, struct dcp_peer_background, transaction); + + if (!background->sent) + kref_put(&background->wait->refs, dcp_peer_video_free); + kfree(background); +} + +static int dcp_peer_queue_background(struct apple_dcp *dcp, + struct dcp_swap_submit_req_v27 *wire, + struct dcp_peer_video_wait *wait) +{ + struct dcp_peer_background *background; + int ret; + + background = kzalloc_obj(*background); + if (!background) + return -ENOMEM; + background->wire = *wire; + background->wait = wait; + kref_get(&wait->refs); + background->transaction.start = dcp_peer_background_start; + background->transaction.release = dcp_peer_background_release; + ret = iomfb_queue(dcp, &background->transaction); + if (ret) + dcp_peer_background_release(&background->transaction); + return ret; +} + +static int dcp_peer_probe_background(struct apple_dcp *dcp, struct apple_dcp *peer) +{ + struct dcp_swap_start_req_h17p start = {}; + struct dcp_swap_start_resp_h17p a, b; + struct dcp_swap_submit_req_v27 *submit; + struct dcp_swap_submit_resp_v27 response; + struct dcp_peer_video_wait *wait, *peer_wait; + unsigned long deadline; + int ret; + + submit = kzalloc(sizeof(*submit), GFP_KERNEL); + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + peer_wait = kzalloc(sizeof(*peer_wait), GFP_KERNEL); + if (!submit || !wait || !peer_wait) { + kfree(submit); + kfree(wait); + kfree(peer_wait); + return -ENOMEM; + } + kref_init(&wait->refs); + init_completion(&wait->done); + wait->size = sizeof(response); + kref_init(&peer_wait->refs); + init_completion(&peer_wait->done); + peer_wait->size = sizeof(response); + ret = dcp_peer_video_call(dcp, "A406", &start, sizeof(start), &a, sizeof(a)); + if (ret || a.ret) { + ret = ret ?: -EIO; + goto out; + } + ret = dcp_peer_video_call(peer, "A406", &start, sizeof(start), &b, sizeof(b)); + if (ret || b.ret || a.swap_id != b.swap_id) { + dev_err(dcp->dev, "peer swap-start mismatch/error: %u/%u result=%d\n", + a.swap_id, ret ? 0 : b.swap_id, ret); + ret = ret ?: -EIO; + goto out; + } + submit->swap.swap_id = a.swap_id; + submit->swap.swap_enabled = submit->swap.swap_completed = IOMFB_SET_BACKGROUND | 7; + submit->swap.bg_color = 0xffff0000; + memset(submit->surf_null, 1, sizeof(submit->surf_null)); + submit->unknown_pointer_null = submit->unknown_output_null = true; + ret = dcp_peer_queue_background(dcp, submit, wait); + if (ret) + goto out; + /* Queue both engines before waiting, with each present tracked by its + * own IOMFB transaction and completion state. */ + submit->swap.bg_color = 0xff0000ff; + ret = dcp_peer_queue_background(peer, submit, peer_wait); + if (ret) + goto out; + if (!wait_for_completion_timeout(&peer_wait->done, msecs_to_jiffies(2000))) { + ret = -ETIMEDOUT; + goto out; + } + memcpy(&response, peer_wait->reply, sizeof(response)); + if (response.ret) { + ret = -EIO; + goto out; + } + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000))) { + ret = -ETIMEDOUT; + goto out; + } + memcpy(&response, wait->reply, sizeof(response)); + if (response.ret) { + ret = -EIO; + goto out; + } + deadline = jiffies + msecs_to_jiffies(2000); + while (!READ_ONCE(dcp->have_swap_complete) || !READ_ONCE(peer->have_swap_complete) || + READ_ONCE(dcp->last_swap_id) != a.swap_id || + READ_ONCE(peer->last_swap_id) != a.swap_id) { + if (time_after_eq(jiffies, deadline)) { + ret = -ETIMEDOUT; + goto out; + } + msleep(2); + } + dev_info(dcp->dev, "peer background swap %u completed on both engines (optical unverified)\n", + a.swap_id); +out: + dev_info(dcp->dev, "peer background test result=%d\n", ret); + kref_put(&wait->refs, dcp_peer_video_free); + kref_put(&peer_wait->refs, dcp_peer_video_free); + kfree(submit); + return ret; +} + +static ssize_t peer_swap_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled) || !enabled) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || READ_ONCE(dcp->peer_frames.active) || !dcp->peer_tiling.member || + !peer->peer_tiling.member || !dcp->peer_tiling.modeset_complete || + !peer->peer_tiling.modeset_complete || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + dcp->crtc->base.state->active || peer->crtc->base.state->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto out; + } + peer_dev = peer->dev; + } + ret = dcp_peer_probe_background(dcp, peer); +out: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_swap_test); + +/* End a diagnostic pair using the firmware's power/element sequence. Keep the + * reservation on failure: a failed ACK is not evidence that DMA has stopped. + */ +static ssize_t peer_stop_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled) || !enabled) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + dcp->crtc->base.state->active || peer->crtc->base.state->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto out; + } + peer_dev = peer->dev; + } + ret = dcp_peer_probe_quiesce(dcp, peer); + dev_info(dev, "peer stop test result=%d\n", ret); +out: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_stop_test); diff --git a/drivers/gpu/drm/apple/dcp-peer-v27.h b/drivers/gpu/drm/apple/dcp-peer-v27.h new file mode 100644 index 0000000000000..cea2a4f13e43b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v27.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_V27_H +#define APPLE_DCP_PEER_V27_H + +#include +#include +#include + +#include "dcp-peer-v26_6.h" + +struct dcp_dptx_caps { + u32 width, total_rate, active_rate; +}; + +/* Per-engine state used by D114/D115 in a VFTG pair. This is not + * display-wall emulation: membership is set only while a split mode is active. + */ +/* The record is struct dcp_peer_tiling, shared with the desktop 26.6 pair. */ + +static inline bool dcp_peer_tiling_get_v27( + const struct dcp_peer_tiling *own, + const struct dcp_peer_tiling *related, u32 event, u32 param, u32 *value) +{ + const struct dcp_peer_tiling *target = own; + + /* The getter redirects event 2 only for an active pair. */ + if (event == 2 && own->member) { + if (!related) + return false; + target = related; + } + switch (param) { + case 1: *value = target->member; return true; + case 2: + if (event == 1 || target->member) + *value = target->phase; + return true; + case 3: return true; /* The non-wall case preserves the input. */ + case 4: + if (event != 1 && !target->member) + return true; + if (!target->display_id_valid) + return false; + *value = target->display_id; + return true; + case 5: case 12: *value = target->x; return true; + case 6: + if (event == 1 || target->member) + *value = target->ready; + return true; + case 8: *value = own->vftg; return true; + case 9: *value = own->primary; return true; + case 10: + if (event == 1 || target->member) + *value = target->modeset_state; + return true; + case 11: *value = 0; return true; /* This pair is not a display wall. */ + case 13: *value = target->y; return true; + case 14: *value = target->mode_flag; return true; + default: return false; + } +} + +static inline bool dcp_peer_tiling_set_v27(struct dcp_peer_tiling *own, + struct dcp_peer_tiling *related, u32 event, u32 param, u32 value) +{ + /* Unlike the getter, the setter always redirects event 2. */ + struct dcp_peer_tiling *target = event == 2 ? related : own; + + if (!target) + return false; + switch (param) { + case 2: target->phase = value; return true; + case 6: target->ready = !!value; return true; + case 10: target->modeset_state = value; return true; + default: return false; + } +} + +/* AV display hints and display allocation records. */ +struct dcp_display_hints_v27 { + u8 hints[0x6c]; + u8 padding[4]; + __le32 primary_id, peer_id; + u8 extra_pipes, flags; + u8 allocation_reserved[22]; + __le32 status; + u8 tail[12]; +} __packed; + +static_assert(sizeof(struct dcp_display_hints_v27) == 160); +static_assert(offsetof(struct dcp_display_hints_v27, primary_id) == 0x70); +static_assert(offsetof(struct dcp_display_hints_v27, status) == 0x90); + +/* Extra pipe count, as the T602X display crossbar computes it. + * Caps come from the live DPTX service, never from a forced clock limit. + */ +static inline u32 dcp_peer_extra_pipes(const u8 *hints, + const struct dcp_dptx_caps *caps) +{ + u32 width = get_unaligned_le32(hints + 0x38); + u32 total = get_unaligned_le32(hints + 0x40); + u32 active = get_unaligned_le32(hints + 0x44); + u32 extra; + + if (!(hints[0] & 1) || !width || !total || !active || + !caps->width || !caps->total_rate || !caps->active_rate) + return 0; + extra = (width - 1) / caps->width; + extra = max(extra, (total - 1) / caps->total_rate); + return max(extra, (active - 1) / caps->active_rate); +} + +/* Peer validity rule of the T8132 display crossbar, inherited by T6050. + * Endpoint zero is the internal display; a peer must be on the same die. + */ +static inline bool dcp_peer_index_allowed(u32 primary, u32 peer) +{ + static const u8 masks[] = { 0, 0x1c, 0x18, 0x10, 0 }; + + return primary < ARRAY_SIZE(masks) && peer < ARRAY_SIZE(masks) && + (masks[primary] & BIT(peer)); +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-v27.inc b/drivers/gpu/drm/apple/dcp-peer-v27.inc new file mode 100644 index 0000000000000..604e0935c5846 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v27.inc @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +/* Logical peer allocation for a merged mode. This only reserves an otherwise + * unrouted DCP; it never replaces a physical route. + */ +static bool peer_allocation_probe = true; +module_param(peer_allocation_probe, bool, 0444); +MODULE_PARM_DESC(peer_allocation_probe, "Firmware27 logical peer allocation for merged modes (default: on)"); + +bool dcp_peer_tiling_state_v27(struct apple_dcp *dcp, bool write, + u32 event, u32 param, u32 *value) +{ + struct apple_dcp *peer; + + guard(mutex)(&dcp_peer_lock); + if (!peer_allocation_probe || dcp->peer_unbinding || dcp->crashed) + return false; + peer = dcp->peer_reserved ?: dcp->peer_owner; + if (!peer || peer->peer_unbinding || peer->crashed) + return false; + /* A resource reservation is not a framebuffer pair. Before the + * split-mode preparation, keep the established untiled callback path; + * zero-initialized fields are not yet valid tiling-manager state. + */ + if (!dcp->peer_tiling.vftg || !peer->peer_tiling.vftg) + return false; + if (write) + return dcp_peer_tiling_set_v27(&dcp->peer_tiling, + &peer->peer_tiling, event, param, *value); + return dcp_peer_tiling_get_v27(&dcp->peer_tiling, &peer->peer_tiling, + event, param, value); +} + +void dcp_peer_display_ids_v27(struct apple_dcp *dcp, u32 own, u32 related) +{ + struct apple_dcp *peer; + + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved ?: dcp->peer_owner; + if (!peer || own == related || dcp->peer_unbinding || peer->peer_unbinding) + return; + /* D575 carries external display IDs, not crossbar endpoint indices. */ + dcp->peer_tiling.display_id = own; + dcp->peer_tiling.display_id_valid = true; + peer->peer_tiling.display_id = related; + peer->peer_tiling.display_id_valid = true; +} + +static void dcp_peer_release_locked(struct apple_dcp *dcp) +{ + if (dcp->peer_reserved) { + /* A link-allocation release is not a DMA-stop acknowledgement. + * Keep a prepared pair owned until the AP has powered both down. + * Firmware may still need the tiling rendezvous state to stop. + */ + if (dcp->peer_tiling.vftg || dcp->peer_tiling.member || + dcp->peer_reserved->peer_tiling.vftg || + dcp->peer_reserved->peer_tiling.member) { + dcp->peer_tiling.modeset_complete = false; + dcp->peer_reserved->peer_tiling.modeset_complete = false; + return; + } + memset(&dcp->peer_tiling, 0, sizeof(dcp->peer_tiling)); + memset(&dcp->peer_reserved->peer_tiling, 0, + sizeof(dcp->peer_reserved->peer_tiling)); + dcp->peer_reserved->peer_owner = NULL; + dcp->peer_reserved = NULL; + } +} + +int dcp_peer_hints_v27(struct apple_dcp *dcp, struct dptx_port *port, + const void *data, size_t size, void *reply, size_t reply_size) +{ + struct dcp_display_hints_v27 *r = reply; + struct apple_dcp *peer = NULL; + struct drm_crtc *crtc; + u32 extra; + + if (size != sizeof(*r) || reply_size != sizeof(*r)) + return -EINVAL; + memcpy(reply, data, sizeof(*r)); + /* The first word is hint flags, not a return code. The completion + * status has a field of its own, leaving the request hints unchanged. + */ + r->status = 0; + if (!peer_allocation_probe) + return 0; + if (!of_device_is_compatible(dcp->dev->of_node, "apple,t6050-dcpext") || + port->unit || !dcp->crtc) + return 0; + + extra = dcp_peer_extra_pipes(r->hints, &port->caps); + memset(&r->primary_id, 0, 32); + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_unbinding || dcp->crashed) + return -ENODEV; + if (!extra || !(r->hints[0] & 1)) { + dcp_peer_release_locked(dcp); + memset(&r->primary_id, 0, 32); + return 0; + } + /* This candidate implements a non-tiled, two-pipe reservation. Tiled + * panels and more than one peer require the full display allocator. + */ + if (extra != 1 || (r->hints[0] & 2) || dcp->peer_owner) + return 0; + peer = dcp->peer_reserved; + if (!peer) { + drm_for_each_crtc(crtc, dcp->crtc->base.dev) { + struct apple_dcp *p = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + /* A dedicated, unrouted peer cannot race Type-C's allocator + * or a fixed connector's HPD handler in this bring-up step. + */ + if (!p || !p->active || p->crashed || p->peer_unbinding || + p->peer_owner || p->peer_reserved || p->fixed_phy || + p->nr_typec_routes || p->active_typec_route || + (p->connector && p->connector->connected) || + !dcp_peer_index_allowed(dcp->index, p->index) || + p->dptx_die != dcp->dptx_die || + !p->crtc || p->crtc->base.dev != dcp->crtc->base.dev || + p->fw_compat != DCP_FIRMWARE_V_27_0 || + !p->dptxport[0].enabled || + p->dptxport[0].caps.width < port->caps.width || + p->dptxport[0].caps.total_rate < port->caps.total_rate || + p->dptxport[0].caps.active_rate < port->caps.active_rate || + p->dptxport[0].connected || p->dptxport[1].connected || + !of_device_is_compatible(p->dev->of_node, "apple,t6050-dcpext")) + continue; + if (!peer || p->index < peer->index) + peer = p; + } + if (!peer) { + dev_info(dcp->dev, "no eligible idle peer for %u extra pipe\n", extra); + return 0; + } + dcp->peer_reserved = peer; + peer->peer_owner = dcp; + dev_info(dcp->dev, "reserved peer DCP%u for %ux%u, total %u active %u\n", + peer->index, get_unaligned_le32(r->hints + 0x38), + get_unaligned_le32(r->hints + 0x3c), + get_unaligned_le32(r->hints + 0x40), + get_unaligned_le32(r->hints + 0x44)); + } + /* The display allocation update writes primary/peer endpoint IDs. + * The extra pipe count supplies byte8. This is full dual-pipe, so the + * T6050 limited-dual-pipe flag (byte9 bit1) remains clear. + */ + r->primary_id = cpu_to_le32(dcp->index); + r->peer_id = cpu_to_le32(peer->index); + r->extra_pipes = extra; + r->flags &= ~3; + return 0; +} diff --git a/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc new file mode 100644 index 0000000000000..e45561945803e --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Opt-in A364/A365 bring-up. Installs derived modes without selecting them; + * paired power, modeset and swap ownership must be implemented separately. + */ +#include "iomfb-peer-v27.h" + +struct dcp_peer_video_wait { + struct kref refs; + struct completion done; + size_t size; + u8 reply[sizeof(struct dcp_peer_get_video_resp_v27)]; +}; + +static void dcp_peer_video_free(struct kref *ref) +{ + kfree(container_of(ref, struct dcp_peer_video_wait, refs)); +} + +static void dcp_peer_video_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_video_wait *wait = cookie; + + if (wait->size) + memcpy(wait->reply, out, wait->size); + complete(&wait->done); + kref_put(&wait->refs, dcp_peer_video_free); +} + +static int dcp_peer_video_call_stream(struct apple_dcp *dcp, bool independent, + const char *tag, + void *in, size_t in_size, void *out, size_t out_size) +{ + struct dcp_method_entry method = { .name = "peer_video_test" }; + struct dcp_peer_video_wait *wait; + int ret = 0; + + if (out_size > sizeof(wait->reply)) + return -EINVAL; + /* An unrelated AP thread gets its own firmware stream. A369/A367/etc. + * must not be mistaken for nested callbacks of the pending A413. + * Stream1 (the driver's OOB context) is independent of stream0. + */ + if (independent && (dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth)) + return -EBUSY; + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + memcpy(method.tag, tag, sizeof(method.tag)); + wait->size = out_size; + init_completion(&wait->done); + kref_init(&wait->refs); + kref_get(&wait->refs); + if (independent) { + /* Not behind the pending primary A413: it waits for this handoff. */ + ret = dcp_push_oob_concurrent(dcp, &method, in_size, out_size, in, + dcp_peer_video_done, wait); + if (ret) { + kref_put(&wait->refs, dcp_peer_video_free); + kref_put(&wait->refs, dcp_peer_video_free); + return ret; + } + } else { + dcp_push(dcp, false, &method, in_size, out_size, in, + dcp_peer_video_done, wait); + } + if (!wait_for_completion_timeout(&wait->done, + msecs_to_jiffies(!memcmp(tag, "A413", 4) ? 8500 : 2000))) + ret = -ETIMEDOUT; + else if (out_size) + memcpy(out, wait->reply, out_size); + /* A late ACK owns only the heap cookie, never a caller's stack. */ + kref_put(&wait->refs, dcp_peer_video_free); + return ret; +} + +static int dcp_peer_video_call(struct apple_dcp *dcp, const char *tag, + void *in, size_t in_size, void *out, size_t out_size) +{ + return dcp_peer_video_call_stream(dcp, false, tag, in, in_size, out, out_size); +} + +#include "dcp-peer-modeset-probe-v27.inc" + +static bool dcp_peer_video_idle(struct apple_dcp *dcp) +{ + return dcp->active && !dcp->crashed && !dcp->peer_unbinding && + !READ_ONCE(dcp->mode_state.changing) && !dcp->ch_cmd.depth && !dcp->ch_cb.depth && + !dcp->ch_async.depth && !dcp->ch_oobcmd.depth && + !dcp->ch_oobcb.depth && !dcp->ch_oobasync.depth; +} + +/* Shared by the explicit diagnostic and normal DRM mode selection. + * Install4 chooses a pair only when firmware's A364 requires one. + */ +static int dcp_peer_install_video_v27(struct apple_dcp *dcp, struct apple_dcp *peer, + u32 timing, u32 color, u32 install) +{ + struct device *dev = dcp->dev; + struct dcp_peer_get_video_req_v27 req = {}; + struct dcp_peer_get_video_resp_v27 resp; + struct dcp_peer_insert_video_req_v27 insert = {}; + struct dcp_peer_insert_video_resp_v27 inserted; + __le32 enabled = cpu_to_le32(1); + int ret; + + req.timing_id = cpu_to_le32(timing); + req.color_id = cpu_to_le32(color); + ret = dcp_peer_video_call(dcp, "A364", &req, sizeof(req), &resp, sizeof(resp)); + if (ret) + return ret; + dev_info(dev, "peer video timing=%u color=%u success=%u merge=%u timing=%*ph color=%*ph\n", + timing, color, resp.success, resp.requires_merge, + 64, resp.video.timing, 32, resp.video.color); + dev_info(dev, "peer timing tail=%*ph\n", 16, resp.video.timing + 64); + if (!(resp.success & 1)) { + ret = -ENOENT; + return ret; + } + if (!install) + return ret; + /* Mode3 explicitly tests a reserved pair using an existing lower-rate + * timing. It does not advertise a mode, raise a limit, or add scanout. + * Keep this diagnostic independent of production mode selection. + */ + if (!(resp.requires_merge & 1) && install != 3) { + if (install == 4 && (dcp->peer_tiling.vftg || peer->peer_tiling.vftg)) { + u8 power[16] = {1}, response[8]; + + ret = dcp_peer_probe_quiesce(dcp, peer); + if (ret) + return ret; + /* A mode-only DRM commit does not call CRTC poweron again. */ + ret = dcp_peer_video_call(dcp, "A484", power, sizeof(power), + response, sizeof(response)); + if (ret || get_unaligned_le32(response + 4)) + return ret ?: -EIO; + } + return -EOPNOTSUPP; + } + ret = dcp_peer_probe_quiesce(dcp, peer); + if (ret) + return ret; + insert.video = resp.video; + dcp_peer_split_timing_v27(&insert.video); + insert.timing_id = cpu_to_le32(timing + 10000); + insert.color_id = cpu_to_le32(color + 10000); + ret = dcp_peer_probe_prepare_vftg(dcp, peer, &insert); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A366", &enabled, sizeof(enabled), NULL, 0); + if (ret) + return ret; + ret = dcp_peer_video_call(dcp, "A365", &insert, sizeof(insert), + &inserted, sizeof(inserted)); + if (!ret && !(inserted.success & 1)) + ret = -EIO; + if (ret) + return ret; + /* The first call's in/out timing and color go to the peer. */ + insert.video = inserted.video; + ret = dcp_peer_video_call(peer, "A365", &insert, sizeof(insert), + &inserted, sizeof(inserted)); + if (!ret && !(inserted.success & 1)) + ret = -EIO; + dev_info(dev, "peer derived mode timing=%u color=%u width=%u height=%u clock=%u result=%d (not selected)\n", + timing + 10000, color + 10000, + get_unaligned_le32(insert.video.timing + 8), + get_unaligned_le32(insert.video.timing + 0x28), + get_unaligned_le32(insert.video.timing + 0x44), ret); + if (!ret && install >= 2) + ret = dcp_peer_probe_modeset(dcp, peer, &insert, + resp.requires_merge & 1); + return ret; +} + +static ssize_t peer_video_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + u32 timing, color, install; + char extra; + int ret; + + if (sscanf(buf, "%u %u %u %c", &timing, &color, &install, &extra) != 3 || + install > 3 || timing > U32_MAX - 10000 || color > U32_MAX - 10000) + return -EINVAL; + if (!peer_allocation_probe) + return -EOPNOTSUPP; + /* Trylocks avoid the sysfs-removal/device-lock inversion. Hold both + * devices across RPC waits so neither devm-backed DCP can be unbound. + */ + if (!device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + (install && (dcp->crtc->base.state->active || + peer->crtc->base.state->active)) || peer->fixed_phy || + peer->nr_typec_routes || peer->peer_tiling.member || + dcp->peer_tiling.member || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + (peer->connector && peer->connector->connected)) { + ret = -EBUSY; + goto unlock; + } + if (!device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + peer_dev = peer->dev; + } + ret = dcp_peer_install_video_v27(dcp, peer, timing, color, install); +unlock: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_video_test); + +static ssize_t peer_link_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled)) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + if (!dcp || !dcp->active || dcp->crashed || !dcp->fixed_phy || + dcp->nr_typec_routes || dcp->connector_type != DRM_MODE_CONNECTOR_HDMIA) { + ret = -EOPNOTSUPP; + goto unlock; + } + { + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_tiling.vftg || dcp->peer_tiling.member) { + ret = -EBUSY; + goto unlock; + } + } + /* Use the existing disconnect/connect lifecycle, serialized with HPD. + * This diagnostic models link re-enumeration, not a physical cable pull. + */ + ret = dcp_typec_test_link(to_platform_device(dev), enabled); +unlock: + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_link_test); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f233c079997cc..1c288d4ad91b8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -68,6 +69,7 @@ #include "iomfb_internal.h" #include "iomfb_m4.h" #include "ibootep.h" +#include "iomfb_internal.h" #include "parser.h" #include "trace.h" @@ -117,9 +119,12 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) return -ENOMEM; /* Keep the DMA device, domain usage and driver code valid until reset. */ + if (!try_module_get(THIS_MODULE)) { + kfree(session); + return -ENODEV; + } session->dev = get_device(dcp->dev); session->group = iommu_group_get(dcp->dev); - __module_get(THIS_MODULE); WRITE_ONCE(dcp->retain_dma, true); mutex_lock(&dcp_retained_sessions_lock); list_add_tail(&session->link, &dcp_retained_sessions); @@ -235,6 +240,9 @@ bool dcp_t6041_ext_enabled(void) #include "iomfb_v26_6_swap.h" #include "dcp-peer-v26_6.inc" +/* The firmware 27 pair shares dcp_peer_lock and dcp_peer_forget() above. */ +#include "dcp-peer-v27.inc" +#include "dcp-peer-probe-v27.inc" /* * The backlight maximum of a desktop 26.x panel whose node has no @@ -555,9 +563,10 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) } /* - * A desktop 26.x external processor does not hand out the sink's EDID through - * its DCPAV service, but exports the TimingElements and DisplayAttributes - * of every sink it drives, on HDMI and Type-C alike. + * A desktop 26.x external processor, and T6050's on firmware 27, does not + * hand out the sink's EDID through its DCPAV service, but exports the + * TimingElements and DisplayAttributes of every sink it drives, on HDMI and + * Type-C alike. */ static bool dcp_sink_metadata(struct apple_dcp *dcp) { @@ -1338,6 +1347,7 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ return; } dcp->crashed = true; + dcp_peer_forget(dcp); dev_err(dcp->dev, "DCP has crashed\n"); if (dcp_fail_stop(dcp) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { @@ -1400,6 +1410,13 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) phy_addr = iommu_iova_to_phys(domain, bfr->iova); if (!phy_addr) return -ENOMEM; + /* Firmware 27's DVAs resolve through its own retained mappings. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + !dcp_fw_phys_in_dram(phy_addr)) { + dev_err(dcp->dev, "shmem_setup: iova %pad is not in DRAM\n", + &bfr->iova); + return -EACCES; + } /* memremap() below needs the whole buffer physically contiguous */ ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, @@ -3819,6 +3836,7 @@ int dcp_start(struct platform_device *pdev) return iomfb_v26_6_start(dcp); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) return dcp_start_desktop(dcp); + init_completion(&dcp->typec_display_ready); /* * The T6030 firmware session is adopted for the internal panel. @@ -3860,7 +3878,9 @@ int dcp_start(struct platform_device *pdev) ret); } - if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + if ((dcp->phy || (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp->hw.num_dptx_ports)) && + dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { ret = ibootep_init(dcp); if (ret) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", @@ -4001,6 +4021,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_poweroff_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweroff_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -4098,6 +4121,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_sleep_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_sleep_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -4123,7 +4149,7 @@ static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) /* The 14.7 panel has its own known-level blank and restore hooks. */ return (!iomfb_v14_7_panel_nits(dcp) && dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || - (dcp->fw_compat == DCP_FIRMWARE_H17P && + ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && dcp->connector_type == DRM_MODE_CONNECTOR_eDP); } @@ -4187,10 +4213,13 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { - iomfb_poweron_h17p(dcp); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_poweron_v27(dcp); + else + iomfb_poweron_h17p(dcp); if (!READ_ONCE(dcp->pipe_enabled_h17p)) { dev_err(dcp->dev, "initial display pipe enable failed\n"); WRITE_ONCE(dcp->crashed, true); @@ -4288,6 +4317,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_H17P: iomfb_poweron_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweron_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -4827,6 +4859,23 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } dcp->nr_disp_registers = count; + for (i = 0; i < count; i++) + dcp->disp_firmware_base[i] = dcp->disp_registers[i]->start; + if (of_property_present(dcp->dev->of_node, "apple,firmware-physical-regs")) { + if (of_property_count_u64_elems(dcp->dev->of_node, + "apple,firmware-physical-regs") != count) + return -EINVAL; + ret = of_property_read_u64_array(dcp->dev->of_node, + "apple,firmware-physical-regs", + dcp->disp_firmware_base, count); + if (ret) + return ret; + for (i = 0; i < count; i++) + if (!IS_ALIGNED(dcp->disp_firmware_base[i], 4) || + resource_size(dcp->disp_registers[i]) > + U64_MAX - dcp->disp_firmware_base[i]) + return -EINVAL; + } return 0; } @@ -4882,6 +4931,13 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + if (hw->firmware_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0 || strcmp(compat_str, "27.0.0")) + return DCP_FIRMWARE_UNKNOWN; + return DCP_FIRMWARE_V_27_0; + } + /* * SoCs introduced with H17-generation firmware pin its interface; the * loader may not recognise their boot firmware version. @@ -5048,6 +5104,12 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release disable_delayed_work_sync(&dcp->hdmi_settle_wq); disable_delayed_work_sync(&dcp->hdmi_recheck_wq); disable_delayed_work_sync(&dcp->link_recovery_wq); + disable_work_sync(&dcp->tb_event_work); + scoped_guard(mutex, &dcp->tb_event_lock) { + dcp->tb_events = 0; + dcp->tb_event_route = NULL; + dcp->tb_event_generation = 0; + } disable_delayed_work_sync(&dcp->hybrid_wq); dcp_hdmi_disable(dcp); } @@ -5063,6 +5125,7 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) enable_delayed_work(&dcp->hdmi_settle_wq); enable_delayed_work(&dcp->hdmi_recheck_wq); enable_delayed_work(&dcp->link_recovery_wq); + enable_work(&dcp->tb_event_work); enable_delayed_work(&dcp->hybrid_wq); dcp_hdmi_enable(dcp); scoped_guard(mutex, &dcp->hpd_mutex) @@ -5084,6 +5147,56 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) } } +static int dcp_firmware_metadata_v27(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct device_node *fb; + struct resource resource; + int clocks, index, ret; + + /* + * The framebuffer region is the one the boot loader scanned out of; it + * backs the firmware's default framebuffer. An external processor the + * boot loader drove no display from (T6050's dcpext4, the HDMI port's) + * has none: it binds without one, and its default-framebuffer requests + * are refused (dfb_region_size 0). + */ + index = of_property_match_string(dev->of_node, "memory-region-names", + "framebuffer"); + if (index == -ENODATA || index == -EINVAL) { + dcp->dfb_region_size = 0; + dev_info(dev, "no boot framebuffer\n"); + goto clocks; + } + if (index < 0) + return index; + fb = of_parse_phandle(dev->of_node, "memory-region", index); + if (!fb) + return -EINVAL; + ret = of_address_to_resource(fb, 0, &resource); + of_node_put(fb); + if (ret) + return ret; + dcp->dfb_region_size = resource_size(&resource); + if (!dcp->dfb_region_size) + return -EINVAL; + +clocks: + clocks = of_property_count_u32_elems(dev->of_node, "apple,clock-ids"); + if (clocks == -ENODATA && + of_property_present(dev->of_node, "apple,clock-ids")) + clocks = 0; + if (clocks < 0 || clocks > 64) + return -EINVAL; + ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); + if (ret < 0) + return ret; + if (ret != clocks) + return -EINVAL; + dcp->num_firmware_clocks = clocks; + return 0; +} + static bool dcp_external_is_idle(struct apple_dcp *dcp) { u32 control = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -5180,6 +5293,14 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat u32 cpu_ctrl; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + INIT_LIST_HEAD(&dcp->iomfb_boolean_properties); + ret = devm_add_action_or_reset(dev, + iomfb_firmware_state_cleanup_v27, dcp); + if (ret) + return ret; + } + if (READ_ONCE(dcp->quiescing)) return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); @@ -5212,7 +5333,8 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat return dev_err_probe(dev, ret, "Previous scanout has not been stopped\n"); - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + ret = dma_set_mask_and_coherent(dev, + DMA_BIT_MASK(dcp->hw.firmware_dma_bits ?: 42)); if (ret) return ret; @@ -5349,12 +5471,18 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_firmware_metadata_v27(dcp); + if (ret) + goto err_piodma; + } /* * T6041: the device tree carries no DCP clock, and the boot loader * left the coprocessor clocked; D408 then reports 0 Hz, as the * driver this instance was ported from did. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP || + (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && !dcp->num_firmware_clocks)) dcp->clk = devm_clk_get_optional(dev, NULL); else dcp->clk = devm_clk_get(dev, NULL); @@ -5456,8 +5584,24 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when); /* - * The T8122 dcpext boots its firmware here. If that fails, it must not take - * the internal panel's DRM device down with it: keep it as an unusable pipe. + * The external processors whose failure must not take the DRM device down: + * T8122's, T6041's and T6050's dcpext. One that fails to probe or bind joins + * as an unusable pipe (ext_dead), and the other displays go on without it. + * With every processor failed the DRM device has no pipe and does not bind. + */ +static bool dcp_ext_is_failsafe(struct device_node *np) +{ + return of_device_is_compatible(np, "apple,t8122-dcpext") || + of_device_is_compatible(np, "apple,t6041-dcpext") || + of_device_is_compatible(np, "apple,t6050-dcpext"); +} + +/* + * A failsafe dcpext (dcp_ext_is_failsafe()) boots its firmware here. If that + * fails, it must not take the other displays' DRM device down with it: keep + * it as an unusable pipe. Its routes have registered by now and cannot be + * taken back, so a deferral here would be final as well: probe waits for the + * suppliers bind looks up (dcp_ext_bind_suppliers()) so that none comes here. */ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { @@ -5485,6 +5629,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v if (!dcp) return; + dcp_peer_forget(dcp); /* * Stop the native initializer before looking up its owner, including * the dead-session path. Owner allocation may have landed in an open @@ -5891,8 +6036,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->desktop->client = GENMASK_ULL(63, 48) | (get_random_u64() & GENMASK_ULL(47, 4)); } - dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext") || - of_device_is_compatible(dev->of_node, "apple,t6041-dcpext"); + dcp->ext_failsafe = dcp_ext_is_failsafe(dev->of_node); dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); dcp->desktop_external = of_device_is_compatible(dev->of_node, "apple,t6041-dcpext"); dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); @@ -5952,6 +6096,8 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->link_recovery_wq, dcp_link_recovery_work); + mutex_init(&dcp->tb_event_lock); + INIT_WORK(&dcp->tb_event_work, dcp_tb_event_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_relink_wq, dcp_typec_relink_work); @@ -5970,6 +6116,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) disable_delayed_work(&dcp->typec_relink_wq); disable_delayed_work(&dcp->announce_wq); disable_delayed_work(&dcp->link_recovery_wq); + disable_work(&dcp->tb_event_work); dcp->announce.stopped = true; /* * A native external pipe's bind enables these; firmware callbacks may @@ -6277,7 +6424,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when) { WRITE_ONCE(dcp->ext_dead, true); - dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the internal display goes on without it\n", + dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the other displays go on without it\n", when, err); dcp_typec_routes_disable(dcp); } @@ -6320,6 +6467,46 @@ static int dcp_ext_route_suppliers(struct device *dev) return 0; } +/* -EPROBE_DEFER while the device of available node @np has not bound. */ +static int dcp_ext_supplier_bound(struct device_node *np) +{ + struct platform_device *pdev; + int ret = 0; + + if (!np || !of_device_is_available(np)) + return 0; + pdev = of_find_device_by_node(np); + if (!pdev || !device_is_bound(&pdev->dev)) + ret = -EPROBE_DEFER; + if (pdev) + put_device(&pdev->dev); + return ret; +} + +/* + * Whether the suppliers that component bind looks up, and that probe does not + * otherwise wait for, are there: the PIODMA child's DART (a child node's + * supplier does not hold up the parent's probe) and the mailbox. Bind runs + * after the Type-C routes registered, where a deferral can only leave the + * dcpext unusable for the boot (dcp_comp_bind()). Returns -EPROBE_DEFER until + * they have bound; a missing one is left to bind, which reports it. + */ +static int dcp_ext_bind_suppliers(struct device *dev) +{ + struct device_node *piodma __free(device_node) = + of_get_child_by_name(dev->of_node, "piodma"); + struct device_node *dart __free(device_node) = + piodma ? of_parse_phandle(piodma, "iommus", 0) : NULL; + struct device_node *mbox __free(device_node) = + of_parse_phandle(dev->of_node, "mboxes", 0); + int ret; + + ret = dcp_ext_supplier_bound(dart); + if (!ret) + ret = dcp_ext_supplier_bound(mbox); + return ret; +} + /* * A T8132 dcpext's Type-C routes are only enabled for one the boot * loader handed over (see the display gate), and its USB-C port controllers @@ -6348,10 +6535,11 @@ static int dcp_platform_probe_t8132_ext(struct platform_device *pdev) } /* - * The T8122 dcpext is a component of the panel's DRM device, so it must - * always register one, or the panel stays on the boot framebuffer. Keep - * deferring while its suppliers may still appear (bounded by the deferred - * probe timeout); on any other failure join as an unusable pipe. + * A failsafe dcpext (dcp_ext_is_failsafe()) is a component of the other + * displays' DRM device, so it must always register one, or they stay on the + * boot framebuffer. Keep deferring while its suppliers, its routes' and its + * bind's, may still appear (bounded by the deferred probe timeout); on any + * other failure join as an unusable pipe. */ static int dcp_platform_probe(struct platform_device *pdev) { @@ -6369,11 +6557,12 @@ static int dcp_platform_probe(struct platform_device *pdev) if (of_device_is_compatible(dev->of_node, "apple,t8132-dcpext")) return dcp_platform_probe_t8132_ext(pdev); - if (!of_device_is_compatible(dev->of_node, "apple,t8122-dcpext") && - !of_device_is_compatible(dev->of_node, "apple,t6041-dcpext")) + if (!dcp_ext_is_failsafe(dev->of_node)) return dcp_platform_probe_pipe(pdev); ret = dcp_ext_route_suppliers(dev); + if (!ret) + ret = dcp_ext_bind_suppliers(dev); if (ret) goto unusable; @@ -6923,9 +7112,26 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t6041_dcpext = { .sink_metadata_edid = true, }; +static const struct apple_dcp_hw_data apple_dcp_hw_t6050 = { + .firmware_dma_bits = 41, + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6050_dcpext = { + .firmware_dma_bits = 41, + .num_dptx_ports = 2, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, + .sink_metadata_edid = true, +}; + static const struct of_device_id of_match[] = { { .compatible = "apple,t6041-dcp", .data = &apple_dcp_hw_t6041, }, { .compatible = "apple,t6041-dcpext", .data = &apple_dcp_hw_t6041_dcpext, }, + { .compatible = "apple,t6050-dcp", .data = &apple_dcp_hw_t6050 }, + { .compatible = "apple,t6050-dcpext", .data = &apple_dcp_hw_t6050_dcpext }, { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t6020-dcpext", .data = &apple_dcp_hw_t6020_dcpext, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, @@ -6955,6 +7161,7 @@ static struct platform_driver apple_platform_driver = { .shutdown = dcp_platform_shutdown, .driver = { .name = "apple-dcp", + .dev_groups = dcp_peer_probe_groups, .of_match_table = of_match, .pm = pm_sleep_ptr(&dcp_platform_pm_ops), }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index d65367c8af13f..20fd3ec5f59cb 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -58,6 +58,19 @@ void dcp_poweroff(struct platform_device *pdev); int dcp_typec_prepare_route(struct apple_dcp *dcp); void dcp_poweron(struct platform_device *pdev); void dcp_external_recover(struct platform_device *pdev); +int dcp_peer_hints_v27(struct apple_dcp *dcp, struct dptx_port *port, + const void *data, size_t size, void *reply, size_t reply_size); +bool dcp_peer_tiling_state_v27(struct apple_dcp *dcp, bool write, + u32 event, u32 param, u32 *value); +void dcp_peer_display_ids_v27(struct apple_dcp *dcp, u32 own, u32 related); +int dcp_peer_modeset_v27(struct apple_dcp *dcp, u32 timing, u32 color, bool vrr); +int dcp_peer_color_v27(struct apple_dcp *dcp, struct iomfb_set_matrix_req *matrix); +int dcp_peer_stop_v27(struct apple_dcp *dcp); +struct apple_dcp *dcp_peer_scanout_v27(struct apple_dcp *dcp); +void dcp_peer_swap_armed_v27(struct apple_dcp *dcp, u32 id); +void dcp_peer_swap_failed_v27(struct apple_dcp *dcp); +bool dcp_peer_swap_complete_v27(struct apple_dcp *dcp, u32 id, ktime_t now, + void (*report)(struct apple_dcp *, u32, ktime_t)); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_get_connector_type(struct platform_device *pdev); @@ -114,6 +127,7 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, unsigned int base_ms); void dcp_external_retry_work(struct work_struct *work); void dcp_external_sink_irq(struct apple_dcp *dcp); +void dcp_tb_event_work(struct work_struct *work); void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay); int dcp_dptx_recover_irq(struct apple_dcp *dcp); int dcp_dptx_park(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index b151887274a26..00f29157241f8 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -96,7 +96,7 @@ static u32 brightness_part2[] = { static bool dcp_backlight_presents_nits(struct apple_dcp *dcp) { - return (dcp->fw_compat == DCP_FIRMWARE_H17P && + return ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || iomfb_v14_7_panel_nits(dcp); } @@ -274,7 +274,7 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, unsigned long flags; int ret; - if ((dcp->fw_compat != DCP_FIRMWARE_H17P && + if ((dcp->fw_compat != DCP_FIRMWARE_H17P && dcp->fw_compat != DCP_FIRMWARE_V_27_0 && !iomfb_is_m4(dcp) && !iomfb_v14_7_panel_nits(dcp)) || !dcp_has_panel(dcp) || !dcp->crtc || !kick) return -EINVAL; diff --git a/drivers/gpu/drm/apple/dcp_external_announce_test.c b/drivers/gpu/drm/apple/dcp_external_announce_test.c index a1bd2dcd4b2ab..f07b3777cc751 100644 --- a/drivers/gpu/drm/apple/dcp_external_announce_test.c +++ b/drivers/gpu/drm/apple/dcp_external_announce_test.c @@ -102,6 +102,7 @@ static struct announce_test *announce_test_init(struct kunit *test) dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; iomfb_queue_init(dcp); mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_event_lock); spin_lock_init(&dcp->relink_lock); mutex_init(&dcp->dcpavserv.lock); spin_lock_init(&dcp->metadata.lock); @@ -115,11 +116,13 @@ static struct announce_test *announce_test_init(struct kunit *test) INIT_DELAYED_WORK(&dcp->hybrid_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->backlight.fade_work, announce_test_noop); + INIT_DELAYED_WORK(&dcp->brightness.fade_work, announce_test_noop); INIT_DELAYED_WORK(&dcp->backlight.retry_work, announce_test_noop); INIT_WORK(&dcp->external_stop_work, announce_test_noop); INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, announce_test_noop); INIT_WORK(&dcp->vblank_wq, announce_test_noop); INIT_WORK(&dcp->dimensions_wq, announce_test_noop); + INIT_WORK(&dcp->tb_event_work, announce_test_noop); INIT_WORK(&t->con.hotplug_wq, announce_test_event); mutex_init(&t->drm.mode_config.mutex); diff --git a/drivers/gpu/drm/apple/dcpext_scanout.c b/drivers/gpu/drm/apple/dcpext_scanout.c index f3266931bb456..e652b8ac207a0 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.c +++ b/drivers/gpu/drm/apple/dcpext_scanout.c @@ -693,7 +693,10 @@ static ssize_t dcpext_pattern_store(struct device *dev, struct device_attribute } mutex_unlock(&dcp->hpd_mutex); /* Firmware DMA and callbacks may outlive any error returned by this worker. */ - __module_get(THIS_MODULE); + if (!try_module_get(THIS_MODULE)) { + atomic_set(&scanout->requested, 0); + return -ENODEV; + } queue_work(system_dfl_wq, &scanout->work); return count; } diff --git a/drivers/gpu/drm/apple/dptx-drive-v27.inc b/drivers/gpu/drm/apple/dptx-drive-v27.inc new file mode 100644 index 0000000000000..4fabf1347a8cd --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-drive-v27.inc @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ +struct dptxport_drive_v27 { + __le32 status_or_count; + u8 reserved0[12]; + __le32 count_or_status; + u8 reserved1[12]; + struct { + __le32 type; + __le32 voltage; + __le32 pre; + } lane[]; +} __packed; + +/* + * A Thunderbolt DP tunnel has no PHY lanes to drive. DCP still trains the + * link through it: SetDriveSettings (count at +0x10, entries at +0x20) is + * answered with status 0 at +0x00 and its entries are kept as sent; + * GetDriveSettings (count at +0x00) returns status 0 at +0x10 and the kept + * entries at +0x20, mode words included. Every other byte is the request's, + * and any value is accepted, so DCP always gets its reply. + */ +static int dptxport_call_drive_tunnel_v27(struct dptx_port *dptx, bool set, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + const u8 *request = data; + u8 *reply = reply_; + u32 count, i; + + if (reply_size < 0x20) + return -EINVAL; + memset(reply, 0, reply_size); + memcpy(reply, request, min(data_size, reply_size)); + + if (set) { + count = data_size >= 0x14 ? get_unaligned_le32(request + 0x10) : 0; + dptx->tunnel_drive_count = 0; + for (i = 0; i < count && i < ARRAY_SIZE(dptx->tunnel_drive) && + data_size >= 0x20 + 12 * (i + 1); i++) { + dptx->tunnel_drive[i][0] = get_unaligned_le32(request + 0x20 + 12 * i); + dptx->tunnel_drive[i][1] = get_unaligned_le32(request + 0x24 + 12 * i); + dptx->tunnel_drive[i][2] = get_unaligned_le32(request + 0x28 + 12 * i); + dptx->tunnel_drive_count = i + 1; + } + put_unaligned_le32(0, reply); + return 0; + } + + count = data_size >= 4 ? get_unaligned_le32(request) : 0; + put_unaligned_le32(0, reply + 0x10); + for (i = 0; i < count && i < ARRAY_SIZE(dptx->tunnel_drive) && + reply_size >= 0x20 + 12 * (i + 1); i++) { + bool kept = i < dptx->tunnel_drive_count; + + put_unaligned_le32(kept ? dptx->tunnel_drive[i][0] : 0, reply + 0x20 + 12 * i); + put_unaligned_le32(kept ? dptx->tunnel_drive[i][1] : 0, reply + 0x24 + 12 * i); + put_unaligned_le32(kept ? dptx->tunnel_drive[i][2] : 0, reply + 0x28 + 12 * i); + } + return 0; +} + +static int dptxport_call_drive_v27(struct apple_epic_service *service, bool set, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + const struct dptxport_drive_v27 *request = data; + struct dptxport_drive_v27 *reply = reply_; + struct dptx_port *dptx = service->cookie; + union phy_configure_opts opts = {}; + u32 count; + size_t size; + int ret; + + if (service->ep->dcp->dptx_tunnel) + return dptxport_call_drive_tunnel_v27(dptx, set, data, data_size, + reply_, reply_size); + if (data_size < sizeof(*request) || reply_size < sizeof(*reply)) + return -EINVAL; + count = le32_to_cpu(set ? request->count_or_status : request->status_or_count); + if (count > 4 || count > dptx->lane_count) + return -EINVAL; + size = sizeof(*request) + count * sizeof(request->lane[0]); + if ((set && data_size < size) || (!set && reply_size < size)) + return -EINVAL; + + if (set) { + for (u32 i = 0; i < count; i++) { + + if (le32_to_cpu(request->lane[i].type) || + le32_to_cpu(request->lane[i].voltage) > 3 || + le32_to_cpu(request->lane[i].pre) > 3 || + le32_to_cpu(request->lane[i].voltage) + + le32_to_cpu(request->lane[i].pre) > 3) + return -EINVAL; + opts.dp.voltage[i] = le32_to_cpu(request->lane[i].voltage); + opts.dp.pre[i] = le32_to_cpu(request->lane[i].pre); + } + opts.dp.lanes = count; + opts.dp.link_rate = dptx->phy_ops.dp.link_rate; + opts.dp.set_voltages = 1; + if (count && !service->ep->dcp->dptx_tunnel) { + if (!dptx->atcphy) + return -ENODEV; + ret = phy_configure(dptx->atcphy, &opts); + if (ret) + return ret; + } + for (u32 i = 0; i < count; i++) { + dptx->drive_voltage[i] = opts.dp.voltage[i]; + dptx->drive_pre[i] = opts.dp.pre[i]; + } + } + + memset(reply, 0, reply_size); + if (set) { + reply->count_or_status = cpu_to_le32(count); + } else { + + reply->status_or_count = cpu_to_le32(count); + } + + if (reply_size >= size) { + for (u32 i = 0; i < count; i++) { + reply->lane[i].voltage = cpu_to_le32(dptx->drive_voltage[i]); + reply->lane[i].pre = cpu_to_le32(dptx->drive_pre[i]); + } + } + return 0; +} diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h index 3d990a9ca1ea1..2cb8f68210eb8 100644 --- a/drivers/gpu/drm/apple/dptx-service.h +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -5,14 +5,18 @@ #include "afk.h" #include "dptxep.h" +/* + * T8140 external firmware names its one port by the DCP index; the others, + * firmware 27 among them, by port unit 0 or 1. + */ static inline int dptx_service_port(const char *name, const char *class, - s64 unit, bool wire_v2, u32 index, + s64 unit, bool unit_is_index, u32 index, u32 num_ports) { if (!name || !class || strcmp(name, "dcpdptx-port-epic") || strcmp(class, "AppleDCPDPTXRemotePort")) return -EINVAL; - if (wire_v2) + if (unit_is_index) return num_ports == 1 && unit == index ? 0 : -EINVAL; return unit >= 0 && unit < 2 ? unit : -EINVAL; } diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index b2426f0a0506a..394e22fba90b1 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -87,6 +87,8 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +#include "dptx-drive-v27.inc" + /* * SetTiledDisplayHints payload, as DCP sends it for an LG UltraFine 5K * (offsets in bytes): tile location (column, row) at 0x08/0x0c, tiles @@ -102,8 +104,9 @@ struct dptxport_apcall_set_tiled { /* * A Thunderbolt DP tunnel uses the same CORE|ATC|DIE|CONNECTED target as a - * direct alt-mode PHY: CORE is the DFP port (0 = dpphy, 1/2 = dpin0/dpin1) - * and ATC the route's own ATC index. + * direct alt-mode PHY: CORE is the DFP port (0 = dpphy, 1..4 = dpin0..3) + * and ATC the route's own ATC index. ATC2's DP IN 0 is 0x8021, its DP-alt + * PHY 0x8020. */ static u32 dptxport_remote_target(struct apple_dcp *dcp, u8 core, u8 atc, u8 die) @@ -123,9 +126,10 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); /* - * Attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | - * supports HPD << 8. Without the DP IN role DCP gives up on a tunnel - * whose AUX channel is slower to answer than a direct PHY's. + * Attributes: role (bits 7:0; 0 = direct PHY, 1 = Thunderbolt/USB4 + * DP IN) | supports HPD (bit 8): 0x101 for a tunnel, 0x100 for a PHY. + * Without the DP IN role DCP gives up on a tunnel whose AUX channel is + * slower to answer than a direct PHY's. */ u32 attrs = 0x100 | (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); @@ -275,7 +279,7 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); - if (service->ep->wire_v2) { + if (dcp_is_external_h17p(service->ep->dcp)) { memset(reply, 0, reply_size); put_unaligned_le32(3, reply_ + 20); put_unaligned_le32(3, reply_ + 24); @@ -284,6 +288,11 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + /* Protocol-27 drive settings: type, voltage, pre-emphasis. */ + reply->max_drive_settings[0] = cpu_to_le32(0); + memcpy(reply->_unk1, &reply->max_drive_settings[1], sizeof(__le32)); + } return 0; } @@ -351,12 +360,22 @@ static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, void *reply_, size_t reply_size) { struct dptxport_apcall_link_rate *reply = reply_; + struct apple_dcp *dcp = service->ep->dcp; + u32 link_rate = LINK_RATE_HBR3; if (reply_size < sizeof(*reply)) return -EINVAL; + /* + * A T6050 DP tunnel's pixel clock runs UHBR10 and UHBR20 as well + * (not UHBR13.5); DCP picks the 128b/132b rates the sink and the + * tunnel's capabilities allow, up to this one. + */ + if (dcp->dptx_tunnel && of_device_is_compatible(dcp->dev->of_node, "apple,t6050-dcpext")) + link_rate = LINK_RATE_UHBR20; + reply->retcode = cpu_to_le32(0); - reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + reply->link_rate = cpu_to_le32(link_rate); return 0; } @@ -373,8 +392,11 @@ static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, if (reply_size < sizeof(*reply)) return -EINVAL; - if (!dptx->atcphy) { - /* USB4 DP IN: no ATC DP PHY to validate. */ + if (!dptx->atcphy || dcp->dptx_tunnel) { + /* + * USB4 DP IN: nothing to validate. A T6050 tunnel's ATC PHY only + * provides its pixel clock. + */ dptx->lane_count = 4; reply->retcode = cpu_to_le32(0); reply->lane_count = cpu_to_le64(4); @@ -424,7 +446,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); - if (service->ep->wire_v2) { + if (dcp_is_external_h17p(dcp)) { ret = dptx_j700_validate_lanes(dptx, lane_count); if (ret) return ret; @@ -444,7 +466,8 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dptx->phy_ops.dp.lanes = lane_count; /* USB4 DP IN has no ATC PHY; still accept the lane count. */ dptx->phy_ops.dp.set_lanes = - service->ep->wire_v2 || dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); + dcp_is_external_h17p(dcp) || dcp_is_usb4_output(dcp) || + (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -584,11 +607,20 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, phy_link_rate = 8100; phy_set_rate = true; break; + case LINK_RATE_UHBR10: + case LINK_RATE_UHBR20: + /* Only a tunnel's pixel clock takes a 128b/132b rate. */ + if (!service->ep->dcp->dptx_tunnel) + goto unsupported; + phy_link_rate = link_rate == LINK_RATE_UHBR10 ? 10000 : 20000; + phy_set_rate = true; + break; case 0: phy_link_rate = 0; phy_set_rate = true; break; default: +unsupported: dev_err(service->ep->dcp->dev, "DPTXPort: Unsupported link rate 0x%x requested\n", link_rate); @@ -607,8 +639,9 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { ret = dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, dptx->unit, link_rate); - /* T6030 14.7: refuse a rate whose tunnel clock failed. */ - if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + /* 14.7 and 27: refuse a rate whose tunnel clock failed. */ + if (ret && (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 || + service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0)) { reply->retcode = cpu_to_le32(1); reply->link_rate = cpu_to_le32(dptx->link_rate); return 0; @@ -749,15 +782,20 @@ dptxport_call_activate(struct apple_epic_service *service, * No PHY mode change for a tunnel: the ATC PHY stays in USB4/TBT * mode for the whole connection. The DP IN adapter may only be woken * (DPTX_INACTIVE=0) from here; waking it earlier hangs the machine. - * Activate always replies success to DCP. + * Activate replies success to DCP, except that firmware 27 is told + * when the PHY could not switch to DisplayPort. */ - if (dptx->atcphy && !dcp->phy_managed_by_typec) - phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dptx->atcphy && !dcp->phy_managed_by_typec) { + ret = phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return ret; + } if (dcp->dptx_tunnel) { ret = dcp_tunnel_dpin_activate(dcp, dptx->unit, true); + if (ret) + dev_err(dcp->dev, "DPTX tunnel activation failed: %d\n", ret); /* T6030 14.7: report a failed tunnel activation to DCP. */ if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) { - dev_err(dcp->dev, "DPTX tunnel activation failed: %d\n", ret); if (reply_size < sizeof(__le32)) return -EINVAL; memcpy(reply, data, min(reply_size, data_size)); @@ -862,6 +900,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) { int xret = dcp_tunnel_crossbar_up(service->ep->dcp, dptx->unit); + if (xret) + dev_err(service->ep->dcp->dev, + "DPTX tunnel crossbar up failed: %d\n", xret); /* T6030 14.7: fail the link config without a crossbar. */ if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) ret = xret; @@ -908,6 +949,19 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_MAX_DRIVE_SETTINGS: return dptxport_call_get_max_drive_settings(service, reply, reply_size); + case DPTX_APCALL_SET_RESOURCE_AVAILABLE: + /* Firmware 27 reports that a DPTX resource is available again. */ + dev_dbg(service->ep->dcp->dev, "DPTXPort: SET_RESOURCE_AVAILABLE\n"); + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; + case DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE: + /* Asked on every connect: a real display, no virtual device mode. */ + if (reply_size < sizeof(struct dptxport_apcall_get_support)) + return -EINVAL; + memset(reply, 0, reply_size); + return 0; case DPTX_APCALL_DEVICE_NOT_RESPONDING: case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: case DPTX_APCALL_DEVICE_NOT_STARTED: @@ -924,6 +978,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, if (dptxport_dual_pipe_alloc(service, data, data_size, reply, reply_size)) return 0; + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_peer_hints_v27(service->ep->dcp, dptx, data, + data_size, reply, reply_size); memcpy(reply, data, min(reply_size, data_size)); ret = dptxport_call_set_tiled_display_hint(service, data, data_size, reply, @@ -934,9 +991,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return ret; } case DPTX_APCALL_GET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, false, data, data_size, + reply, reply_size); return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_SET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, true, data, data_size, + reply, reply_size); return dptxport_call_set_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_ACTIVATE: @@ -1011,7 +1074,7 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name (!class || !class[0] || !strcmp(class, "AppleDCPDPTXRemotePort"))) port = unit >= 0 && unit < 2 ? unit : -EINVAL; else - port = dptx_service_port(name, class, unit, service->ep->wire_v2, + port = dptx_service_port(name, class, unit, dcp_is_external_h17p(dcp), dcp->index, dcp->hw.num_dptx_ports); if (port < 0) @@ -1020,6 +1083,21 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name return dptx_service_bind(service, &dcp->dptxport[port], unit); } +static void dptxport_init_props(struct apple_epic_service *service, s64 unit, + const void *props, size_t size) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_parse_ctx ctx; + struct dcp_dptx_caps caps; + + if (unit < 0 || unit >= ARRAY_SIZE(dcp->dptxport) || + parse(props, size, &ctx) || parse_dptx_caps(&ctx, &caps)) + return; + dcp->dptxport[unit].caps = caps; + dev_info(dcp->dev, "DPTX%lld caps: width=%u total=%u active=%u\n", + unit, caps.width, caps.total_rate, caps.active_rate); +} + static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -1051,12 +1129,6 @@ static int dptxport_v2_group_call(struct apple_epic_service *service, u16 group, return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) return -EOPNOTSUPP; - if (command == DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE) { - if (room < sizeof(struct dptxport_apcall_get_support)) - return -EINVAL; - memset(reply, 0, room); - return 0; - } if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) return dptx_j700_training(service, false, data, size, reply, room); return dptxport_call(service, command, data, size, reply, room); @@ -1093,11 +1165,22 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, { struct dptx_port *dptx = service->cookie; - /* Keep native AFK-v2 training and its refusal policy distinct. */ - if (service->ep->wire_v2) + /* + * Keep the T8140 external (J700) training and its refusal policy + * distinct. It keys on that profile, not on the AFK v2 codec, which + * firmware 27 speaks with the common DPTX calls. + */ + if (dcp_is_external_h17p(service->ep->dcp)) return dptxport_v2_group_call(service, group, command, data, size, reply, room); + /* + * Firmware 27: bit 0 of the group word only flags a variable-length + * payload. Both share one call table, so dispatch by command. + */ + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + group &= ~1; + /* Legacy dispatch is by command, except the tested 13.5 group-1 calls. */ if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_13_5 && group == 1 && (command == 3 || command == 4)) @@ -1143,6 +1226,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .prepare = dptxport_prepare, .teardown = dptxport_teardown, .init = dptxport_init, + .init_props = dptxport_init_props, .call = dptxport_call, .call_v2 = dptxport_group_call, }, @@ -1152,6 +1236,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .prepare = dptxport_prepare, .teardown = dptxport_teardown, .init = dptxport_init, + .init_props = dptxport_init_props, .call = dptxport_call, }, { @@ -1197,13 +1282,6 @@ static int dptxport_compact_call(struct apple_epic_service *service, u16 group, return dptxport_compact_group1(command, data, size, reply, room); if (group) return -EOPNOTSUPP; - /* Asked on every connect; no virtual device is supported. */ - if (command == DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE) { - if (room < sizeof(struct dptxport_apcall_get_support)) - return -EINVAL; - memset(reply, 0, room); - return 0; - } return dptxport_call(service, command, data, size, reply, room); } @@ -1306,7 +1384,8 @@ int dptx_controller_init(struct apple_dcp *dcp) int dptxep_init(struct apple_dcp *dcp) { struct apple_dcp_afkep *ep; - bool wire_v2 = dcp->hw.iomfb_method_profile == + bool wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; int ret; u32 port; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 211e72d9855ea..a088727c57a98 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -3,6 +3,7 @@ #include #include +#include "dcp-peer-v27.h" enum dptx_apcall { DPTX_APCALL_ACTIVATE = 0, @@ -30,6 +31,7 @@ enum dptx_apcall { DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, DPTX_APCALL_DEVICE_NOT_STARTED = 24, + DPTX_APCALL_SET_RESOURCE_AVAILABLE = 25, DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE = 26, }; @@ -44,11 +46,15 @@ enum dptx_link_rate { LINK_RATE_HBR = 0x0a, LINK_RATE_HBR2 = 0x14, LINK_RATE_HBR3 = 0x1e, + /* 128b/132b: 0x80 | the rate's DPCD 0x2215 bit */ + LINK_RATE_UHBR10 = 0x81, + LINK_RATE_UHBR20 = 0x82, }; struct apple_epic_service; struct dptx_port { + struct dcp_dptx_caps caps; bool enabled, connected; /* A published session closed; unlike initial discovery, this is terminal. */ bool closed; @@ -63,6 +69,14 @@ struct dptx_port { u32 link_rate, pending_link_rate; u32 drive_settings[2]; u32 trained_voltage[4], trained_pre[4]; + u32 drive_voltage[4], drive_pre[4]; + /* + * Firmware 27 on a Thunderbolt DP tunnel: the drive settings DCP last + * set, {mode, voltage swing, pre-emphasis} per lane, as it sent them. + * Nothing applies them; GetDriveSettings returns them. + */ + u32 tunnel_drive[4][3]; + u32 tunnel_drive_count; /* * The sink's tile topology as DCP reports it in SetTiledDisplayHints: * tiles across and down, and this stream's tile. Valid once diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index a92af3295fcf8..9621a86b42228 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -712,7 +712,8 @@ int dpavservep_init(struct apple_dcp *dcp) if (IS_ERR(ep)) return PTR_ERR(ep); /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ - ep->wire_v2 = dcp->hw.iomfb_method_profile == + ep->wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; ep->inline_v2 = false; dcp->dcpavservep = ep; diff --git a/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc new file mode 100644 index 0000000000000..b7b617b37994d --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +struct dcp_boolean_property_v27 { + struct list_head node; + char name[64]; + bool value; +}; + +static bool trampoline_set_boolean_property_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + struct dcp_boolean_property_v27 *property; + char name[64]; + + trace_iomfb_callback(dcp, tag, __func__); + + memcpy(name, request, sizeof(name)); + name[sizeof(name) - 1] = '\0'; + put_unaligned_le32(0, out); + + list_for_each_entry(property, &dcp->iomfb_boolean_properties, node) + if (!strcmp(property->name, name)) + goto update; + if (dcp->iomfb_boolean_property_count >= 64) + return true; + property = kzalloc(sizeof(*property), GFP_KERNEL); + if (!property) + return true; + memcpy(property->name, name, sizeof(name)); + dcp->iomfb_boolean_property_count++; + list_add_tail(&property->node, &dcp->iomfb_boolean_properties); +update: + property->value = request[64] & 1; + put_unaligned_le32(1, out); + return true; +} + +void iomfb_firmware_state_cleanup_v27(void *data) +{ + struct apple_dcp *dcp = data; + struct dcp_boolean_property_v27 *property, *next; + + external_counts_cleanup_v27(dcp); + list_for_each_entry_safe(property, next, &dcp->iomfb_boolean_properties, node) { + list_del(&property->node); + kfree(property); + dcp->iomfb_boolean_property_count--; + } +} diff --git a/drivers/gpu/drm/apple/iomfb-dfb.h b/drivers/gpu/drm/apple/iomfb-dfb.h new file mode 100644 index 0000000000000..8175cb2d3c922 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-dfb.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +#ifndef __APPLE_IOMFB_DFB_H__ +#define __APPLE_IOMFB_DFB_H__ + +#include +#include +#include +#include +#include + +static inline u64 apple_dcp_dfb_round_size(u64 size) +{ + unsigned int bits = fls64(size); + u64 unit, half, tail; + + if (bits <= 24) + return size; + unit = 1ULL << (bits - 24); + half = unit >> 1; + tail = size & (unit - 1); + size &= ~(unit - 1); + if (tail > half || (tail == half && (size & unit))) + size += unit; + return size; +} + +static inline u64 apple_dcp_dfb_plane_size(u32 width, u32 height, u32 bpe, + u32 compression) +{ + u64 stride, data, cols, rows, tw = 16, th = 16; + unsigned int order; + + if (!compression) { + stride = ALIGN((u64)width * bpe, 128); + return ALIGN(apple_dcp_dfb_round_size(stride * height), 0x4000); + } + if (compression == 2 && bpe > 4) { + th = 8; + if (bpe == 16) + tw = 8; + } + cols = DIV_ROUND_UP(width, tw); + rows = DIV_ROUND_UP(height, th); + if (compression == 3) { + stride = tw * th * cols * bpe; + data = ALIGN(rows * stride, 128); + return data + 8 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows); + } + order = ilog2(0x4000 / bpe); + stride = ALIGN((u64)width * bpe, (u64)bpe << (order / 2)); + data = ALIGN(ALIGN((u64)height, 1ULL << ((order + 1) / 2)) * stride, 128); + return data + ALIGN(2 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows), 128); +} + +/* + * The DFB's surface format and bytes per element (the surface record's pix_size) + * for the firmware's format and compression; *alpha_plane for a separate alpha plane. + */ +static inline int apple_dcp_dfb_format(u32 format, u32 compression, + u32 *surface_format, u32 *bpe, + bool *alpha_plane) +{ + *alpha_plane = false; + switch (format) { + case 0x62336138: + if (compression == 2) { + format = 0x77343061; + *bpe = 8; + } else { + *bpe = 4; + *alpha_plane = true; + } + break; + case 0x77343061: + *bpe = 8; + break; + case 0x42475241: + case 0x77333072: + case 0x6c313072: + *bpe = 4; + break; + default: + return -EOPNOTSUPP; + } + *surface_format = format; + return 0; +} + +static inline int apple_dcp_dfb_allocation(u32 width, u32 height, u32 format, + u32 compression, u64 region_size, + u32 *surface_format, u32 *allocation) +{ + u32 bpe; + u64 size; + bool alpha_plane; + int ret; + + if (!width || !height || width > 16384 || height > 16384 || !region_size) + return -EINVAL; + if (compression != 0 && compression != 2 && compression != 3) + return -EOPNOTSUPP; + ret = apple_dcp_dfb_format(format, compression, &format, &bpe, &alpha_plane); + if (ret) + return ret; + size = apple_dcp_dfb_plane_size(width, height, bpe, compression); + if (alpha_plane) + size += apple_dcp_dfb_plane_size(width, height, 1, compression); + if (size > U32_MAX) + return -EOVERFLOW; + if (compression) + size = ALIGN(apple_dcp_dfb_round_size(size), 0x4000); + if (size > region_size) + return -ENOSPC; + if (!size || size > U32_MAX) + return -EOVERFLOW; + *surface_format = format; + *allocation = size; + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc new file mode 100644 index 0000000000000..1b3710c1c7627 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +struct dcp_display_product_request_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 vendor_length; + __le32 product_id; + u8 vendor_id_null, product_id_null; +} __packed; + +struct dcp_display_product_reply_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 product_id; + u8 success, padding; +} __packed; + +static_assert(sizeof(struct dcp_display_product_request_v27) == 24); +static_assert(sizeof(struct dcp_display_product_reply_v27) == 20); +static_assert(offsetof(struct dcp_display_product_reply_v27, product_id) == 14); + +struct dcp_display_identity_callback_v27 { + void *output; + enum dcp_context_id context; +}; + +static void display_identity_done_v27(struct apple_dcp *dcp, void *out, void *cookie) +{ + const struct dcp_display_product_reply_v27 *reply = out; + struct dcp_display_identity_callback_v27 *cb = cookie; + bool vendor_matches = reply && (!reply->vendor[0] || + (!memcmp(reply->vendor, "APP", 3) && !reply->vendor[3])); + bool display_x1 = reply && (reply->success & 1) && vendor_matches && + (le32_to_cpu(reply->product_id) >> 1) == 0x5721; + + put_unaligned_le32(display_x1, cb->output); + dcp_ack(dcp, cb->context); + kfree(cb); +} + +static bool trampoline_display_identity_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + static const struct dcp_method_entry method = { + .name = "display_product_id", .tag = {'A', '3', '7', '5'}, + }; + static const enum dcp_context_id contexts[] = { + DCP_CONTEXT_CB, DCP_CONTEXT_ASYNC, DCP_CONTEXT_OOBCB, DCP_CONTEXT_OOBASYNC, + }; + struct dcp_channel *channels[] = { + &dcp->ch_cb, &dcp->ch_async, &dcp->ch_oobcb, &dcp->ch_oobasync, + }; + struct dcp_display_product_request_v27 request = { + .vendor_length = cpu_to_le32(10), + }; + struct dcp_display_identity_callback_v27 *cb; + int context = -1; + + trace_iomfb_callback(dcp, tag, __func__); + + for (unsigned int i = 0; i < ARRAY_SIZE(channels); i++) { + struct dcp_channel *ch = channels[i]; + + if (ch->depth && ch->output[ch->depth - 1] == out) { + context = contexts[i]; + break; + } + } + if (WARN_ON(context < 0)) + return true; + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return true; + cb->output = out; + cb->context = context; + if (dcp_push_nested(dcp, context, &method, sizeof(request), + sizeof(struct dcp_display_product_reply_v27), &request, + display_identity_done_v27, cb)) { + kfree(cb); + return true; + } + return false; +} diff --git a/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc new file mode 100644 index 0000000000000..ab5602821d3ca --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +static DEFINE_SPINLOCK(external_counts_lock_v27); +static u32 external_pipe_count_v27; +static u32 external_apple_display_count_v27; +static u32 external_suppression_count_v27; + +static bool trampoline_update_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + bool plug = request[0] & 1; + u32 apple_display = request[1] & 1; + u32 pipes = get_unaligned_le32(request + 2); + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) + return true; + spin_lock_irqsave(&external_counts_lock_v27, flags); + if ((plug && (pipes > U32_MAX - external_pipe_count_v27 || + apple_display > U32_MAX - external_apple_display_count_v27)) || + (!plug && (pipes > dcp->external_pipes_v27 || + apple_display > dcp->external_displays_v27))) { + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + dev_warn_ratelimited(dcp->dev, "invalid external display count update\n"); + return true; + } + dcp->external_pipes_v27 += plug ? pipes : -pipes; + dcp->external_displays_v27 += plug ? apple_display : -apple_display; + external_pipe_count_v27 += plug ? pipes : -pipes; + external_apple_display_count_v27 += plug ? apple_display : -apple_display; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static bool trampoline_get_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + u32 counts[3]; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + counts[0] = external_pipe_count_v27; + counts[1] = external_apple_display_count_v27; + counts[2] = external_suppression_count_v27; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + + for (unsigned int i = 0; i < ARRAY_SIZE(counts); i++) + if (!(request[12 + i] & 1)) + put_unaligned_le32(counts[i], (u8 *)out + 4 * i); + return true; +} + +static bool trampoline_apply_suppression_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + if (((request[0] & 1) && external_suppression_count_v27 == U32_MAX) || + (!(request[0] & 1) && !dcp->external_suppression_v27)) { + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; + } + dcp->external_suppression_v27 += (request[0] & 1) ? 1U : -1U; + external_suppression_count_v27 += (request[0] & 1) ? 1U : -1U; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static void external_counts_cleanup_v27(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&external_counts_lock_v27, flags); + external_pipe_count_v27 -= dcp->external_pipes_v27; + external_apple_display_count_v27 -= dcp->external_displays_v27; + external_suppression_count_v27 -= dcp->external_suppression_v27; + dcp->external_pipes_v27 = 0; + dcp->external_displays_v27 = 0; + dcp->external_suppression_v27 = 0; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); +} diff --git a/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc new file mode 100644 index 0000000000000..c2a681328c102 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include "iomfb-peer-v27.h" + +static void dcp_peer_swap_error(struct apple_dcp *owner, const char *where, u32 status) +{ + lockdep_assert_held(&owner->iomfb.lock); + dev_err(owner->dev, "paired swap %s failed: %#x\n", where, status); + dcp_peer_swap_failed_v27(owner); + /* One engine may already have accepted this frame. A failed peer ACK + * is not a DMA-retirement witness for either scanout. The clean queue + * retains its active transaction and next-scanout references on crash. */ + WRITE_ONCE(owner->crashed, true); + dcp_swap_watchdog_complete(owner); + dcp_present_failed(owner); + schedule_work(&owner->vblank_wq); +} + +static void dcp_peer_submit_error_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + u32 *status = data; + dcp_peer_swap_error(owner, "peer submit", *status); +} + +static void dcp_peer_submitted(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct apple_dcp *owner = cookie; + struct dcp_swap_submit_resp_v27 *resp = data; + u32 status = resp ? resp->ret : ~0U; + u32 id; + + if (owner != dcp) { + if (status && iomfb_queue_callback(owner, dcp_peer_submit_error_on_owner, + &status, sizeof(status), NULL)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } + return; + } + lockdep_assert_held(&owner->iomfb.lock); + if (READ_ONCE(owner->peer_frames.failed)) + return; + id = owner->swap.h17p.swap.swap_id; + if (status) { + dcp_peer_swap_error(owner, "primary submit", status); + return; + } + if (dcp_present_submit(owner, id, true)) { + dcp_peer_swap_error(owner, "submit state", -EPROTO); + return; + } + /* Per-endpoint receive ordering handles the primary ACK before its + * completion. The paired watermark still requires both completions. */ + owner->swap_start = ktime_get(); + owner->swap_submit_timestamp = arch_timer_read_counter(); + dcp_swap_watchdog_arm(owner); +} + +static void dcp_peer_started_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + struct dcp_swap_start_resp_h17p *resp = data; + struct apple_dcp *peer = dcp_peer_scanout_v27(owner); + u32 id = owner->swap.h17p.swap.swap_id; + + lockdep_assert_held(&owner->iomfb.lock); + if (IS_ERR_OR_NULL(peer) || peer != cookie || resp->ret || resp->swap_id != id) { + dcp_peer_swap_error(owner, "start/ID", resp->ret ?: -EPROTO); + return; + } + /* Keep the logical atomic input intact. Both split wire records are + * primary-owned; dcp_push copies the peer's record into its own queue. */ + iomfb_serialize_present_h17p(&owner->present_h17p, &owner->swap.h17p); + if (!iomfb_apply_backlight_h17p(owner, &owner->swap.h17p, &owner->present_h17p)) { + dcp_peer_swap_error(owner, "surface preparation", -EINVAL); + return; + } + iomfb_serialize_present_v27(&owner->present_v27, &owner->swap.h17p); + owner->present_v27.swap.brightness_update = owner->present_h17p.swap[0x354] & 1; + owner->present_v27.swap.brightness_nits = get_unaligned_le64(owner->present_h17p.swap + 0x35e); + dcp_peer_split_swap_v27(&owner->present_v27, &owner->peer_present_v27, + owner->peer_tiling.width); + if (dcp_present_retires(owner)) + dcp_arm_retained_framebuffers(owner, id); + dcp_peer_swap_armed_v27(owner, id); + trace_iomfb_swap_submit(owner, id); + trace_iomfb_swap_submit(peer, id); + dcp_push(owner, false, &dcp_methods[dcpep_swap_submit], sizeof(owner->present_v27), + sizeof(struct dcp_swap_submit_resp_v27), &owner->present_v27, + dcp_peer_submitted, owner); + if (READ_ONCE(owner->crashed)) + return; + dcp_push(peer, false, &dcp_methods[dcpep_swap_submit], sizeof(owner->peer_present_v27), + sizeof(struct dcp_swap_submit_resp_v27), &owner->peer_present_v27, + dcp_peer_submitted, owner); + if (READ_ONCE(peer->crashed)) + dcp_peer_swap_error(owner, "peer publication", -EIO); +} + +static void dcp_peer_started(struct apple_dcp *peer, void *data, void *cookie) +{ + struct apple_dcp *owner = cookie; + struct dcp_swap_start_resp_h17p failed = { .ret = ~0U }; + + if (iomfb_queue_callback(owner, dcp_peer_started_on_owner, + data ?: &failed, sizeof(failed), peer)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } +} + +static bool dcp_peer_start_second(struct apple_dcp *owner, + struct dcp_swap_start_resp_h17p *resp) +{ + struct dcp_swap_start_req_h17p req = {}; + struct apple_dcp *peer = dcp_peer_scanout_v27(owner); + + if (!peer) + return false; + if (IS_ERR(peer) || resp->ret) { + dcp_peer_swap_error(owner, "primary start", resp->ret ?: -EIO); + return true; + } + owner->swap.h17p.swap.swap_id = resp->swap_id; + dcp_swap_start(peer, false, &req, dcp_peer_started, owner); + return true; +} diff --git a/drivers/gpu/drm/apple/iomfb-peer-v27.h b/drivers/gpu/drm/apple/iomfb-peer-v27.h new file mode 100644 index 0000000000000..ea3b586e037c9 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-peer-v27.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_IOMFB_PEER_V27_H +#define APPLE_IOMFB_PEER_V27_H + +#include +#include +#include +#include +#include "iomfb_plane.h" +#include "iomfb_v27.h" + +/* A364 and A365 share the video-data prefix, but not the final + * flags: A364 has a requires-merge boolean before three pointer-null flags; + * A365 has only two pointer-null flags and padding. + */ +struct dcp_peer_video_data_v27 { + u8 timing[0x50]; + u8 color[0x20]; +} __packed; + +struct dcp_peer_get_video_req_v27 { + struct dcp_peer_video_data_v27 video; + __le32 timing_id, color_id; + u8 requires_merge, timing_null, color_null, requires_merge_null; +} __packed; + +struct dcp_peer_get_video_resp_v27 { + struct dcp_peer_video_data_v27 video; + u8 requires_merge, success, padding[2]; +} __packed; + +struct dcp_peer_insert_video_req_v27 { + struct dcp_peer_video_data_v27 video; + __le32 timing_id, color_id; + u8 timing_null, color_null, padding[2]; +} __packed; + +struct dcp_peer_insert_video_resp_v27 { + struct dcp_peer_video_data_v27 video; + u8 success, padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_peer_get_video_req_v27) == 0x7c); +static_assert(sizeof(struct dcp_peer_get_video_resp_v27) == 0x74); +static_assert(sizeof(struct dcp_peer_insert_video_req_v27) == 0x7c); +static_assert(sizeof(struct dcp_peer_insert_video_resp_v27) == 0x74); + +/* The digital-out mode setter. + * Keep vertical timings, color data and all unmodified fields intact. + * The signed horizontal offset uses division toward zero. + */ +static inline void dcp_peer_split_timing_v27(struct dcp_peer_video_data_v27 *v) +{ + static const u8 half_fields[] = { 4, 8, 12, 16, 20, 28, 68 }; + + for (unsigned int i = 0; i < sizeof(half_fields); i++) { + u8 *field = v->timing + half_fields[i]; + + put_unaligned_le32(get_unaligned_le32(field) >> 1, field); + } + put_unaligned_le32((s32)get_unaligned_le32(v->timing + 24) / 2, + v->timing + 24); +} + +/* VFTG splits a surface at the half-panel boundary. + * Source and destination use pixels here, as in the existing IOMFB plane + * path. Use rational arithmetic in the kernel rather than floating point; + * round a shared source boundary once so neither half loses/duplicates it. + * DRM has already clipped negative/offscreen destination coordinates. + */ +static inline bool dcp_peer_clip_surface_v27(const struct dcp_rect *src, + const struct dcp_rect *dst, u32 tile_x, u32 tile_width, + struct dcp_rect *tile_src, struct dcp_rect *tile_dst) +{ + u64 left = max_t(u64, dst->x, tile_x); + u64 right = min_t(u64, (u64)dst->x + dst->w, + (u64)tile_x + tile_width); + u64 first, last; + + memset(tile_src, 0, sizeof(*tile_src)); + memset(tile_dst, 0, sizeof(*tile_dst)); + if (!src->w || !src->h || !dst->w || !dst->h || left >= right) + return false; + first = div_u64((left - dst->x) * src->w + dst->w / 2, dst->w); + last = div_u64((right - dst->x) * src->w + dst->w / 2, dst->w); + if (first >= last || (u64)src->x + last > U32_MAX) + return false; + *tile_src = *src; + tile_src->x += first; + tile_src->w = last - first; + *tile_dst = *dst; + tile_dst->x = left - tile_x; + tile_dst->w = right - left; + return true; +} + + +/* Clone descriptors only: both DCPs use the same display DMA domain/IOVA. */ +static inline void dcp_peer_split_swap_v27(struct dcp_swap_submit_req_v27 *left, + struct dcp_swap_submit_req_v27 *right, + u32 tile_width) +{ + *right = *left; + for (unsigned int i = 0; i < ARRAY_SIZE(left->surf_null); i++) { + struct dcp_rect src = left->swap.src_rect[i]; + struct dcp_rect dst = left->swap.dst_rect[i]; + bool present = !left->surf_null[i]; + struct dcp_swap_submit_req_v27 *halves[] = {left, right}; + + for (unsigned int side = 0; side < 2; side++) { + struct dcp_swap_submit_req_v27 *req = halves[side]; + struct dcp_rect clipped_src, clipped_dst; + + if (present && dcp_peer_clip_surface_v27(&src, &dst, + side * tile_width, tile_width, &clipped_src, &clipped_dst)) { + req->swap.src_rect[i] = clipped_src; + req->swap.dst_rect[i] = clipped_dst; + } else { + req->surf_null[i] = true; + req->surf_iova[i] = 0; + req->swap.surf_ids[i] = 0; + memset(&req->swap.src_rect[i], 0, sizeof(req->swap.src_rect[i])); + memset(&req->swap.dst_rect[i], 0, sizeof(req->swap.dst_rect[i])); + } + } + } +} + +#endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8bca8d2cebeca..6608b74536cd1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -237,6 +237,10 @@ static int dcp_tx_offset(enum dcp_context_id id) case DCP_CONTEXT_OOBCB: case DCP_CONTEXT_OOBCMD: return 0x08000; + case DCP_CONTEXT_ASYNC_CB: + return 0x20000; + case DCP_CONTEXT_OOBASYNC_CB: + return 0x28000; default: return -EINVAL; } @@ -245,8 +249,10 @@ static int dcp_tx_offset(enum dcp_context_id id) static int dcp_channel_offset(enum dcp_context_id id) { switch (id) { + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return 0x40000; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return 0x48000; case DCP_CONTEXT_CB: @@ -316,8 +322,10 @@ static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, return &dcp->ch_oobcb; case DCP_CONTEXT_OOBCMD: return &dcp->ch_oobcmd; + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return &dcp->ch_async; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return &dcp->ch_oobasync; default: @@ -353,7 +361,7 @@ static u8 dcp_pop_depth(u8 *depth) /* Older firmware and the H17G method profile retain their existing transport. */ static bool iomfb_uses_queue(struct apple_dcp *dcp) { - return dcp->fw_compat == DCP_FIRMWARE_H17P && + return (dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } @@ -388,7 +396,8 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) return 0; - if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; if (dcp->iomfb.queued >= 32) return -EBUSY; @@ -432,7 +441,8 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) /* The active operation may still be visible to firmware. */ return; } - if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending) + if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending || + READ_ONCE(dcp->iomfb.callback_count)) return; if (transaction) { @@ -478,7 +488,8 @@ static void iomfb_queue_work(struct work_struct *work) goto unlock; } if (dcp->iomfb.active || !iomfb_channels_idle(dcp) || - dcp->present_state_h17p.pending || list_empty(&dcp->iomfb.pending)) + dcp->present_state_h17p.pending || READ_ONCE(dcp->iomfb.callback_count) || + list_empty(&dcp->iomfb.pending)) goto unlock; transaction = list_first_entry(&dcp->iomfb.pending, @@ -571,12 +582,92 @@ static void iomfb_backlight_retry(struct work_struct *work) schedule_work(&dcp->bl_update_wq); } +/* A peer endpoint may finish a dependency while this endpoint has an active + * transaction. Continue on its serialized owner, not by taking another + * endpoint's IOMFB mutex from a receive callback. */ +struct iomfb_callback { + struct list_head link; + dcp_callback_t callback; + void *cookie; + u8 data[]; +}; + +int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, + const void *data, size_t size, void *cookie) +{ + struct iomfb_callback *item; + unsigned long flags; + int ret = 0; + + /* This queue carries control replies and completion watermarks only. */ + if (!callback || size > 64 || (size && !data)) + return -EINVAL; + item = kzalloc(struct_size(item, data, size), GFP_KERNEL); + if (!item) + return -ENOMEM; + item->callback = callback; + item->cookie = cookie; + if (size) + memcpy(item->data, data, size); + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + if (READ_ONCE(dcp->iomfb.stopped) || READ_ONCE(dcp->crashed)) { + ret = -ESHUTDOWN; + } else if (dcp->iomfb.callback_count >= 64) { + ret = -EBUSY; + } else { + list_add_tail(&item->link, &dcp->iomfb.callbacks); + dcp->iomfb.callback_count++; + /* Stop takes this lock before cancelling work, closing enqueue/stop. */ + schedule_work(&dcp->iomfb.callback_work); + } + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); + if (ret) + kfree(item); + return ret; +} + +static struct iomfb_callback *iomfb_pop_callback(struct apple_dcp *dcp) +{ + struct iomfb_callback *item = NULL; + unsigned long flags; + + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + if (!list_empty(&dcp->iomfb.callbacks)) { + item = list_first_entry(&dcp->iomfb.callbacks, struct iomfb_callback, link); + list_del(&item->link); + dcp->iomfb.callback_count--; + } + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); + return item; +} + +static void iomfb_callback_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, iomfb.callback_work); + struct iomfb_callback *item; + + mutex_lock(&dcp->iomfb.lock); + while ((item = iomfb_pop_callback(dcp))) { + if (!dcp->iomfb.stopped && !READ_ONCE(dcp->crashed)) { + WRITE_ONCE(dcp->iomfb.owner, current); + item->callback(dcp, item->data, item->cookie); + WRITE_ONCE(dcp->iomfb.owner, NULL); + } + kfree(item); + } + iomfb_queue_advance(dcp); + mutex_unlock(&dcp->iomfb.lock); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_LIST_HEAD(&dcp->iomfb.dropped); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); + spin_lock_init(&dcp->iomfb.callback_lock); + INIT_LIST_HEAD(&dcp->iomfb.callbacks); + INIT_WORK(&dcp->iomfb.callback_work, iomfb_callback_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); @@ -584,14 +675,22 @@ void iomfb_queue_init(struct apple_dcp *dcp) void iomfb_queue_stop(struct apple_dcp *dcp) { + struct iomfb_callback *item; + unsigned long flags; + if (!iomfb_uses_queue(dcp)) return; mutex_lock(&dcp->iomfb.lock); - dcp->iomfb.stopped = true; + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + WRITE_ONCE(dcp->iomfb.stopped, true); + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); iomfb_discard_pending(dcp); mutex_unlock(&dcp->iomfb.lock); cancel_work_sync(&dcp->iomfb.work); + cancel_work_sync(&dcp->iomfb.callback_work); + while ((item = iomfb_pop_callback(dcp))) + kfree(item); cancel_delayed_work_sync(&dcp->iomfb.timeout); cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } @@ -612,7 +711,7 @@ bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) mutex_lock(&dcp->iomfb.lock); idle = READ_ONCE(dcp->crashed) || dcp->iomfb.stopped || (!dcp->iomfb.active && list_empty(&dcp->iomfb.pending) && - !dcp->present_state_h17p.pending); + !dcp->present_state_h17p.pending && !READ_ONCE(dcp->iomfb.callback_count)); mutex_unlock(&dcp->iomfb.lock); if (idle) return true; @@ -809,9 +908,9 @@ static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, } /* Call a DCP function given by a tag */ -void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, - u32 in_len, u32 out_len, void *data, dcp_callback_t cb, - void *cookie) +static void dcp_push_context(struct apple_dcp *dcp, enum dcp_context_id context, + const struct dcp_method_entry *call, u32 in_len, + u32 out_len, void *data, dcp_callback_t cb, void *cookie) { struct dcp_method_entry resolved = *call; @@ -823,14 +922,6 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca } #endif - /* Reply chains stay on the serialized receiver; callers enqueue copies. */ - if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { - if (iomfb_queue_command(dcp, oob, call, in_len, out_len, - data, cb, cookie)) - WRITE_ONCE(dcp->crashed, true); - return; - } - if (dcp_fail_stop(dcp) && READ_ONCE(dcp->crashed)) return; @@ -839,7 +930,6 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); call = &resolved; - enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); struct dcp_packet_header header = { @@ -893,6 +983,82 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca dcpep_msg(context, data_len, offset)); } +void dcp_push(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ + if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { + if (iomfb_queue_command(dcp, oob, call, in_len, out_len, + data, cb, cookie)) + WRITE_ONCE(dcp->crashed, true); + return; + } + + dcp_push_context(dcp, dcp_call_context(dcp, oob), call, + in_len, out_len, data, cb, cookie); +} + +/* + * Push one command on the independent OOB command stream while a transaction + * on the command stream is still pending (dcp_push() would queue it behind + * that transaction). For queries the firmware serves concurrently, such as a + * paired modeset's DSC handoff while the primary's A413 waits for its peer. + */ +int dcp_push_oob_concurrent(struct apple_dcp *dcp, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie) +{ + int ret = 0; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + ret = -ESHUTDOWN; + } else if (dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth || dcp->ch_oobasync.depth || + READ_ONCE(dcp->iomfb.owner)) { + ret = -EBUSY; + } else { + WRITE_ONCE(dcp->iomfb.owner, current); + dcp_push_context(dcp, dcp_call_context(dcp, true), call, + in_len, out_len, data, cb, cookie); + WRITE_ONCE(dcp->iomfb.owner, NULL); + } + mutex_unlock(&dcp->iomfb.lock); + return ret; +} + +/* A nested query must return on the firmware-initiated stream it answers. */ +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + enum dcp_context_id context; + struct dcp_channel *ch; + + if (READ_ONCE(dcp->iomfb.owner) != current || READ_ONCE(dcp->crashed)) + return -ESHUTDOWN; + switch (origin) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_OOBCB: + context = origin; + break; + case DCP_CONTEXT_ASYNC: + context = DCP_CONTEXT_ASYNC_CB; + break; + case DCP_CONTEXT_OOBASYNC: + context = DCP_CONTEXT_OOBASYNC_CB; + break; + default: + return -EINVAL; + } + ch = dcp_get_channel(dcp, context); + if (!ch || !ch->depth || ch->depth >= DCP_MAX_CALL_DEPTH || + (u64)sizeof(struct dcp_packet_header) + in_len + out_len > 0x8000) + return -EINVAL; + dcp_push_context(dcp, context, call, in_len, out_len, data, cb, cookie); + return READ_ONCE(dcp->crashed) ? -EPROTO : 0; +} + /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ int dcp_parse_tag(char tag[4]) { @@ -1530,6 +1696,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, bool handled, zero_reply; if ((dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || offset >= 0x8000 || length > 0x8000 - offset)) { @@ -1567,13 +1734,15 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, dcp->crashed = true; return; } - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && (tag < 0 || tag >= IOMFB_MAX_CB || (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && - !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unsupported IOMFB callback %d (%u, %u)\n", + !(dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + iomfb_v27_callback_size_valid(tag, hdr->in_len, hdr->out_len) : + iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len))))) { + dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; return; @@ -1609,6 +1778,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, READ_ONCE(dcp->desktop->modeset_active)) WRITE_ONCE(dcp->desktop->modeset_callback_error, true); if (dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { if (hdr->out_len) memset(out, 0, hdr->out_len); @@ -1675,7 +1845,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) * its output cannot be trusted. The tag is left out of the check: it * has only been seen preserved on 13.5 firmware. */ - if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + if ((dcp->fw_compat != DCP_FIRMWARE_H17P && dcp->fw_compat != DCP_FIRMWARE_V_27_0)) { record = out - sent->in_len - sizeof(*record); if (record->in_len != sent->in_len || record->out_len != sent->out_len) { @@ -2835,6 +3005,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_H17P: ret = iomfb_modeset_h17p(dcp, crtc_state); break; + case DCP_FIRMWARE_V_27_0: + ret = iomfb_modeset_v27(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -3119,8 +3292,9 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) if (transaction->brightness_only) return false; } - atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, - IOMFB_OPAQUE_X_NEEDED); + if (dcp->fw_compat != DCP_FIRMWARE_V_27_0) + atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED); iomfb_scanout_complete_h17p(dcp); return true; } @@ -3200,7 +3374,10 @@ static void iomfb_backlight_start(struct apple_dcp *dcp, return; dcp->swap.h17p = scanout->request; - iomfb_present_backlight_h17p(dcp); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_present_backlight_v27(dcp); + else + iomfb_present_backlight_h17p(dcp); } static void iomfb_backlight_release(struct iomfb_transaction *transaction) @@ -3258,7 +3435,10 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, return; } dcp->iomfb.next_scanout = atomic->scanout; - iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_flush_v27(dcp, atomic->crtc, atomic->state); + else + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); } static void iomfb_atomic_release(struct iomfb_transaction *transaction) @@ -3416,6 +3596,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) case DCP_FIRMWARE_H17P: iomfb_flush_h17p(dcp, crtc, commit); break; + case DCP_FIRMWARE_V_27_0: + iomfb_flush_v27(dcp, crtc, commit); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -3440,6 +3623,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_start_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_start_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -3515,7 +3701,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * has nothing answering it on the AP side, and without this one the * display DART faults. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) @@ -3560,6 +3746,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_shutdown_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_shutdown_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 2ae15b13b8d43..1021825209fb3 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -4,6 +4,8 @@ #ifndef __APPLE_DCPEP_H__ #define __APPLE_DCPEP_H__ +#include +#include #include #include "version_utils.h" @@ -15,6 +17,7 @@ enum dcp_context_id { /* Callback */ DCP_CONTEXT_CB = 0, + DCP_CONTEXT_ASYNC_CB = 1, /* Command */ DCP_CONTEXT_CMD = 2, @@ -24,6 +27,7 @@ enum dcp_context_id { /* Out-of-band callback */ DCP_CONTEXT_OOBCB = 4, + DCP_CONTEXT_OOBASYNC_CB = 5, /* Out-of-band command */ DCP_CONTEXT_OOBCMD = 6, @@ -184,6 +188,8 @@ enum dcpep_method { * H17P pipe-configuration calls made before the first swap. Most * names are unknown; they are identified by method number. */ + dcpep_pipe_query_353, + dcpep_pipe_set_352, dcpep_pipe_cfg_415, dcpep_pipe_cfg_031, dcpep_pipe_cfg_414, @@ -476,6 +482,44 @@ struct dcp_hotplug_resp_v26_6 { u8 opaque[0x4c]; } __packed; +/* + * A name and a serialized dictionary: the firmware's analytics event + * (D113 on 27.x, D114 on desktop 26.6). + */ +struct dcp_dict_passthrough_req { + u8 key[0x40]; + u8 value[0x1000]; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_dict_passthrough_resp { + u8 value[0x1000]; + u8 ret; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_dict_passthrough_req) == 0x1044); +static_assert(sizeof(struct dcp_dict_passthrough_resp) == 0x1004); + +/* + * The analytics event passes its dictionary by pointer, so the firmware + * deserializes the reply's copy when the call returns, and stops if that + * fails. Return the dictionary as it came, and report the event not sent. + * A zeroed reply is not a dictionary. + */ +static inline void +dcp_dict_passthrough_reply(const struct dcp_dict_passthrough_req *req, + struct dcp_dict_passthrough_resp *resp) +{ + if (req->value_null) + memset(resp->value, 0, sizeof(resp->value)); + else + memcpy(resp->value, req->value, sizeof(resp->value)); + resp->ret = false; + memset(resp->padding, 0, sizeof(resp->padding)); +} + /* A boolean argument, padded to four bytes. */ struct dcp_bool_arg { u8 value; diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index b34a1171123e3..9bb6323d49c79 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -100,6 +100,9 @@ typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); } /* Call a DCP function given by a tag */ +int dcp_push_oob_concurrent(struct apple_dcp *dcp, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie); void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie); @@ -118,3 +121,6 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit); void dcp_hdr_metadata_forget(struct apple_dcp *dcp); +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ad87814e9bb9b..b5efa1613b182 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -32,6 +32,7 @@ #include "dcp-edr.h" #include "iomfb.h" #include "iomfb_internal.h" +#include "iomfb-dfb.h" #include "parser.h" #include "dcp-refresh.h" #include "dcp-property.h" @@ -369,10 +370,24 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, cb(dcp, NULL, cookie); return; } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + iomfb_serialize_present_v27(&dcp->present_v27, request); + dcp->present_v27.swap.brightness_update = + dcp->present_h17p.swap[0x354] & 1; + dcp->present_v27.swap.brightness_nits = + get_unaligned_le64(dcp->present_h17p.swap + 0x35e); + data = &dcp->present_v27; +#else data = &dcp->present_h17p; +#endif } - dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + sizeof(struct dcp_swap_submit_req_v27), +#else + sizeof(*request), +#endif sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); } #else @@ -407,7 +422,7 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * programmed, which is the panel's native mode, instead of requesting one. */ #if DCP_FW_H17(DCP_FW_VER) -#define DCP_INHERIT_BOOT_MODE true +#define DCP_INHERIT_BOOT_MODE (DCP_FW_VER < DCP_FW_VERSION(27, 0, 0)) #else #define DCP_INHERIT_BOOT_MODE false #endif @@ -420,7 +435,7 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * still sends its clear swap first. */ #if DCP_FW_H17(DCP_FW_VER) -#define DCP_HAS_CLIENT_TEARDOWN false +#define DCP_HAS_CLIENT_TEARDOWN ((DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0))) #else #define DCP_HAS_CLIENT_TEARDOWN true #endif @@ -600,29 +615,27 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } -static void dcpep_cb_swap_complete(struct apple_dcp *dcp, - struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +static void dcp_report_swap_complete(struct apple_dcp *dcp, u32 id, ktime_t now) { - ktime_t now = ktime_get(); bool retire = true; #if DCP_FW_DESKTOP_26(DCP_FW_VER) - if (dcp_peer_swap_complete_26_6(dcp, resp->swap_id)) + if (dcp_peer_swap_complete_26_6(dcp, id)) return; #endif #if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + !dcp_present_complete_h17p(&dcp->present_state_h17p, id)) { /* * Abort semantics are not captured on H17P. If an aborted * present completes after all, its framebuffers were already * unarmed and its event signalled; record it and carry on. */ if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, - resp->swap_id)) { + id)) { dev_warn_ratelimited(dcp->dev, "completion for aborted present %u ignored\n", - resp->swap_id); + id); return; } /* @@ -631,13 +644,13 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, * in flight now is a different one. */ if (dcp_present_was_retired_h17p(&dcp->present_state_h17p, - resp->swap_id)) { + id)) { dev_warn_ratelimited(dcp->dev, "completion for retired present %u ignored\n", - resp->swap_id); + id); return; } - dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dev_err(dcp->dev, "unexpected present completion %u\n", id); dcp->crashed = true; return; } @@ -646,30 +659,59 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, #endif #if DCP_FW_DESKTOP_26(DCP_FW_VER) /* Late or duplicate completions cannot retire the replacement's flip. */ - if (!dcp_desktop_present_complete(dcp, resp->swap_id)) + if (!dcp_desktop_present_complete(dcp, id)) return; - dcp_backlight_desktop_finish(dcp, resp->swap_id, false, true); + dcp_backlight_desktop_finish(dcp, id, false, true); #endif - trace_iomfb_swap_complete(dcp, resp->swap_id); - dcp->last_swap_id = resp->swap_id; + dcp->last_swap_id = id; dcp->have_swap_complete = true; dcp_swap_watchdog_complete(dcp); if (retire) { - dcp_release_retained_framebuffers(dcp, resp->swap_id); + dcp_release_retained_framebuffers(dcp, id); dcp_drm_crtc_page_flip(dcp, now); } if (READ_ONCE(dcp->crc_enabled)) { #if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - iomfb_queue_crc_h17p(dcp, resp->swap_id); + iomfb_queue_crc_h17p(dcp, id); return; } #endif u32 crc32 = 0; - drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); + drm_crtc_add_crc_entry(&dcp->crtc->base, true, id, &crc32); } } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +struct dcp_peer_completion { u32 id; ktime_t time; }; +static void dcp_peer_report_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + struct dcp_peer_completion *completion = data; + dcp_report_swap_complete(owner, completion->id, completion->time); +} +static void dcp_peer_queue_completion(struct apple_dcp *owner, u32 id, ktime_t time) +{ + struct dcp_peer_completion completion = { id, time }; + if (iomfb_queue_callback(owner, dcp_peer_report_on_owner, &completion, + sizeof(completion), NULL)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } +} +#endif + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + ktime_t now = ktime_get(); + trace_iomfb_swap_complete(dcp, resp->swap_id); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_swap_complete_v27(dcp, resp->swap_id, now, dcp_peer_queue_completion)) + return; +#endif + dcp_report_swap_complete(dcp, resp->swap_id, now); +} + /* special */ static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) { @@ -710,7 +752,7 @@ static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, struct dcp_h17p_dfb_surface_req, u32); -static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +static void __maybe_unused dcp_ack_dfb_surface(struct apple_dcp *dcp) { struct dcp_channel *ch = &dcp->ch_cb; u8 *succ = ch->output[ch->depth - 1]; @@ -725,7 +767,7 @@ static void dcp_ack_dfb_surface(struct apple_dcp *dcp) * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose * meaning is not known. */ -static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +static void __maybe_unused dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) { s[0x002] = 0x01; memcpy(s + 0x00b, "8a3b", 4); @@ -753,7 +795,14 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) */ static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_channel *ch = &dcp->ch_cb; + + *(u8 *)ch->output[ch->depth - 1] = out && !get_unaligned_le32(out); + dcp_ack(dcp, DCP_CONTEXT_CB); +#else dcp_ack_dfb_surface(dcp); +#endif } static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, @@ -773,12 +822,48 @@ static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, if (!req) return true; + +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + { + struct dcp_dfb_dimensions_h17p dim; + struct dcp_surface *surface = (void *)req->surface; + u32 format, allocation, bpe; + bool alpha_plane; + int ret = -EINVAL; + + memcpy(&dim, in, sizeof(dim)); + if (dcp->dfb_format_valid && dcp->dfb_compression_valid) + ret = apple_dcp_dfb_allocation(dim.width, dim.height, + dcp->dfb_format, dcp->dfb_compression, + dcp->dfb_region_size, &format, &allocation); + if (ret) { + dev_err(dcp->dev, "unsupported DFB layout %ux%u format=%#x compression=%u (%d)\n", + dim.width, dim.height, dcp->dfb_format, + dcp->dfb_compression, ret); + *(u8 *)out = false; + kfree(req); + return true; + } + + apple_dcp_dfb_format(dcp->dfb_format, dcp->dfb_compression, + &format, &bpe, &alpha_plane); + surface->is_premultiplied = 1; + surface->format = format; + surface->pix_size = dcp_surface_element_geometry(dcp) ? bpe : 1; + surface->pel_w = 1; + surface->pel_h = 1; + surface->buf_size = allocation; + surface->surface_id = 5; + surface->has_comp = 1; + surface->has_planes = 1; + } +#else dcp_fill_dfb_surface(dcp, req->surface); +#endif dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); kfree(req); - /* deferred ACK: the A472 completion reports success for us */ return false; } #endif @@ -837,6 +922,9 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr switch (prop->id) { #if !DCP_FW_DESKTOP_26(DCP_FW_VER) /* The desktop 26.x firmware publishes no panel level here. */ + #if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + case 20: +#else case IOMFB_PROPERTY_NITS_V13_3: /* * 13.x firmware publishes another property under the 14.7 ID, with @@ -845,6 +933,7 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr */ #if DCP_FW_VERSION(14, 7, 0) <= DCP_FW_VER case IOMFB_PROPERTY_NITS_V14_7: +#endif #endif { #if DCP_FW_H17(DCP_FW_VER) @@ -1279,7 +1368,8 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d * dma_map_resource() them instead fails and hands the firmware * DMA_MAPPING_ERROR as a device address. */ - dma_addr_t dva = rsrc->start; + dma_addr_t dva = dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + dcp->disp_firmware_base[req->index] : rsrc->start; #elif DCP_FW_DESKTOP_26(DCP_FW_VER) /* * The desktop 26.x firmware takes the register windows' @@ -1294,7 +1384,12 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d #endif return (struct DCP_FW_NAME(dcp_map_reg_resp)){ - .addr = rsrc->start, + .addr = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + dcp->disp_firmware_base[req->index], +#else + rsrc->start, +#endif .length = resource_size(rsrc), #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .dva = dva, @@ -1491,6 +1586,17 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dev_warn(dcp->dev, "TimingElements without valid modes!\n"); dcp_180hz_notify(dcp); dcp_firmware_metadata_timing(dcp); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp->peer_reserved) + for (unsigned int i = 0; i < dcp->nr_modes; i++) { + struct dcp_display_mode *m = &dcp->modes[i]; + + if ((u64)m->mode.clock * 1000 > dcp->dptxport[0].caps.total_rate) + dev_info(dcp->dev, "peer mode timing=%u color=%u " DRM_MODE_FMT "\n", + m->timing_mode_id, m->color_mode_id, + DRM_MODE_ARG(&m->mode)); + } +#endif } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; @@ -1636,6 +1742,27 @@ static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_dfb_dimensions_h17p dim; + + if (!out) + return; + memcpy(&dim, out, sizeof(dim)); + + if (dim.width > 16384 || dim.height > 16384 + ) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); + +#else struct dcp_dfb_dimensions_h17p dim; /* @@ -1656,12 +1783,25 @@ static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) dim.width, dim.height); dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); + +#endif } static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) { u32 zero = 0; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_dfb_dimensions_h17p dim; + + if (!out) + return; + memcpy(&dim, out, sizeof(dim)); + if (dim.width && dim.height) { + boot_2(dcp, out, cookie); + return; + } +#endif dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); } @@ -1669,6 +1809,12 @@ static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_layout_req_h17p req = {}; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + dcp->dfb_format_valid = *((u8 *)out + 0x28) & 1; + memcpy(&dcp->dfb_format, out, sizeof(dcp->dfb_format)); +#endif dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); } @@ -1676,6 +1822,12 @@ static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_info_h17p info = {}; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + dcp->dfb_compression_valid = *((u8 *)out + 4) & 1; + memcpy(&dcp->dfb_compression, out, sizeof(dcp->dfb_compression)); +#endif dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); } @@ -1683,6 +1835,16 @@ static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) { u64 zero = 0; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + if (!(*(u8 *)out & 1)) { + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); + return; + } +#endif dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); } #endif @@ -1863,6 +2025,7 @@ static void release_swap_cookie(struct kref *ref) */ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_SWAP(dcp); unsigned int i; @@ -1912,6 +2075,7 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) memmove(r + off - 2, r + off, n - off); memset(r + n - 2, 0, 2); } +#endif } #else static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } @@ -2177,7 +2341,12 @@ static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cooki static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, + .param = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + 15, +#else + IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { 0 }, #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .count = 3, @@ -2357,6 +2526,21 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct dcp_wait_cookie *poff_cookie; typeof(DCP_FW_SWAP(dcp)) *swap = &DCP_FW_SWAP(dcp); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + /* + * Firmware 27 has its own paired stop sequence. Its hardware profile + * does not use H17G's single-pipe methods, so stop an active pair + * before applying the guard for the single-pipe path below. + */ + ret = dcp_peer_stop_v27(dcp); + if (ret < 0) { + dev_err(dcp->dev, "paired power-off failed: %d\n", ret); + return; + } + if (ret > 0) + goto close_client; +#endif + #if DCP_FW_H17(DCP_FW_VER) /* H17P has no known safe pipe-off or surface-free transition. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) @@ -2371,7 +2555,6 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) if (ret > 0) goto peer_stopped; #endif - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -2413,6 +2596,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) #if DCP_FW_DESKTOP_26(DCP_FW_VER) peer_stopped: +#endif +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +close_client: #endif poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) @@ -2539,6 +2725,14 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + ret = dcp_peer_stop_v27(dcp); + if (ret < 0) { + dev_err(dcp->dev, "paired sleep failed: %d\n", ret); + return; + } +#endif + #if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; @@ -2652,6 +2846,12 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) action |= DCP_HOTPLUG_NOTIFY; } dcp_handle_hotplug_actions(dcp, action); + /* Fresh modes and positive firmware HPD, not lane-count negotiation, + * make IOMFB ready for the resume modeset. */ + if (*connected && dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp_is_typec_output(dcp)) + complete_all(&dcp->typec_display_ready); + } #if DCP_FW_DESKTOP_26(DCP_FW_VER) @@ -2923,6 +3123,10 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) dcp_swap_watchdog_arm(dcp); } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +#include "iomfb-peer-runtime-v27.inc" +#endif + static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; @@ -2938,6 +3142,10 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); return; } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_start_second(dcp, resp)) + return; +#endif DCP_FW_SWAP(dcp).swap.swap_id = resp->swap_id; if (dcp_present_retires(dcp)) dcp_arm_retained_framebuffers(dcp, resp->swap_id); @@ -3520,6 +3728,16 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out || get_unaligned_le32(out)) { + struct dcp_wait_cookie *wait = cookie; + + wait->status = -EOPNOTSUPP; + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + return; + } +#endif dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); } @@ -3528,7 +3746,12 @@ static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, void *cookie) { struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, + .param = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + 15, +#else + IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { min_vrr, /* minRR, 16.16 fixed-point Hz */ 0, /* mediaTargetRate */ @@ -3611,7 +3834,17 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, DCP_IOMFB_METHODS_H17P_EXTERNAL) { dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); - } else if (mode->vrr && + } else +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if ((ret = dcp_peer_modeset_v27(dcp, mode->timing_mode_id, + mode->color_mode_id, crtc_state->vrr_enabled)) != -EOPNOTSUPP) { + cookie->status = ret; + if (!ret) + crtc_state->color_mgmt_changed = true; + complete_set_digital_out_mode(dcp, NULL, cookie); + } else +#endif + if (mode->vrr && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp_set_adaptive_sync(dcp, @@ -3916,6 +4149,12 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru dcp_ctm_to_fw(mat.matrix, crtc_state->ctm ? crtc_state->ctm->data : NULL); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_color_v27(dcp, &mat)) { + dcp_peer_swap_error(dcp, "color matrix", -EIO); + return; + } +#endif #if DCP_FW_DESKTOP_26(DCP_FW_VER) dcp->desktop->gamma_pending = dcp_gamma_update(dcp, crtc_state); dcp->desktop->matrix = mat; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 1f98ace1d9a89..a99bfed67183f 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1938,7 +1938,10 @@ static ssize_t dcpext_start_store(struct device *dev, struct device_attribute *a if (atomic_cmpxchg(&dcp->external_requested, 0, 1)) return -EBUSY; /* Keep code and firmware callbacks alive after this one-shot request. */ - __module_get(THIS_MODULE); + if (!try_module_get(THIS_MODULE)) { + atomic_set(&dcp->external_requested, 0); + return -ENODEV; + } schedule_work(&dcp->external_work); return count; } @@ -2066,10 +2069,13 @@ bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) smp_load_acquire(&dcp->external_phase) != DCPEXT_IDLE) return false; if (!atomic_cmpxchg(&dcp->external_requested, 0, 1)) { + /* Firmware callbacks may outlive any later module removal. */ + if (!try_module_get(THIS_MODULE)) { + atomic_set(&dcp->external_requested, 0); + return false; + } WRITE_ONCE(dcp->external_phase, DCPEXT_STARTING); dev_info(dcp->dev, "display attached: starting the external display processor\n"); - /* Firmware callbacks may outlive any later module removal. */ - __module_get(THIS_MODULE); queue_work(system_freezable_wq, &dcp->external_work); } } @@ -2395,17 +2401,23 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp) v14->board->name); dcp_v14_link_init(&v14->link, dev, NULL); + /* The mailbox's callbacks need this module until reboot. */ + if (!try_module_get(THIS_MODULE)) + return dev_err_probe(dev, -ENODEV, + "%s display not started: the module is being removed\n", + v14->board->name); rtk = apple_rtkit_init(dev, v14, "mbox", 0, &dcp_v14_rtkit_ops); - if (IS_ERR(rtk)) + if (IS_ERR(rtk)) { + module_put(THIS_MODULE); return dev_err_probe(dev, PTR_ERR(rtk), "%s display not started: RTKit init failed\n", v14->board->name); + } /* From here nothing is freed and the module stays loaded. */ v14->rtk = rtk; v14->link.rtk = rtk; dcp->rtk = rtk; dcp_v14_panel_session = true; - __module_get(THIS_MODULE); ret = apple_rtkit_wake(rtk); for (n = 0; ret == -ETIME && n < DCP_V14_RTKIT_RETRIES; n++) diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c index de3cb84dba440..1f456dc17776a 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -1161,16 +1161,22 @@ int iomfb_v26_6_bind(struct apple_dcp *dcp) return dev_err_probe(dev, -EINVAL, "J613/25G83 same-boot clock observation required\n"); dcp_v26_link_init(&v26->link, dev, NULL); + /* The mailbox's callbacks need this module until reboot. */ + if (!try_module_get(THIS_MODULE)) + return dev_err_probe(dev, -ENODEV, + "%s display not started: the module is being removed\n", + v26->board->name); rtk = apple_rtkit_init(dev, v26, "mbox", 0, &dcp_v26_rtkit_ops); - if (IS_ERR(rtk)) + if (IS_ERR(rtk)) { + module_put(THIS_MODULE); return dev_err_probe(dev, PTR_ERR(rtk), "%s display not started: RTKit init failed\n", v26->board->name); + } /* From here nothing is freed and the module stays loaded. */ v26->rtk = rtk; v26->link.rtk = rtk; dcp_v26_session = true; - __module_get(THIS_MODULE); ret = apple_rtkit_wake(rtk); if (ret) { diff --git a/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c b/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c index 064eff4beff53..c2d0bee01e05f 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6_desktop.c @@ -52,46 +52,24 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { #include "iomfb_template.c" -struct dcp_dict_passthrough_req_v26_6 { - u8 key[0x40]; - u8 value[0x1000]; - u8 value_null; - u8 padding[3]; -} __packed; - -struct dcp_dict_passthrough_resp_v26_6 { - u8 value[0x1000]; - u8 ret; - u8 padding[3]; -} __packed; - -static_assert(sizeof(struct dcp_dict_passthrough_req_v26_6) == 0x1044); -static_assert(sizeof(struct dcp_dict_passthrough_resp_v26_6) == 0x1004); - /* - * D114 periodically publishes dictionary-valued statistics. The value is an - * in/out pointer, which the firmware deserializes again: the firmware's own - * client returns the serialized buffer unchanged and reports false. An - * all-zero reply is not a harmless false. + * D114 periodically publishes dictionary-valued statistics: the analytics + * event, answered by dcp_dict_passthrough_reply(). */ static bool trampoline_passthrough_dict_v26_6(struct apple_dcp *dcp, int tag, void *out, void *in) { - const struct dcp_dict_passthrough_req_v26_6 *req = in; - struct dcp_dict_passthrough_resp_v26_6 *resp = out; + const struct dcp_dict_passthrough_req *req = in; + struct dcp_dict_passthrough_resp *resp = out; trace_iomfb_callback(dcp, tag, "dcpep_cb_passthrough_dict"); dcp_trace_panel_property(dcp, tag, (const char *)req->key, strnlen((const char *)req->key, sizeof(req->key)), req->value_null ? NULL : req->value, sizeof(req->value)); - if (!req->value_null) { + if (!req->value_null) dcp_panel_edid_receive(dcp, req->value, sizeof(req->value)); - memcpy(resp->value, req->value, sizeof(resp->value)); - } - - resp->ret = false; - memset(resp->padding, 0, sizeof(resp->padding)); + dcp_dict_passthrough_reply(req, resp); return true; } @@ -101,7 +79,7 @@ static bool trampoline_panel_dictionary_v26_6(struct apple_dcp *dcp, int tag, void *out, void *in) { /* Service dictionary setters prefix the same record with a handle. */ - const struct dcp_dict_passthrough_req_v26_6 *req = tag == 413 ? in + 4 : in; + const struct dcp_dict_passthrough_req *req = tag == 413 ? in + 4 : in; const char *key = (const char *)req->key; struct dcp_display_metadata metadata; struct dcp_parse_ctx ctx; @@ -297,8 +275,8 @@ static const struct dcp_cb_min_v26_6_desktop cb_min[IOMFB_MAX_CB] = { [111] = CB_MIN(0, sizeof(u8)), [112] = CB_MIN(0, sizeof(u8)), [113] = CB_MIN(0, sizeof(u8)), - [114] = CB_MIN(sizeof(struct dcp_dict_passthrough_req_v26_6), - sizeof(struct dcp_dict_passthrough_resp_v26_6)), + [114] = CB_MIN(sizeof(struct dcp_dict_passthrough_req), + sizeof(struct dcp_dict_passthrough_resp)), [115] = CB_MIN(sizeof(struct dcpep_get_tiling_state_req), sizeof(struct dcpep_get_tiling_state_resp)), [116] = CB_MIN(sizeof(struct dcpep_get_tiling_state_req), sizeof(u32)), diff --git a/drivers/gpu/drm/apple/iomfb_v27.c b/drivers/gpu/drm/apple/iomfb_v27.c new file mode 100644 index 0000000000000..c6df81bf8abdf --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright The Asahi Linux Contributors */ +/* Copyright 2026 Aurora Silicon */ + +#include +#include + +#include "dcp-link.h" +#include "iomfb_v27.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" +#include "dcp-internal.h" + +void iomfb_serialize_present_v27(struct dcp_swap_submit_req_v27 *wire, + const struct dcp_swap_submit_req_h17p *request) +{ + memset(wire, 0, sizeof(*wire)); + memcpy(wire->swap.timestamp, request->swap.timestamp, + sizeof(wire->swap.timestamp)); + wire->swap.unk_38 = request->swap.unk_38; + /* The common geometry runs from swap_id through bg_color. */ + memcpy(&wire->swap.swap_id, &request->swap.swap_id, + offsetof(struct dcp_swap_v27, reserved_158) - + offsetof(struct dcp_swap_v27, swap_id)); + memcpy(wire->surf, request->surf, sizeof(wire->surf)); + memcpy(wire->surf_iova, request->surf_iova, + sizeof(*request) - + offsetof(struct dcp_swap_submit_req_h17p, surf_iova)); +} + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); + +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A027", dcpep_setup_video_limits), + IOMFB_METHOD("A136", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A137", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A389", dcpep_set_create_dfb), + IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A450", dcpep_set_parameter_dcp), + IOMFB_METHOD("A454", dcpep_create_default_fb), + IOMFB_METHOD("A037", dcpep_get_dfb_compression_info), + IOMFB_METHOD("A036", dcpep_get_dfb_info), + IOMFB_METHOD("A103", dcpep_get_dfb_layout), + IOMFB_METHOD("A453", dcpep_get_dfb_state), + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), + IOMFB_METHOD("A105", dcpep_commit_dfb_info), + IOMFB_METHOD("A035", dcpep_dfb_query), + IOMFB_METHOD("A384", dcpep_dfb_ready_query), + IOMFB_METHOD("A353", dcpep_pipe_query_353), + IOMFB_METHOD("A352", dcpep_pipe_set_352), + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), + IOMFB_METHOD("A033", dcpep_pipe_cfg_031), + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), + IOMFB_METHOD("A489", dcpep_pipe_query_478), + IOMFB_METHOD("A485", dcpep_pipe_query_474), + IOMFB_METHOD("A487", dcpep_pipe_set_476), + IOMFB_METHOD("A428", iomfbep_set_brightness_correction), + IOMFB_METHOD("A458", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A466", dcpep_first_client_open), + IOMFB_METHOD("A467", iomfbep_last_client_close), + IOMFB_METHOD("A475", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A478", dcpep_flush_supports_power), + IOMFB_METHOD("A479", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A484", dcpep_set_power_state), + IOMFB_METHOD("A483", dcpep_register_dfb_surface), +}; + +#define iomfb_modeset_h17p iomfb_modeset_v27 +#define iomfb_flush_h17p iomfb_flush_v27 +#define iomfb_poweron_h17p iomfb_poweron_v27 +#define iomfb_poweroff_h17p iomfb_poweroff_v27 +#define iomfb_sleep_h17p iomfb_sleep_v27 +#define iomfb_shutdown_h17p iomfb_shutdown_v27 +#define iomfb_present_backlight_h17p iomfb_present_backlight_v27 +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(27, 0, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + u64 scratch = dcp->disp_firmware_base[dcp->disp_bw_scratch_index] + + dcp->disp_bw_scratch_offset; + u64 clock_request = 0; + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u32 index = le32_to_cpu(request->clock_id); + u64 rate = 0; + + if (index < dcp->num_firmware_clocks) + rate = clk_get_rate(dcp->firmware_clocks[index].clk); + else + dev_err(dcp->dev, "DCP requested undeclared provider clock index %u\n", + index); + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + /* The opt-in peer allocator needs the firmware's tiling metadata to verify + * the VFTG relationship before implementing its framebuffer lifecycle. + * Keep both halves: the endpoint IDs are at the end of this payload. + */ + if (dcp->peer_reserved || dcp->peer_owner) { + dev_info(dcp->dev, "peer HPD connected=%llu null=%u head=%*ph\n", + le64_to_cpu(request->connected), request->tiled_display_null, + 40, request->tiled_display); + dev_info(dcp->dev, "peer HPD tail=%*ph\n", 36, + request->tiled_display + 40); + } + if (!(request->tiled_display_null & 1) && request->connected && + !(get_unaligned_le32(request->tiled_display + 0xc) & BIT(3))) + dcp_peer_display_ids_v27(dcp, + get_unaligned_le32(request->tiled_display + 0x3b), + get_unaligned_le32(request->tiled_display + 0x47)); + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +static bool iomfbep_cb_video_range_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + __le32 *resp = out; + + resp[0] = cpu_to_le32(0x435b0000); + resp[1] = 0; + return true; +} + +struct dcp_h17p_get_data_prop_req { + char obj[4]; + char key[0x40]; + __le32 length; + u8 data_null; + u8 length_null; + u8 padding[2]; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_req) == 0x4c); + +struct dcp_h17p_get_data_prop_resp { + u8 data[0xc00]; + __le32 length; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_resp) == 0xc04); + +static bool trampoline_get_data_prop_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_get_data_prop_req *req = in; + struct dcp_h17p_get_data_prop_resp *resp = out; + const struct property *prop; + u32 cap = min_t(u32, le32_to_cpu(req->length), sizeof(resp->data)); + + if (memcmp(req->obj, "VORP", sizeof(req->obj)) || + strnlen(req->key, sizeof(req->key)) == sizeof(req->key)) + return true; + + prop = of_find_property(dcp->dev->of_node, req->key, NULL); + if (!prop || prop->length < 0 || prop->length > cap) + return true; + memcpy(resp->data, prop->value, prop->length); + resp->length = cpu_to_le32(prop->length); + return true; +} + +struct dcp_allocate_buffer_req_v27 { + __le32 options; + __le64 size; + __le32 alignment; + u8 dva_null; + u8 capacity_null; + u8 padding[2]; +} __packed; + +struct dcp_allocate_buffer_resp_v27 { + __le64 dva; + __le64 capacity; + __le32 mem_desc_id; +} __packed; + +struct dcp_buffer_lifecycle_req_v27 { + __le32 mem_desc_id; + __le32 options; +} __packed; + +static_assert(sizeof(struct dcp_allocate_buffer_req_v27) == 0x14); +static_assert(sizeof(struct dcp_allocate_buffer_resp_v27) == 0x14); +static_assert(offsetof(struct dcp_allocate_buffer_resp_v27, mem_desc_id) == 0x10); +static_assert(sizeof(struct dcp_buffer_lifecycle_req_v27) == 8); + +static bool trampoline_allocate_buffer_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req_v27 *req = in; + struct dcp_allocate_buffer_resp_v27 *resp = out; + struct dcp_allocate_buffer_req allocation = { + .unk0 = le32_to_cpu(req->options), + .size = le64_to_cpu(req->size), + .unk2 = le32_to_cpu(req->alignment), + }; + struct dcp_allocate_buffer_resp buffer; + + if (!allocation.size || allocation.size > SIZE_MAX - SZ_16K || + allocation.unk2 > PAGE_SIZE || + (allocation.unk2 && !is_power_of_2(allocation.unk2))) + return true; + buffer = dcpep_cb_allocate_buffer(dcp, &allocation); + if (!req->dva_null) + resp->dva = cpu_to_le64(buffer.dva); + if (!req->capacity_null) + resp->capacity = cpu_to_le64(ALIGN(buffer.dva_size, SZ_16K)); + resp->mem_desc_id = cpu_to_le32(buffer.mem_desc_id); + return true; +} + +static bool trampoline_buffer_lifecycle_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_buffer_lifecycle_req_v27 *req = in; + __le32 *status = out; + u32 id = le32_to_cpu(req->mem_desc_id); + + if (!id || id >= DCP_MAX_MAPPINGS || + !test_bit(id, dcp->memdesc_map) || !dcp->memdesc[id].buf) { + *status = cpu_to_le32(14); + return true; + } + if (tag == 452) + dma_wmb(); + else + dma_rmb(); + *status = 0; + return true; +} + +static bool trampoline_ideal_screen_space_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const char *gamut; + u32 space = 1; + + if (!of_property_read_string(dcp->dev->of_node, + "apple,artwork-display-gamut", &gamut)) { + if (!strcmp(gamut, "P3")) + space = 2; + else if (!strcmp(gamut, "AdobeRGB")) + space = 3; + } + *(__le32 *)out = cpu_to_le32(space); + return true; +} + +static bool trampoline_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + struct dcpep_get_tiling_state_resp *resp = out; + + resp->value = req->value; + resp->ret = 0; + if (!(req->value_null & 1) && + dcp_peer_tiling_state_v27(dcp, false, req->event, req->param, + &resp->value)) { + resp->ret = 1; + return true; + } + if (!req->value_null && (req->param == 1 || req->param == 11) && + dcp->connector && !dcp->connector->base.has_tile) { + resp->value = 0; + resp->ret = 1; + } + return true; +} + +/* D115: same16-byte input as D114, boolean/padded4 output. */ +static bool trampoline_set_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + u32 value = req->value; + + *(__le32 *)out = cpu_to_le32(!(req->value_null & 1) && + dcp_peer_tiling_state_v27(dcp, true, req->event, req->param, &value)); + return true; +} + +#include "iomfb-display-identity-v27.inc" +#include "iomfb-external-counts-v27.inc" +#include "iomfb-boolean-properties-v27.inc" + +#define DCP_ANALYTICS_EVENT_SIZE 0x1044 +#define DCP_ANALYTICS_NAME_SIZE 0x40 +#define DCP_ANALYTICS_DICT_SIZE (DCP_ANALYTICS_EVENT_SIZE - DCP_ANALYTICS_NAME_SIZE) +#define DCP_ANALYTICS_FULL_DUMPS 16 + +/* + * D113: the firmware reports an analytics event, its name in the first 0x40 + * bytes and then the event's serialized payload dictionary. The dictionary is + * an in/out argument: the 0x1004-byte reply area is the dictionary the + * firmware reads back once the call returns. Hand it back unchanged. + * + * Replying with zeroes there made every DCP crash on its first analytics + * event, which the firmware sends about an hour after it starts; all DCPs + * start together at boot, so all of them went down within seconds. + */ +static bool dcpep_cb_core_analytics_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *event = in; + unsigned int len = DCP_ANALYTICS_EVENT_SIZE; + char name[128]; + + static_assert(DCP_ANALYTICS_DICT_SIZE == 0x1004); + + strscpy(name, event, min(sizeof(name), (size_t)DCP_ANALYTICS_NAME_SIZE)); + dev_info_ratelimited(dcp->dev, "CoreAnalytics event %s, reply area %16ph\n", + name, out); + if (dcp->analytics_dumps < DCP_ANALYTICS_FULL_DUMPS) { + dcp->analytics_dumps++; + while (len && !event[len - 1]) + len--; + print_hex_dump(KERN_INFO, "dcp analytics: ", DUMP_PREFIX_OFFSET, + 32, 1, event, len, true); + } + /* The same record and answer as desktop 26.6's analytics event. */ + dcp_dict_passthrough_reply(in, out); + return true; +} + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, + [102] = trampoline_nop, + [104] = trampoline_nop, + [106] = trampoline_nop, + [107] = trampoline_true, + [108] = trampoline_true, + [109] = trampoline_true, + [110] = trampoline_true, + [111] = trampoline_create_backlight_service, + [112] = trampoline_true, + [113] = dcpep_cb_core_analytics_v27, + [116] = trampoline_nop, + [117] = trampoline_zero, + [118] = trampoline_zero, + [119] = trampoline_zero, + [114] = trampoline_tiling_state_v27, + [115] = trampoline_set_tiling_state_v27, + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, + [122] = trampoline_false, + [123] = trampoline_zero, + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, + [129] = trampoline_allocate_bandwidth, + [130] = trampoline_update_external_counts_v27, + [131] = trampoline_get_external_counts_v27, + [132] = trampoline_apply_suppression_v27, + [133] = trampoline_display_identity_v27, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [204] = trampoline_ideal_screen_space_v27, + [208] = trampoline_nop, + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [400] = trampoline_get_data_prop_v27, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency_v27, + [411] = trampoline_map_reg, + [413] = trampoline_true, + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, + [451] = trampoline_allocate_buffer_v27, + [452] = trampoline_buffer_lifecycle_v27, + [453] = trampoline_buffer_lifecycle_v27, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [569] = trampoline_set_boolean_property_v27, + [572] = trampoline_zero, + [574] = iomfbep_cb_video_range_h17p, + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + [586] = iomfbep_cb_create_dfb_surface, + [587] = trampoline_nop, + [588] = trampoline_nop, + [589] = trampoline_swap_info_h17p, + + [592] = trampoline_nop, + [594] = trampoline_swap_complete, + [596] = trampoline_swap_complete_intent_gated, + [597] = trampoline_abort_swap_ap_gated, + [598] = trampoline_enable_backlight_message_ap_gated, + [599] = trampoline_nop, + [601] = trampoline_false, + [602] = trampoline_false, + [603] = trampoline_nop, +}; + +static const struct { + u16 input, output; +} callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4 }, + [1] = { 0x0, 0x4 }, + [2] = { 0x0, 0x0 }, + [3] = { 0x4, 0x14 }, + [6] = { 0x54, 0x50 }, + [300] = { 0x10, 0x0 }, + [100] = { 0x0, 0x0 }, + [101] = { 0x0, 0x4 }, + [102] = { 0x44, 0x0 }, + [104] = { 0x44, 0x0 }, + [106] = { 0x40, 0x0 }, + [107] = { 0x0, 0x4 }, + [108] = { 0x0, 0x4 }, + [109] = { 0x0, 0x4 }, + [110] = { 0x0, 0x4 }, + [111] = { 0x0, 0x4 }, + [112] = { 0x0, 0x4 }, + [113] = { 0x1044, 0x1004 }, + [116] = { 0xc, 0x0 }, + [117] = { 0x14, 0x4 }, + [118] = { 0x3e88, 0x4 }, + [119] = { 0x4, 0x4 }, + [114] = { 0x10, 0x8 }, + [115] = { 0x10, 0x4 }, + [120] = { 0x0, 0x4 }, + [121] = { 0x0, 0x4 }, + [122] = { 0x0, 0x4 }, + [123] = { 0x0, 0x4 }, + [124] = { 0x64, 0x24 }, + [126] = { 0x4, 0x4 }, + [127] = { 0x1008, 0x4 }, + [128] = { 0x40, 0x4 }, + [129] = { 0x1c, 0x14 }, + [130] = { 0x8, 0x0 }, + [131] = { 0x10, 0xc }, + [132] = { 0x4, 0x0 }, + [133] = { 0x0, 0x4 }, + [201] = { 0xc, 0x10 }, + [202] = { 0x18, 0x0 }, + [206] = { 0x0, 0x4 }, + [207] = { 0x0, 0x4 }, + [204] = { 0x0, 0x4 }, + [208] = { 0x4, 0x0 }, + [209] = { 0x0, 0x8 }, + [400] = { 0x4c, 0xc04 }, + [401] = { 0x50, 0xc }, + [404] = { 0x48, 0x0 }, + [406] = { 0x48, 0x0 }, + [408] = { 0x8, 0x8 }, + [411] = { 0x10, 0x1c }, + [413] = { 0x1048, 0x4 }, + [414] = { 0x50, 0x4 }, + [415] = { 0x4c, 0x4 }, + [451] = { 0x14, 0x14 }, + [452] = { 0x8, 0x4 }, + [453] = { 0x8, 0x4 }, + [454] = { 0x4, 0x4 }, + [552] = { 0x1044, 0x4 }, + [561] = { 0x1044, 0x4 }, + [563] = { 0x4c, 0x4 }, + [565] = { 0x48, 0x4 }, + [567] = { 0x80, 0x4 }, + [569] = { 0x44, 0x4 }, + [572] = { 0x4, 0x4 }, + [574] = { 0x4, 0x8 }, + [575] = { 0x58, 0x4c }, + [576] = { 0x4, 0x0 }, + [586] = { 0x8, 0x4 }, + [587] = { 0x0, 0x0 }, + [588] = { 0x18, 0x0 }, + [589] = { 0xe4, 0xe0 }, + [592] = { 0x0, 0x0 }, + [594] = { 0x730, 0x0 }, + [596] = { 0x14, 0x0 }, + [597] = { 0x4, 0x0 }, + [598] = { 0x4, 0x0 }, + [599] = { 0x4, 0x0 }, + [601] = { 0x0, 0x4 }, + [602] = { 0x0, 0x4 }, + [603] = { 0x0, 0x0 }, +}; + +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output) +{ + if (tag >= ARRAY_SIZE(callback_sizes) || !cb_handlers[tag]) + return false; + return input >= callback_sizes[tag].input && + output >= callback_sizes[tag].output; +} + +void iomfb_start_v27(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v27.h b/drivers/gpu/drm/apple/iomfb_v27.h new file mode 100644 index 0000000000000..ac1ded5eb047c --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright The Asahi Linux Contributors */ +/* Copyright 2026 Aurora Silicon */ + +#ifndef APPLE_IOMFB_V27_H +#define APPLE_IOMFB_V27_H + +#include "iomfb_h17p.h" + +struct dcp_swap_v27 { + + u64 timestamp[7]; + u64 unk_38; + + u64 debug_info[2]; + + u64 event_wait[SWAP_SURFACES]; + u64 event_signal_on_glass; + u64 event_signal[SWAP_SURFACES]; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 bg_color; + u8 reserved_158[0x1fc]; + + u8 brightness_update; + u8 reserved_355[9]; + u64 brightness_nits; + u8 reserved_366[0x37a]; +} __packed; + +struct dcp_surface_v27 { + struct dcp_surface base; + u8 padding[47]; +} __packed; + +struct dcp_swap_submit_req_v27 { + struct dcp_swap_v27 swap; + struct dcp_surface_v27 surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; + u64 unk_u64_a[SWAP_SURFACES]; + + u8 unkbool; + u64 unkdouble; + u64 unkU64; + u8 unkbool2; + u32 clear; + u32 unkU32Ptr; + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; + u8 unkoutbool_null; + u8 unknown_pointer_null; + u8 unknown_output_null; + u8 padding[2]; +} __packed; + +struct dcp_swap_submit_resp_v27 { + u8 unkoutbool; + u32 unkU32out; + u32 ret; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); +static_assert(sizeof(struct dcp_surface_v27) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_v27) == 0xff4); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf) == 0x6e0); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf_iova) == 0xf90); +static_assert(offsetof(struct dcp_swap_submit_req_v27, swap_null) == 0xfea); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); + +void iomfb_serialize_present_v27(struct dcp_swap_submit_req_v27 *wire, + const struct dcp_swap_submit_req_h17p *request); +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output); +void iomfb_firmware_state_cleanup_v27(void *data); +int iomfb_modeset_v27(struct apple_dcp *dcp, struct drm_crtc_state *state); +void iomfb_flush_v27(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit); +void iomfb_start_v27(struct apple_dcp *dcp); +void iomfb_shutdown_v27(struct apple_dcp *dcp); +void iomfb_poweron_v27(struct apple_dcp *dcp); +void iomfb_poweroff_v27(struct apple_dcp *dcp); +void iomfb_sleep_v27(struct apple_dcp *dcp); +void iomfb_present_backlight_v27(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 6e48211d69f53..ca95e4f71eead 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -15,6 +15,7 @@ #include "parser.h" #include "dcp-refresh.h" +#include "dcp-peer-v27.h" #include "trace.h" #define DCP_PARSE_HEADER 0xd3 @@ -1761,12 +1762,58 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada return 0; } +int parse_dptx_caps(struct dcp_parse_ctx *handle, struct dcp_dptx_caps *caps) +{ + struct iterator it; + struct dcp_dptx_caps parsed = {}; + int ret = 0; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; ++it.idx) { + u32 *field = NULL; + char *key = parse_string(it.handle); + s64 value; + + if (IS_ERR(key)) + return PTR_ERR(key); + if (!strcmp(key, "MaxPixelWidth")) + field = &parsed.width; + else if (!strcmp(key, "MaxTotalPixelRate")) + field = &parsed.total_rate; + else if (!strcmp(key, "MaxActivePixelRate")) + field = &parsed.active_rate; + kfree(key); + if (!field) { + ret = skip(it.handle); + if (ret) + return ret; + if (it.handle->pos > it.handle->len) + return -EINVAL; + continue; + } + if (*field) + return -EINVAL; + ret = parse_int_bound(it.handle, &value, 1, U32_MAX); + if (ret) + return ret; + *field = value; + } + if (!parsed.width || !parsed.total_rate || !parsed.active_rate) + return -ENOENT; + *caps = parsed; + return 0; +} + int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit, bool h17p_keys) + const char **class, s64 *unit, bool h17p_keys, + bool epic_unit_first) { int ret = 0; struct iterator it; bool parsed_unit = false; + bool explicit_unit = false; bool parsed_name = false; bool parsed_class = false; @@ -1793,7 +1840,9 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, * service to ask and idles the panel, so no vblank and no swap * ever completes. Only H17P callers accept them. If a * dictionary carries both spellings, the later value wins and - * the earlier string is freed. + * the earlier string is freed, except that with + * @epic_unit_first (firmware 27) EPICUnit is the unit in + * either order. */ if (!strcmp(key, "EPICName") || (h17p_keys && !strcmp(key, "compartment"))) { @@ -1814,10 +1863,13 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, else parsed_class = true; } else if (!strcmp(key, "EPICUnit") || - (h17p_keys && !strcmp(key, "interface-id"))) { + (h17p_keys && !explicit_unit && !strcmp(key, "interface-id"))) { ret = parse_int(it.handle, unit); - if (!ret) + if (!ret) { parsed_unit = true; + explicit_unit |= epic_unit_first && + !strcmp(key, "EPICUnit"); + } } else { ret = skip(it.handle); } diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 3fa384ecf68c4..45aaa1739ea93 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -17,6 +17,7 @@ typedef void (*dcp_identity_key_fn)(void *data, const char *key, u32 key_len, int parse_identity_keys(const void *blob, size_t len, dcp_identity_key_fn emit, void *data); struct apple_connector; struct apple_dcp_typec_route; +struct dcp_dptx_caps; struct dcp_parse_ctx { struct apple_dcp *dcp; @@ -224,7 +225,9 @@ int parse_panel_product_metadata(struct dcp_parse_ctx *handle, int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit, bool h17p_keys); + const char **class, s64 *unit, bool h17p_keys, + bool epic_unit_first); +int parse_dptx_caps(struct dcp_parse_ctx *handle, struct dcp_dptx_caps *caps); struct dcp_sound_format_mask { u64 formats; /* SNDRV_PCM_FMTBIT_* */ diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c index 7fc03175a2324..ed3f99f6a9ddd 100644 --- a/drivers/gpu/drm/apple/parser_test.c +++ b/drivers/gpu/drm/apple/parser_test.c @@ -56,13 +56,20 @@ static void blob_int(struct blob *b, s64 value) b->len += 8; } -static int parse_service(struct kunit *test, const struct blob *b, bool h17p, - const char **name, const char **class, s64 *unit) +static int parse_service_v27(struct kunit *test, const struct blob *b, bool h17p, + bool v27, const char **name, const char **class, + s64 *unit) { struct dcp_parse_ctx ctx; KUNIT_ASSERT_EQ(test, parse(b->data, b->len, &ctx), 0); - return parse_epic_service_init(&ctx, name, class, unit, h17p); + return parse_epic_service_init(&ctx, name, class, unit, h17p, v27); +} + +static int parse_service(struct kunit *test, const struct blob *b, bool h17p, + const char **name, const char **class, s64 *unit) +{ + return parse_service_v27(test, b, h17p, false, name, class, unit); } static void parser_service_init_test(struct kunit *test) @@ -118,6 +125,36 @@ static void parser_service_init_h17p_keys_test(struct kunit *test) kfree(class); } +/* Firmware 27 takes EPICUnit over interface-id; H17P keeps the later key. */ +static void parser_service_init_v27_unit_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit = -1; + + blob_init(&b); + blob_tag(&b, T_DICT, 4); + blob_string(&b, "EPICName"); + blob_string(&b, "dcpdptx-port-epic"); + blob_string(&b, "EPICProviderClass"); + blob_string(&b, "AppleDCPDPTXRemotePort"); + blob_string(&b, "EPICUnit"); + blob_int(&b, 1); + blob_string(&b, "interface-id"); + blob_int(&b, 3); + + KUNIT_ASSERT_EQ(test, parse_service_v27(test, &b, true, true, &name, &class, + &unit), 0); + KUNIT_EXPECT_EQ(test, unit, 1); + kfree(name); + kfree(class); + + KUNIT_ASSERT_EQ(test, parse_service(test, &b, true, &name, &class, &unit), 0); + KUNIT_EXPECT_EQ(test, unit, 3); + kfree(name); + kfree(class); +} + static void parser_service_init_truncated_test(struct kunit *test) { const char *name, *class; @@ -271,6 +308,7 @@ static void parser_property_complete_test(struct kunit *test) static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_service_init_test), KUNIT_CASE(parser_service_init_h17p_keys_test), + KUNIT_CASE(parser_service_init_v27_unit_test), KUNIT_CASE(parser_service_init_truncated_test), KUNIT_CASE(parser_nesting_test), KUNIT_CASE(parser_bad_header_test), diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index a8c37e29ed65a..bf25e66a9322a 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -145,8 +145,16 @@ static int apple_get_ix_layout(u32 width, u32 height, return 0; } +static bool apple_plane_element_geometry(struct drm_crtc *crtc) +{ + struct apple_dcp *dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + return dcp_surface_element_geometry(dcp); +} + /* Use exactly the same geometry for admission and surface construction. */ static int apple_plane_get_fb_layout(struct drm_framebuffer *fb, + bool element_geometry, struct apple_plane_layout *planes, u32 *buf_size) { @@ -195,7 +203,9 @@ static int apple_plane_get_fb_layout(struct drm_framebuffer *fb, return -EINVAL; p->image.tile_w = drm_format_info_block_width(fmt, i); p->image.tile_h = drm_format_info_block_height(fmt, i); - p->tile_size = p->image.tile_w * p->image.tile_h; + /* tile_size is bytes per element, not pixels. */ + p->tile_size = element_geometry ? fmt->char_per_block[i] : + p->image.tile_w * p->image.tile_h; p->stride = fb->pitches[i]; break; default: @@ -251,7 +261,9 @@ static int apple_plane_atomic_check(struct drm_plane *plane, if (ret < 0) return ret; - ret = apple_plane_get_fb_layout(new_plane_state->fb, layouts, &buf_size); + ret = apple_plane_get_fb_layout(new_plane_state->fb, + apple_plane_element_geometry(new_plane_state->crtc), + layouts, &buf_size); if (ret) return ret; @@ -417,8 +429,9 @@ static void apple_plane_atomic_update(struct drm_plane *plane, struct drm_framebuffer *fb = base->fb; const struct drm_format_info *fmt = fb->format; + bool element_geometry = apple_plane_element_geometry(base->crtc); - if (WARN_ON_ONCE(apple_plane_get_fb_layout(fb, layouts, &buf_size))) { + if (WARN_ON_ONCE(apple_plane_get_fb_layout(fb, element_geometry, layouts, &buf_size))) { memset(&new_state->surf, 0, sizeof(new_state->surf)); return; } @@ -450,7 +463,7 @@ static void apple_plane_atomic_update(struct drm_plane *plane, .surface_id = plane->base.id, .buf_size = buf_size, - /* Only used for compressed or multiplanar surfaces */ + /* Only used for compressed or multiplanar surfaces before firmware 27 */ .pix_size = 1, .pel_w = 1, .pel_h = 1, @@ -459,6 +472,16 @@ static void apple_plane_atomic_update(struct drm_plane *plane, .has_compr_info = 1, }; + if (element_geometry) { + /* Bytes per element, and the element's width and height in pixels */ + new_state->surf.pix_size = fmt->char_per_block[0]; + new_state->surf.pel_w = drm_format_info_block_width(fmt, 0); + new_state->surf.pel_h = drm_format_info_block_height(fmt, 0); + /* Present only when compression_info describes a compressed plane */ + new_state->surf.has_compr_info = + fb->modifier == DRM_FORMAT_MOD_APPLE_INTERCHANGE; + } + /* Populate plane information for planar formats */ struct dcp_surface *surf = &new_state->surf; for (int i = 0; fb->format->num_planes && i < fb->format->num_planes; i++) { diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 38f667e114514..cbd243c4c12bd 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -108,7 +108,7 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); have_mnits: - if ((dcp->fw_compat == DCP_FIRMWARE_H17P && + if (((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { if (!dcp_has_panel(dcp)) diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index 8851ee57c2daf..8cf36e39d3f24 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -61,6 +61,7 @@ enum dcp_firmware_version { * (iomfb_v26_6_desktop.c); not the J613 DCPLink DCP_FIRMWARE_V_26_6. */ DCP_FIRMWARE_V_26_6_DESKTOP, + DCP_FIRMWARE_V_27_0, }; #endif /*__APPLE_VERSION_UTILS_H__*/ diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index db13d06336b3c..e50a4d768369f 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,9 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o aurora-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o + +aurora-y += retain.o diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 1a5cf1ba9a7c1..3713e5e3e7cf5 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -37,7 +37,9 @@ mod mmu; mod object; mod pgtable; mod queue; +mod queue_share; mod regs; +mod retain; mod slotalloc; mod stats; mod stats_raw; @@ -96,6 +98,8 @@ impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { // SAFETY: Module init runs once, before either adapter is registered below. unsafe { binding::init() }; + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { queue_share::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, @@ -122,6 +126,22 @@ module! { license: "Dual MIT/GPL", alias: ["asahi"], params: { + m5_mesh_storage_chunks: u32 { + default: 0x200, + description: "M5 mesh rendering buffer bound to API task (object-stage) render kicks only, in 64 KiB chunks; 0 disables the binding and the task feature", + }, + m5_thermal_governor: u32 { + default: 0, + description: "M5 thermal flags: bit 0 enables the host GPU power budget controller, bit 2 lets the controller lower the CPU frequency ceiling while the GPU is budget-bound, bit 3 runs the fans at their maximum while it governs; default 0 preserves the existing policy", + }, + m5_thermal_target_mc: u32 { + default: 90000, + description: "M5 GPU power budget governor target for the SoC hotspot temperature computed by the SMC, in milli-degrees C (70000..104000)", + }, + m5_depth_msaa_compression: u32 { + default: 1, + description: "Enable single-layer D32 compressed 2x/4x ZLS admission", + }, debug_flags: u64 { default: 0, // permissions: 0o644, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 93c52f1de9968..a5f8802ce0c56 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -141,6 +141,8 @@ impl drm::driver::Driver for AsahiDrmDriver { pub(crate) enum ProbeConfig { Legacy(&'static hw::HwConfig), G17(&'static g17::Config), + /// M5 Pro, Max and Ultra: the ID registers select the configuration. + G17M5, M3(crate::m3::Chip), /// The M5 (T8142) backend, refused unless it is built. M5, @@ -171,6 +173,14 @@ macro_rules! asahi_of_device_table { of::DeviceId::new(c"apple,agx-t8142"), ProbeConfig::M5 ), + ( + of::DeviceId::new(c"gpu,t6050"), + ProbeConfig::G17M5 + ), + ( + of::DeviceId::new(c"apple,agx-g17s"), + ProbeConfig::G17M5 + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -257,7 +267,10 @@ impl platform::Driver for AsahiPlatformDriver { let cfg = match info.ok_or(ENODEV)? { ProbeConfig::Legacy(cfg) => *cfg, - ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + ProbeConfig::G17(cfg) => { + return AsahiPlatformDriver::probe_g17(pdev, regs::G17Resources::new(pdev)?, cfg) + } + ProbeConfig::G17M5 => return AsahiPlatformDriver::probe_g17_m5(pdev), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), #[cfg(CONFIG_DRM_ASAHI_M5)] ProbeConfig::M5 => return AsahiPlatformDriver::probe_m5(pdev), @@ -390,14 +403,32 @@ impl AsahiPlatformDriver { }) } + fn probe_g17_m5<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + // Every T6050 board binds gpu,t6050. Let the ID registers choose the configuration. + let res = regs::G17Resources::new(pdev)?; + let id = res.get_g17_id()?; + let cfg = hw::t6050::config(&id).ok_or_else(|| { + dev_err!(pdev.as_ref(), "No M5 configuration for GPU variant {:#x} with {} dies\n", + id.variant, id.num_dies); + ENODEV + })?; + if let g17::FirmwareProfile::M5NoBareMetalTables = cfg.profile { + dev_err!(pdev.as_ref(), "M5 Pro bare-metal GPU tables not yet available\n"); + return Err(ENODEV); + } + Self::probe_g17(pdev, res, cfg) + } + fn probe_g17<'bound>( pdev: &'bound platform::Device>, + res: regs::G17Resources, cfg: &'static g17::Config, ) -> Result> { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; - let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; // SAFETY: Every adapter calls close_binding in unbind before devres; // probe errors drop this local owner while the bound callback is live. diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 244dca39e41ac..72acee6674ab6 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -20,6 +20,7 @@ use crate::{ memacct, mmu, queue, + queue_share, util::{ align, align_down, @@ -30,7 +31,7 @@ use crate::{ }, // }; use core::mem::MaybeUninit; -use core::ops::Deref; +use core::ops::{Deref, DerefMut}; use core::ops::Range; use core::ptr::addr_of_mut; use kernel::bindings; @@ -58,6 +59,69 @@ use kernel::{ xarray, // }; +// The M5 runtime's UAPI numbers. The header's values are Mesa's; the M5 queue and +// GET_PARAMS paths use these names. + +/// DRM_ASAHI_BIND_SPARSE_TILE: with SINGLE_PAGE, fill the range +/// from kernel sparse tiles (read-only: global zero tile; writable: per-VM +/// sink) instead of the GEM's first page. The GEM only anchors the GPUVA. +pub(crate) const DRM_ASAHI_BIND_SPARSE_TILE: u32 = uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SPARSE_TILE; +/// DRM_ASAHI_FEATURE_SPARSE_TILES: BIND_SPARSE_TILE accepted. +pub(crate) const DRM_ASAHI_FEATURE_SPARSE_TILES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SPARSE_TILES as u64; +pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_MATRIX: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_MATRIX as u64; +/// DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: render programs accept +/// DRM_ASAHI_RENDER_PROGRAM_TASK and such task (object-stage) kicks get the +/// TA mesh rendering buffer, which backs the object stage's payload address. +/// Reported only while m5_mesh_storage_chunks != 0. +pub(crate) const DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_TASK_MESH_STORAGE as u64; +pub(crate) const DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES as u64; +/// D32 2x/4x ZLS admission; no compressed stencil or resolve bits. +pub(crate) const DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION as u64; +/// DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: single-layer +/// 2x/4x ZLS admission in the compressed Z/S form, stencil included. +pub(crate) const DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION as u64; +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_SHADER_WRITES; +/// Renders may carry separate tiling prefixes of both subqueues +/// (DRM_ASAHI_RENDER_STAGE_PREFIXES), as the complete struct drm_asahi_cmd_render_m5. +pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: u64 = + uapi::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 as u64; +/// The tiling half waits only on the render and compute prefixes of the command's stage-prefix +/// fields, its fragment half on the whole ones. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_STAGE_PREFIXES: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; +/// Renders may declare that they write whole attachments only +/// (DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS). +pub(crate) const DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS as u64; +/// The render area covers every attachment's full extent, so its tile stores +/// write no pixel outside it: an overlap with another such render without an +/// ordering dependency is the application's own race. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS; +/// DRM_ASAHI_RENDER_PROGRAM_TASK, only with DRM_ASAHI_RENDER_PROGRAM_OBJECT: the object +/// program is an API task stage. +pub(crate) const DRM_ASAHI_RENDER_PROGRAM_TASK: u32 = uapi::DRM_ASAHI_RENDER_PROGRAM_TASK; +/// Bit 16, as Mesa defines it. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_COMPUTE_USES_MATRIX: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_USES_MATRIX as u32; +/// The compute command neither copies nor resets queries, so it cannot observe +/// timestamps the host publishes after earlier commands complete +/// (DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS). Accepted; DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS +/// is not reported: with it, top-of-pipe timestamps are read out of order. +pub(crate) const DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as u32; + const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; @@ -263,6 +327,27 @@ pub(crate) enum Object { TimestampBuffer(Arc), } +/// A queue in a file's table. Its count in the creating process's share of the device's +/// queues is given back once the queue itself is gone. +pub(crate) struct TableQueue { + queue: KBox, + _share: Option, +} + +impl Deref for TableQueue { + type Target = dyn queue::Queue; + + fn deref(&self) -> &Self::Target { + &*self.queue + } +} + +impl DerefMut for TableQueue { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut *self.queue + } +} + /// State associated with a client. // #[pin_data] pub(crate) struct File { @@ -272,7 +357,7 @@ pub(crate) struct File { // #[pin] vms: xarray::XArray>, // #[pin] - queues: xarray::XArray>>>, + queues: xarray::XArray>>, // #[pin] objects: xarray::XArray>, /// GPU memory this client created and holds. @@ -356,7 +441,7 @@ impl File { let raw_queues = addr_of_mut!((*slot).queues); pin_init::raw_try_init( raw_queues, - xarray::XArray::>>>::new( + xarray::XArray::>>::new( xarray::AllocKind::Alloc1, ), )?; @@ -383,7 +468,7 @@ impl File { } #[allow(clippy::type_complexity)] - fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { + fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>> { // SAFETY: Structural pinned projection for queues. // We never move out of this field. unsafe { self.map_unchecked(|s| &s.queues) } @@ -569,7 +654,9 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + let vm = gpu + .new_vm_for_file(kernel_range.clone(), file_id)? + .with_memory_client(file.inner().mem.clone()); // G16 registers a retained backend owner before returning this VM. Any later // allocation can fail before file::Vm exists to run its normal close path. let failed_vm = ScopeGuard::new_with_data( @@ -925,11 +1012,13 @@ impl File { if op.flags & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE - | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE) + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE + | DRM_ASAHI_BIND_SPARSE_TILE) != 0 { return Err(EINVAL); } + let sparse_tile = op.flags & DRM_ASAHI_BIND_SPARSE_TILE != 0; let read = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; let write = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; let prot = match (read, write) { @@ -956,6 +1045,7 @@ impl File { op.offset, prot, single_page, + sparse_tile, )?)?; } vm.reserve_deferred_user_unmaps(unmaps)?; @@ -1358,6 +1448,15 @@ impl File { return Err(EACCES); } + // The queue counts against the creating process's share of the device's independent + // queues, whichever of its files it is created on, so no process can take every + // execution context and leave the others, the compositor included, unable to create + // one (see [`queue_share`]); a refusal fails with EBUSY. + let share = reg_data + .gpu + .queue_limits() + .map(|limits| queue_share::Share::take_current(limits.max_queues)) + .transpose()?; let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; let vms_xa = file.inner().vms(); @@ -1383,7 +1482,13 @@ impl File { queue.reserve_render_scs(data.render_scs_layout)?; data.queue_id = resv.index().try_into()?; - resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; + resv.fill(Arc::pin_init( + new_mutex!(TableQueue { + queue, + _share: share, + }), + GFP_KERNEL, + )?)?; Ok(0) } @@ -1427,7 +1532,7 @@ impl File { } // Upgrade to Arc to drop the XArray lock early - let queue: Arc>> = file + let queue: Arc> = file .inner() .queues() .lock() diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index 34c65a0d8a097..c9a0ef11d8b16 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -2,10 +2,15 @@ //! Exact GPU firmware identity. //! -//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its two entropy records +//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its entropy records //! (`GKTS` and `ECAP`, each a tag, the length 8 and the bytes) zeroed, plus the TEXT and DATA -//! sizes and the records' offsets. An unknown build is refused with its -//! measured identity, so nothing is written to a firmware the driver was not written for. +//! sizes and the records' offsets. One TEXT region can hold several firmware roles back to back +//! (two GPU roles' TEXT loaded one after the other), each with its own pair of records. A build +//! can also name other ranges the boot loader fills (two sections it patches with the boot +//! arguments and the image header's load fields, which can change from boot to boot); they are +//! zeroed too, so the digest is that of the firmware's TEXT with the same ranges zeroed. An +//! unknown build is refused with its measured identity, so nothing is written to a firmware the +//! driver was not written for. use core::ops::Range; use kernel::{bindings, device, fmt, prelude::*}; @@ -15,9 +20,11 @@ pub(crate) struct FirmwareBuild { pub(crate) name: &'static str, pub(crate) text_size: u64, pub(crate) data_size: u64, - /// Offsets of the GKTS and ECAP entropy records in TEXT; their 8 entropy bytes are zeroed - /// before hashing. - pub(crate) entropy: [u32; 2], + /// Offsets of each role's GKTS and ECAP entropy records in TEXT, in TEXT order; their 8 + /// entropy bytes are zeroed before hashing. + pub(crate) entropy: &'static [[u32; 2]], + /// Other (offset, length) ranges of TEXT the boot loader fills, zeroed before hashing. + pub(crate) boot_filled: &'static [(u32, u32)], pub(crate) text_sha256: [u8; 32], } @@ -65,42 +72,111 @@ pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { Ok(copy?) } -/// Entropy record tags, in TEXT order. +/// Entropy record tags, in TEXT order within a role. const ENTROPY_TAGS: [[u8; 4]; 2] = [*b"GKTS", *b"ECAP"]; -/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` zeroed, or `None` if a -/// record (tag, then the length 8) is not there. `image` is left as it was. -fn digest_at(image: &mut [u8], at: [u32; 2]) -> Option<[u8; 32]> { - let mut saved = [[0u8; 8]; 2]; - for ((off, tag), save) in at.into_iter().zip(ENTROPY_TAGS).zip(saved.iter_mut()) { - let off = off as usize; - if image.get(off..off + 4) != Some(tag.as_slice()) - || image.get(off + 4..off + 8) != Some(8u32.to_le_bytes().as_slice()) - { +/// Firmware roles one TEXT region can hold. +const MAX_ROLES: usize = 2; + +/// The record (tag, then the length 8) that holds a role's entropy of `tag`. +fn record(tag: [u8; 4]) -> [u8; 8] { + let mut record = [0u8; 8]; + record[..4].copy_from_slice(&tag); + record[4..].copy_from_slice(&8u32.to_le_bytes()); + record +} + +/// Whether `image` holds a whole entropy record of `tag` at `off`: the tag, the length 8 and the +/// 8 entropy bytes, all inside `image`. +fn record_at(image: &[u8], off: usize, tag: [u8; 4]) -> bool { + off.checked_add(16) + .and_then(|end| image.get(off..end)) + .is_some_and(|whole| whole[..8] == record(tag)) +} + +/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` (one GKTS/ECAP pair +/// per role) and the `filled` ranges zeroed, or `None` if a record or range is not there. +/// `image` is left as it was. +fn digest_at(image: &mut [u8], at: &[[u32; 2]], filled: &[(u32, u32)]) -> Option<[u8; 32]> { + if at.is_empty() || at.len() > MAX_ROLES { + return None; + } + if !filled.is_empty() { + for (off, tag) in at.iter().flat_map(|pair| pair.iter().zip(ENTROPY_TAGS)) { + if !record_at(image, *off as usize, tag) { + return None; + } + } + let mut copy = KVVec::from_elem(0u8, image.len(), GFP_KERNEL).ok()?; + copy.copy_from_slice(image); + for off in at.iter().flatten() { + copy[*off as usize + 8..*off as usize + 16].fill(0); + } + for &(off, len) in filled { + let off = off as usize; + copy.get_mut(off..off.checked_add(len as usize)?)?.fill(0); + } + return Some(sha256(©)); + } + let records = || { + at.iter() + .flat_map(|pair| pair.iter().map(|&off| off as usize).zip(ENTROPY_TAGS)) + }; + let mut saved = [[0u8; 8]; 2 * MAX_ROLES]; + for ((off, tag), save) in records().zip(saved.iter_mut()) { + if !record_at(image, off, tag) { return None; } - save.copy_from_slice(image.get(off + 8..off + 16)?); + save.copy_from_slice(&image[off + 8..off + 16]); } - for off in at { - image[off as usize + 8..off as usize + 16].fill(0); + for (off, _) in records() { + image[off + 8..off + 16].fill(0); } let digest = sha256(image); - for (off, save) in at.into_iter().zip(saved) { - image[off as usize + 8..off as usize + 16].copy_from_slice(&save); + for ((off, _), save) in records().zip(saved) { + image[off + 8..off + 16].copy_from_slice(&save); } Some(digest) } -/// The first record of each tag in `image`, to report an unknown build. -fn find_entropy(image: &[u8]) -> Option<[u32; 2]> { - let mut found = [0u32; 2]; - for (slot, tag) in found.iter_mut().zip(ENTROPY_TAGS) { - let mut record = [0u8; 8]; - record[..4].copy_from_slice(&tag); - record[4..].copy_from_slice(&8u32.to_le_bytes()); - *slot = u32::try_from(image.windows(8).position(|w| w == record)?).ok()?; +/// The entropy records of each role found in `image`, to report an unknown build: each GKTS +/// record with the first ECAP record after it, at most [`MAX_ROLES`] pairs. +struct Records { + at: [[u32; 2]; MAX_ROLES], + roles: usize, +} + +impl Records { + fn find(image: &[u8]) -> Option { + let [gkts, ecap] = ENTROPY_TAGS.map(record); + let mut found = Records { + at: [[0; 2]; MAX_ROLES], + roles: 0, + }; + let mut from = 0; + while found.roles < MAX_ROLES { + let Some(g) = image.get(from..)?.windows(8).position(|w| w == gkts) else { + break; + }; + let g = from + g; + let Some(e) = image.get(g + 8..)?.windows(8).position(|w| w == ecap) else { + break; + }; + let e = g + 8 + e; + // A record whose entropy bytes run past TEXT is no record ([`record_at`]). + if !record_at(image, g, ENTROPY_TAGS[0]) || !record_at(image, e, ENTROPY_TAGS[1]) { + break; + } + found.at[found.roles] = [u32::try_from(g).ok()?, u32::try_from(e).ok()?]; + found.roles += 1; + from = e + 8; + } + (found.roles > 0).then_some(found) + } + + fn as_slice(&self) -> &[[u32; 2]] { + &self.at[..self.roles] } - Some(found) } /// Finds an exact identity, measuring each candidate at its own entropy offsets. @@ -108,54 +184,128 @@ fn lookup( builds: &'static [FirmwareBuild], text_size: u64, data_size: u64, - mut digest: impl FnMut([u32; 2]) -> Option<[u8; 32]>, + mut digest: impl FnMut(&FirmwareBuild) -> Option<[u8; 32]>, ) -> Option<&'static FirmwareBuild> { builds.iter().find(|build| { build.text_size == text_size && build.data_size == data_size - && digest(build.entropy) == Some(build.text_sha256) + && digest(build) == Some(build.text_sha256) + }) +} + +/// A private, stable copy of a loaded TEXT, with the size of its DATA. +pub(crate) struct Measured { + text: KVVec, + data_size: u64, +} + +/// Copies the TEXT loaded at `text` (DATA `data_size` bytes) twice, before any firmware-visible +/// write, and keeps the copy if both agree (`EAGAIN` if the TEXT changed meanwhile). +pub(crate) fn measure(text: Range, data_size: u64) -> Result { + let size = usize::try_from(text.end.checked_sub(text.start).ok_or(EINVAL)?)?; + let copy = copy_physical(text.start, size)?; + let again = copy_physical(text.start, size)?; + if copy.as_slice() != again.as_slice() { + return Err(EAGAIN); + } + Ok(Measured { + text: copy, + data_size, }) } +impl Measured { + /// The build among `builds` this TEXT is exactly, if any. + pub(crate) fn find( + &mut self, + builds: &'static [FirmwareBuild], + ) -> Option<&'static FirmwareBuild> { + let text_size = self.text.len() as u64; + let text = &mut self.text; + lookup(builds, text_size, self.data_size, |build| { + digest_at(&mut text[..], build.entropy, build.boot_filled) + }) + } + + /// The measured identity, to report a build that is not admitted: normalized as the first + /// of `builds` with this TEXT size would be, else with only its entropy records zeroed. + pub(crate) fn identity( + &mut self, + builds: impl IntoIterator, + ) -> Identity { + let text_size = self.text.len() as u64; + let filled = builds + .into_iter() + .find(|build| build.text_size == text_size && !build.boot_filled.is_empty()) + .map_or(&[][..], |build| build.boot_filled); + let records = Records::find(&self.text); + let digest = records + .as_ref() + .and_then(|r| digest_at(&mut self.text, r.as_slice(), filled)); + Identity { + text_size, + data_size: self.data_size, + filled: filled.len(), + measured: records.zip(digest), + } + } +} + +/// A measured identity, as a pinned [`FirmwareBuild`] would state it. +pub(crate) struct Identity { + text_size: u64, + data_size: u64, + /// How many boot-filled ranges were zeroed besides the entropy. + filled: usize, + measured: Option<(Records, [u8; 32])>, +} + +impl fmt::Display for Identity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TEXT {:#x} DATA {:#x}", self.text_size, self.data_size)?; + let Some((records, digest)) = &self.measured else { + return write!(f, " has no entropy records"); + }; + write!(f, " entropy")?; + for (i, [gkts, ecap]) in records.as_slice().iter().enumerate() { + write!( + f, + "{}{:#x}/{:#x}", + if i == 0 { " " } else { "," }, + gkts, + ecap + )?; + } + if self.filled != 0 { + write!(f, " ({} boot-filled ranges zeroed)", self.filled)?; + } + write!(f, " sha256 ")?; + fmt::Display::fmt(&Hex(digest), f) + } +} + /// Identifies the firmware build loaded at `text` (DATA `data_size` bytes) among `builds`, /// before any firmware-visible write. +#[cfg_attr(not(CONFIG_DRM_ASAHI_G16), allow(dead_code))] pub(crate) fn identify( dev: &device::Device, text: Range, data_size: u64, builds: &'static [FirmwareBuild], ) -> Result<&'static FirmwareBuild> { - let size = usize::try_from(text.end - text.start)?; - let mut copy = copy_physical(text.start, size)?; - let again = copy_physical(text.start, size)?; - if copy.as_slice() != again.as_slice() { - dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); - return Err(EAGAIN); - } - if let Some(build) = lookup(builds, size as u64, data_size, |at| { - digest_at(&mut copy, at) - }) { + let mut measured = measure(text, data_size).inspect_err(|e| { + if *e == EAGAIN { + dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); + } + })?; + if let Some(build) = measured.find(builds) { dev_info!(dev, "GPU firmware build: {}\n", build.name); return Ok(build); } - let Some((entropy, digest)) = - find_entropy(©).and_then(|at| Some((at, digest_at(&mut copy, at)?))) - else { - dev_err!( - dev, - "unsupported GPU firmware build: TEXT {:#x} has no entropy records\n", - size - ); - return Err(EINVAL); - }; dev_err!( dev, - "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} entropy {:#x}/{:#x} sha256 {}\n", - size, - data_size, - entropy[0], - entropy[1], - Hex(&digest) + "unsupported GPU firmware build: {}\n", + measured.identity(builds) ); for build in builds { dev_err!(dev, "supported: {}\n", build.name); @@ -167,48 +317,120 @@ pub(crate) fn identify( mod tests { use super::*; - fn image() -> [u8; 64] { - let mut image = [0x5a; 64]; - for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { - image[off..off + 4].copy_from_slice(&tag); - image[off + 4..off + 8].copy_from_slice(&8u32.to_le_bytes()); + /// `roles` GKTS/ECAP pairs, at 0/32 within each 64 bytes. + fn image() -> [u8; N] { + let mut image = [0x5a; N]; + for role in 0..N / 64 { + for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { + image[role * 64 + off..role * 64 + off + 8].copy_from_slice(&record(tag)); + } } image } #[test] fn entropy_is_normalized_without_changing_the_image() { - let mut first = image(); + let mut first = image::<64>(); let original = first; - let digest = digest_at(&mut first, [0, 32]).unwrap(); + let digest = digest_at(&mut first, &[[0, 32]], &[]).unwrap(); assert_eq!(first, original); let mut second = first; second[8..16].fill(0xff); second[40..48].fill(0x01); - assert_eq!(digest_at(&mut second, [0, 32]), Some(digest)); + assert_eq!(digest_at(&mut second, &[[0, 32]], &[]), Some(digest)); second[63] ^= 1; - assert_ne!(digest_at(&mut second, [0, 32]), Some(digest)); + assert_ne!(digest_at(&mut second, &[[0, 32]], &[]), Some(digest)); + } + + #[test] + fn every_role_is_normalized() { + let mut first = image::<128>(); + let original = first; + let both = &[[0, 32], [64, 96]]; + let digest = digest_at(&mut first, both, &[]).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[72..80].fill(0xff); + second[104..112].fill(0x01); + assert_eq!(digest_at(&mut second, both, &[]), Some(digest)); + // The second role's entropy is part of the digest unless its records are named. + assert_ne!( + digest_at(&mut second, &[[0, 32]], &[]), + digest_at(&mut first, &[[0, 32]], &[]) + ); + second[127] ^= 1; + assert_ne!(digest_at(&mut second, both, &[]), Some(digest)); + let found = Records::find(&first).unwrap(); + assert_eq!(found.as_slice(), both); + assert_eq!( + digest_at(&mut first, &[[0, 32], [64, 96], [0, 32]], &[]), + None + ); + } + + #[test] + fn boot_filled_ranges_are_normalized() { + let mut first = image::<128>(); + let original = first; + let filled = &[(16, 8), (100, 28)]; + let digest = digest_at(&mut first, &[[0, 32]], filled).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[16..24].fill(0x11); + second[100..128].fill(0x22); + assert_eq!(digest_at(&mut second, &[[0, 32]], filled), Some(digest)); + second[99] ^= 1; + assert_ne!(digest_at(&mut second, &[[0, 32]], filled), Some(digest)); + assert_eq!(digest_at(&mut first, &[[0, 32]], &[(120, 9)]), None); + } + + #[test] + fn a_record_without_its_entropy_bytes_is_refused() { + // The role's ECAP record is in the last 8 bytes, so it has no entropy bytes. + let mut tail = image::<64>(); + tail[32..40].fill(0x5a); + tail[56..64].copy_from_slice(&record(ENTROPY_TAGS[1])); + let original = tail; + assert_eq!(digest_at(&mut tail, &[[0, 56]], &[(16, 8)]), None); + assert_eq!(digest_at(&mut tail, &[[0, 56]], &[]), None); + assert_eq!(tail, original); + assert!(Records::find(&tail).is_none()); + // An unknown build with a pinned build's TEXT size is reported, not a panic. + static PINNED: [FirmwareBuild; 1] = [FirmwareBuild { + name: "pinned", + text_size: 64, + data_size: 0, + entropy: &[[0, 32]], + boot_filled: &[(16, 8)], + text_sha256: [0; 32], + }]; + let mut text = KVVec::from_elem(0u8, tail.len(), GFP_KERNEL).unwrap(); + text.copy_from_slice(&tail); + let mut measured = Measured { text, data_size: 0 }; + assert!(measured.find(&PINNED).is_none()); + assert!(measured.identity(&PINNED).measured.is_none()); } #[test] fn malformed_records_are_refused_without_changing_the_image() { - let mut image = image(); - assert_eq!(digest_at(&mut image, [0, 56]), None); + let mut image = image::<64>(); + assert_eq!(digest_at(&mut image, &[[0, 56]], &[]), None); + assert_eq!(digest_at(&mut image, &[], &[]), None); image[36] = 7; let original = image; - assert_eq!(digest_at(&mut image, [0, 32]), None); + assert_eq!(digest_at(&mut image, &[[0, 32]], &[]), None); assert_eq!(image, original); - assert_eq!(find_entropy(&image), None); + assert!(Records::find(&image).is_none()); } fn measured( builds: &'static [FirmwareBuild], data_size: u64, - entropy: [u32; 2], + entropy: &[[u32; 2]], digest: [u8; 32], ) -> Option<&'static FirmwareBuild> { - lookup(builds, 0x60000, data_size, |at| { - (at == entropy).then_some(digest) + lookup(builds, 0x60000, data_size, |build| { + (build.entropy == entropy).then_some(digest) }) } @@ -238,13 +460,13 @@ mod tests { ], ]; for (i, digest) in digests.into_iter().enumerate() { - let found = measured(build.images, 0x138000, [0x59839, 0x5991c], digest).unwrap(); + let found = measured(build.images, 0x138000, &[[0x59839, 0x5991c]], digest).unwrap(); assert!(core::ptr::eq(found, &build.images[i])); - assert!(measured(build.images, 0x13c000, [0x59839, 0x5991c], digest).is_none()); - assert!(measured(build.images, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(measured(build.images, 0x13c000, &[[0x59839, 0x5991c]], digest).is_none()); + assert!(measured(build.images, 0x138000, &[[0x5c8df, 0x5c9c2]], digest).is_none()); assert!(lookup(build.images, 0x60001, 0x138000, |_| Some(digest)).is_none()); } - assert!(measured(build.images, 0x138000, [0x59839, 0x5991c], [0; 32]).is_none()); + assert!(measured(build.images, 0x138000, &[[0x59839, 0x5991c]], [0; 32]).is_none()); } #[cfg(CONFIG_DRM_ASAHI_G16)] @@ -264,10 +486,52 @@ mod tests { ], ]; for (i, digest) in digests.into_iter().enumerate() { - let found = measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], digest).unwrap(); + let found = measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], digest).unwrap(); assert!(core::ptr::eq(found, &builds[i])); - assert!(measured(builds, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(measured(builds, 0x138000, &[[0x5c8df, 0x5c9c2]], digest).is_none()); + } + assert!(measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], [0; 32]).is_none()); + } + + #[test] + fn t6050_admits_only_its_pinned_image() { + for cfg in [&crate::hw::t6050::CONFIG_ULTRA, &crate::hw::t6050::CONFIG_MAX] { + let crate::g17::fw::variant::Firmware::Identified(builds) = cfg.firmware else { + panic!("an M5 board on bare metal must identify its loaded firmware"); + }; + t6050_admits_only(builds); + } + } + + fn t6050_admits_only(builds: &[crate::g17::fw::variant::Build]) { + for image in builds.iter().flat_map(|build| build.images) { + // Both roles' records, inside TEXT. + assert_eq!(image.entropy.len(), 2); + assert!(image + .entropy + .iter() + .flatten() + .all(|&o| u64::from(o) + 16 <= image.text_size)); + let admit = |text, data, at: &[[u32; 2]], digest| { + lookup(core::slice::from_ref(image), text, data, |m| { + (m.entropy == at).then_some(digest) + }) + }; + // The header fields and both boot-filled sections of each role. + assert_eq!(image.boot_filled.len(), 8); + let (t, d, e, h) = ( + image.text_size, + image.data_size, + image.entropy, + image.text_sha256, + ); + assert!(core::ptr::eq(admit(t, d, e, h).unwrap(), image)); + let mut wrong = h; + wrong[31] ^= 1; + assert!(admit(t, d, e, wrong).is_none()); + assert!(admit(t, d + 0x4000, e, h).is_none()); + assert!(admit(t + 0x4000, d, e, h).is_none()); + assert!(admit(t, d, &e[..1], h).is_none()); } - assert!(measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], [0; 32]).is_none()); } } diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 70decd06fca0d..95269e152177b 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -61,8 +61,6 @@ pub(crate) mod glue { fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); fn asahi_g15_release_power(dev: *mut core::ffi::c_void); - fn asahi_g15_module_try_get() -> bool; - fn asahi_g15_module_put(); pub(crate) fn asahi_g15_reprobe(); pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; @@ -127,14 +125,7 @@ pub(crate) mod glue { /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). /// ENODEV, with nothing pinned or marked, once the module's removal has begun. pub(crate) fn retain_module() -> Result { - crate::g15::retain::retain( - &RETAINED, - // SAFETY: Takes a reference to this module unless its removal has begun. - || unsafe { asahi_g15_module_try_get() }, - // SAFETY: Drops only the reference this call took; the winner's is never put. - || unsafe { asahi_g15_module_put() }, - ) - .map_err(|_| ENODEV) + crate::retain::retain_module(&RETAINED) } /// The phram "adt" export: start and size. diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 095b9cb96d51f..2a0346228c1e5 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * G15 (T6030/T6031, M3 Pro/Max) helpers that have no Rust bindings: static inlines, - * macros and THIS_MODULE. Built only with CONFIG_DRM_ASAHI_G15. + * G15 (T6030/T6031, M3 Pro/Max) helpers that have no Rust bindings: static inlines + * and macros. The module pin is retain.c's. Built only with CONFIG_DRM_ASAHI_G15. */ #include #include @@ -26,8 +26,6 @@ bool asahi_g15_board(void); bool asahi_g15_machine(const char *compatible); void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); -bool asahi_g15_module_try_get(void); -void asahi_g15_module_put(void); void asahi_g15_reprobe(void); void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); @@ -247,22 +245,6 @@ void asahi_g15_release_power(struct device *dev) } -/* - * Pin this module until reboot, unless its removal has begun: a deferred or - * asynchronous probe can run after an ordinary rmmod has passed its reference - * check, which __module_get() would not stop. - */ -bool asahi_g15_module_try_get(void) -{ - return try_module_get(THIS_MODULE); -} - -/* Drop a reference asahi_g15_module_try_get() took. */ -void asahi_g15_module_put(void) -{ - module_put(THIS_MODULE); -} - /* * The boot loader's phram "adt" export: the first /reserved-memory child * compatible with "phram" and labelled "adt". diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 9ffd858598b1c..32ecf401dcf18 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -45,7 +45,6 @@ mod adt; mod board; mod boot; -mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; diff --git a/drivers/gpu/drm/asahi/g16/boot.rs b/drivers/gpu/drm/asahi/g16/boot.rs index 0e4d315f64f36..509ed3368df77 100644 --- a/drivers/gpu/drm/asahi/g16/boot.rs +++ b/drivers/gpu/drm/asahi/g16/boot.rs @@ -44,8 +44,6 @@ pub(crate) mod glue { fn asahi_g16_board() -> bool; fn asahi_g16_display_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; fn asahi_g16_hold_power(dev: *mut core::ffi::c_void) -> core::ffi::c_int; - fn asahi_g16_module_try_get() -> bool; - fn asahi_g16_module_put(); fn asahi_g16_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g16_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; fn asahi_g16_dma_unsupported(dev: *mut core::ffi::c_void) -> bool; @@ -87,21 +85,10 @@ pub(crate) mod glue { RETAINED.load(Ordering::Acquire) } - /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`). + /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). /// ENODEV, with nothing pinned or marked, once the module's removal has begun. pub(crate) fn retain_module() -> Result { - if RETAINED.load(Ordering::Acquire) { - return Ok(()); - } - // SAFETY: Takes a reference to this module unless its removal has begun. - if !unsafe { asahi_g16_module_try_get() } { - return Err(ENODEV); - } - if RETAINED.swap(true, Ordering::AcqRel) { - // SAFETY: Drops only the reference this call took; the first one is never put. - unsafe { asahi_g16_module_put() }; - } - Ok(()) + crate::retain::retain_module(&RETAINED) } /// The physical range is disjoint from System RAM. @@ -495,7 +482,8 @@ pub(crate) fn admit( } let regs = regs::Regs::new(pdev, res)?; // Once power admission starts, even a rejected cold check must not detach the GFX - // domain under an inherited firmware owner. Keep the module and platform binding. + // domain under an inherited firmware owner. Keep the module and platform binding; once + // the module's removal has begun nothing is held. glue::retain_module()?; glue::hold_power(dev)?; let (power_offset, power_state) = glue::power_domain_state(dev)?; @@ -741,9 +729,12 @@ pub(crate) fn start( return Err(e); } - // From here on the firmware is touched: nothing it can reach is freed. - // Admission already pinned the module, so this cannot fail. - let _ = glue::retain_module(); + // From here on the firmware is touched: nothing it can reach is freed. Admission pinned + // the module, so this only confirms it. + if let Err(e) = glue::retain_module() { + core::mem::forget(objects); + return Err(e); + } let rtkit = match Rtkit::new(&d, shared) { Ok(rtkit) => rtkit, Err(e) => { diff --git a/drivers/gpu/drm/asahi/g16/glue.c b/drivers/gpu/drm/asahi/g16/glue.c index 82eb76a196da8..ad258092c035b 100644 --- a/drivers/gpu/drm/asahi/g16/glue.c +++ b/drivers/gpu/drm/asahi/g16/glue.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * G16 (T8132, M4) helpers that have no Rust bindings: static inlines, - * macros and THIS_MODULE. Built only with CONFIG_DRM_ASAHI_G16. + * G16 (T8132, M4) helpers that have no Rust bindings: static inlines + * and macros. The module pin is retain.c's. Built only with CONFIG_DRM_ASAHI_G16. */ #include #include @@ -27,8 +27,6 @@ bool asahi_g16_board(void); int asahi_g16_display_prepare(struct device *dev); int asahi_g16_hold_power(struct device *dev); -bool asahi_g16_module_try_get(void); -void asahi_g16_module_put(void); bool asahi_g16_ram_disjoint(u64 start, u64 size); void __iomem *asahi_g16_ioremap_wc(u64 start, size_t size); bool asahi_g16_dma_unsupported(struct device *dev); @@ -197,21 +195,6 @@ int asahi_g16_hold_power(struct device *dev) return dev->pm_domain ? 0 : -EINVAL; } -/* - * Pin this module until reboot, unless its removal has begun: a deferred - * probe can run after rmmod has passed its reference check. - */ -bool asahi_g16_module_try_get(void) -{ - return try_module_get(THIS_MODULE); -} - -/* Drop a reference asahi_g16_module_try_get() took. */ -void asahi_g16_module_put(void) -{ - module_put(THIS_MODULE); -} - /* The range is disjoint from System RAM (ADT region admission). */ bool asahi_g16_ram_disjoint(u64 start, u64 size) { diff --git a/drivers/gpu/drm/asahi/g16/hw.rs b/drivers/gpu/drm/asahi/g16/hw.rs index 696b4bc9f4062..f32d18874e908 100644 --- a/drivers/gpu/drm/asahi/g16/hw.rs +++ b/drivers/gpu/drm/asahi/g16/hw.rs @@ -42,7 +42,8 @@ pub(crate) mod t8132 { name: "Apple GFX 25G83 (normalized loaded image)", text_size: 0x60000, data_size: 0x13c000, - entropy: [0x5c8df, 0x5c9c2], + entropy: &[[0x5c8df, 0x5c9c2]], + boot_filled: &[], text_sha256: [ 0x5f, 0x65, 0xf6, 0x14, 0x58, 0x72, 0x54, 0x70, 0xb4, 0xb0, 0xc0, 0x20, 0x28, 0x47, 0x7b, 0xf2, 0xea, 0x57, 0x3a, 0xfb, 0x4e, 0x0c, 0x49, 0x6e, 0xf9, 0x95, @@ -55,7 +56,8 @@ pub(crate) mod t8132 { name: "Apple GFX 25G83 (normalized loaded image, 8-core J713)", text_size: 0x60000, data_size: 0x13c000, - entropy: [0x5c8df, 0x5c9c2], + entropy: &[[0x5c8df, 0x5c9c2]], + boot_filled: &[], text_sha256: [ 0x3b, 0x36, 0x8e, 0xe4, 0x2d, 0xcc, 0xc6, 0x26, 0xe8, 0x49, 0x67, 0xff, 0x40, 0x87, 0x64, 0x71, 0x0a, 0x64, 0xbf, 0xf1, 0x59, 0x4f, 0x3c, 0xfd, 0x2c, 0x6e, diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 4cf9057f04fb0..38fd612d240da 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -173,7 +173,8 @@ impl platform::Driver for Driver { fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(driver) = this.get_ref() { if let Some(gpu) = &driver.g16 { - gpu.retain(); + // Admission pinned the module; this only confirms the pin. + let _ = gpu.retain(); } driver.close_binding(); } @@ -415,18 +416,17 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the - /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a - /// failed start) until reboot. - pub(crate) fn retain(&self) { - // Admission already pinned the module, so this cannot fail. - let _ = boot::glue::retain_module(); + /// Takes the module reference once; admission already took it, so this only confirms it. + /// Nothing firmware-reachable is freed on any path: the RTKit callbacks hold [`Shared`], + /// which holds the [`Firmware`] (or the [`boot::Orphan`] of a failed start) until reboot. + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs index c76a20db7007a..9deaac406a5a2 100644 --- a/drivers/gpu/drm/asahi/g17/availability.rs +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -98,14 +98,33 @@ impl Availability { /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence /// callbacks run outside the domain lock. pub(crate) fn notify(&self) { - let mut waiters = { + self.advance(); + } + + /// [`Self::notify`] for a caller that sampled `observed` and is giving back its own claim, + /// never published, before it waits. Returns the sample it waits on next: the new + /// generation if nothing else changed the domain since `observed`, so its own notification + /// does not end its wait at once, and `observed` otherwise, so it retries. + pub(crate) fn notify_own(&self, observed: u64) -> u64 { + if self.advance() == observed { + observed.wrapping_add(1) + } else { + observed + } + } + + /// Notifies, returning the generation before this notification. + fn advance(&self) -> u64 { + let (previous, mut waiters) = { let mut state = self.state.lock(); - state.generation = state.generation.wrapping_add(1); - state.waiters.take() + let previous = state.generation; + state.generation = previous.wrapping_add(1); + (previous, state.waiters.take()) }; while let Some(mut entry) = waiters { waiters = entry.next.take(); entry.fence.signal(); } + previous } } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index fd6460db2d1fb..0747a1fe69c87 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -9,6 +9,9 @@ use super::fw::buffer as fw; use crate::hw::t8140::resources as cfg; pub(crate) mod target; +mod geometry; +pub(crate) use geometry::BlockGeometry; +mod layout; use kernel::prelude::*; @@ -19,6 +22,8 @@ struct BufferId { references: u8, owner: bool, token: u64, + /// Hardware context and PB state named by this token's M5 binding. + bound: Option<(u32, u64)>, } impl BufferId { fn free(self) -> bool { @@ -40,6 +45,7 @@ impl BufferIds { references: 0, owner: false, token: 0, + bound: None, }), returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), returned_count: BUFFER_IDS, @@ -88,6 +94,7 @@ impl BufferIds { references: 1, owner: false, token, + bound: None, }; Ok((id, token)) } @@ -101,6 +108,7 @@ impl BufferIds { if entry.free() { let token = self.fresh_token(); self.entries[usize::from(id)].token = token; + self.entries[usize::from(id)].bound = None; } self.entries[usize::from(id)].owner = true; self.remove_returned(id); @@ -114,6 +122,28 @@ impl BufferIds { entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; Ok((id, entry.token)) } + /// A repeated token must name the same M5 PB state and hardware context. + /// Renew it if an idle VM moved; other firmware profiles keep acquire_owner. + pub(crate) fn acquire_owner_bound(&mut self, id: u8, context: u32, pb_va: u64) + -> Result<(u8, u64)> { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Err(EINVAL); + } + let references = entry.references.checked_add(1).ok_or(ENOSPC)?; + let previous = entry.bound; + let old_token = entry.token; + let token = if previous.is_some_and(|bound| bound != (context, pb_va)) { + self.fresh_token() + } else { + old_token + }; + let entry = &mut self.entries[usize::from(id)]; + entry.references = references; + entry.token = token; + entry.bound = Some((context, pb_va)); + Ok((id, token)) + } /// Preflight shared completion bookkeeping before any of its counters change. pub(crate) fn can_release(&self, id: u8) -> Result { let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; @@ -146,16 +176,22 @@ impl BufferIds { /// Each scene and its shorter scratch slot are one lifetime. A completed /// paired render releases both; merely publishing a later scene does not. pub(crate) struct Scenes { - owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], + owners: [Option<(u64, u8)>; fw::SCENES], + scratch_slots: usize, next: usize, } impl Scenes { pub(crate) const fn new() -> Self { Self { - owners: [None; fw::SCRATCH_SLOTS], + owners: [None; fw::SCENES], + scratch_slots: fw::SCRATCH_SLOTS, next: fw::FIRST_SCENE, } } + fn with_slots(scratch_slots: usize) -> Result { + if scratch_slots == 0 || scratch_slots > fw::SCENES { return Err(EINVAL); } + Ok(Self { scratch_slots, ..Self::new() }) + } pub(crate) fn select(&self, ordinal: u64) -> Result { if let Some((_, scene)) = self .owners @@ -167,14 +203,14 @@ impl Scenes { } (0..fw::SCENES) .map(|offset| (self.next + offset) % fw::SCENES) - .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) + .find(|scene| self.owners[scene % self.scratch_slots].is_none()) .ok_or(ENOSPC) } pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { if self.select(ordinal)? != scene { return Err(EINVAL); } - let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; + let slot = &mut self.owners[scene % self.scratch_slots]; if slot.is_none() { *slot = Some((ordinal, scene as u8)); self.next = (scene + 1) % fw::SCENES; @@ -200,16 +236,16 @@ pub(crate) struct Cursors { pub(crate) read: u32, } impl Cursors { - fn valid(self) -> bool { + fn valid(self, capacity: usize) -> bool { [self.total, self.committed, self.read] .into_iter() - .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) + .all(|v| (v as usize) < capacity) } /// Firmware-paused growth permits consumption of the already committed /// prefix. Retired-host growth requires the complete snapshot unchanged. - pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { - if !self.valid() - || !after.valid() + fn permits_capacity(self, after: Self, firmware_request: bool, capacity: usize) -> bool { + if !self.valid(capacity) + || !after.valid(capacity) || self.total != after.total || self.committed != after.committed { @@ -218,7 +254,7 @@ impl Cursors { if !firmware_request { return self == after; } - let capacity = cfg::TVB_BLOCK_SLOTS as u32; + let capacity = capacity as u32; (after.read + capacity - self.read) % capacity <= (self.committed + capacity - self.read) % capacity } @@ -227,23 +263,25 @@ impl Cursors { /// Checked block-table indices for one bulk allocation. One ring slot stays /// empty even when all physical backing has been retained by the owner. pub(crate) struct Growth { + capacity: usize, before: Cursors, added: usize, retained: usize, target: usize, } impl Growth { - pub(crate) fn new(before: Cursors, added: usize) -> Result { - if !before.valid() || before.total != before.committed || added == 0 { + fn with_capacity(before: Cursors, added: usize, capacity: usize) -> Result { + if capacity < 2 || capacity > u32::MAX as usize + || !before.valid(capacity) || before.total != before.committed || added == 0 { return Err(EINVAL); } - let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) - % cfg::TVB_BLOCK_SLOTS; - if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { + let retained = (before.total as usize + capacity - before.read as usize) % capacity; + if retained.checked_add(added).ok_or(EOVERFLOW)? >= capacity { return Err(ENOSPC); } - let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; + let target = (before.total as usize + added) % capacity; Ok(Self { + capacity, before, added, retained, @@ -263,32 +301,25 @@ impl Growth { if index >= self.added { return Err(EINVAL); } - Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) + Ok((self.before.total as usize + index) % self.capacity) } pub(crate) fn available_slot(&self, index: usize) -> Result { if index >= self.available() { return Err(EINVAL); } - Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) + Ok((self.before.read as usize + index) % self.capacity) } } -pub(crate) fn firmware_growth_target(current: usize) -> usize { - current - .saturating_add(fw::GROW_BLOCKS) - .min(cfg::TVB_MAX_BLOCKS) -} - /// The high-water sample estimates capacity, not current occupancy. A growth /// reserves three times the demand only once twice that demand no longer fits. -pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { - let maximum = fw::NORMAL_MAX_BLOCKS; +fn automatic_target_for(used_pages: usize, current: usize, pages: usize, maximum: usize) -> Option { if current >= maximum { return None; } let need = used_pages .saturating_mul(2) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .div_ceil(pages) .min(maximum); if need <= current { return None; @@ -296,7 +327,7 @@ pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Opti Some( used_pages .saturating_mul(3) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .div_ceil(pages) .min(maximum), ) } @@ -340,7 +371,11 @@ pub(crate) struct TvbRelease { /// Packed 128-KiB data blocks mapped at guarded 160-KiB virtual strides. /// Firmware owns their contents; CPU zeroing occurs only on fresh backing. +/// +/// M5 blocks use their own geometry ([`BlockGeometry::m5`]): sparse, at the +/// geometry's backing stride, in the M5 compact window. pub(crate) struct Tvb { + geometry: BlockGeometry, addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. _primary: KernelObject, @@ -352,11 +387,27 @@ impl Tvb { fn bytes(blocks: usize) -> Result { blocks.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW) } + fn backing_bytes(geometry: BlockGeometry, blocks: usize) -> Result { + if geometry == BlockGeometry::NEO { + return Self::bytes(blocks); + } + geometry.backing_bytes(blocks) + } fn map_block( object: &KernelObject, vm: &mmu::Vm, relative: usize, + geometry: BlockGeometry, ) -> Result { + if geometry != BlockGeometry::NEO { + let start = relative.checked_mul(geometry.backing_stride).ok_or(EOVERFLOW)?; + let source = start..start.checked_add(geometry.block_bytes).ok_or(EOVERFLOW)?; + let guard = geometry.stride - geometry.block_bytes; + let mapping = object.map_range(vm, source, geometry.base..geometry.end, + BlockGeometry::PAGE as u64, mmu::PROT_GPU_SHARED_RW, guard)?; + geometry.compact_page(mapping.iova())?; + return Ok(mapping); + } let start = relative.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let mapping = compact_alias( @@ -375,19 +426,29 @@ impl Tvb { vm: &mmu::Vm, blocks: usize, ) -> Result<(Self, KVec)> { - if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { + Self::with_geometry(dev, vm, blocks, BlockGeometry::NEO) + } + pub(crate) fn with_geometry( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + blocks: usize, + geometry: BlockGeometry, + ) -> Result<(Self, KVec)> { + if blocks == 0 || blocks > geometry.max_blocks() { return Err(ERANGE); } - let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + let backing = KernelObject::backing(dev, Self::backing_bytes(geometry, blocks)?, + CpuMap::WriteCombined)?; let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; for index in 0..blocks { - let mapping = Self::map_block(&backing, vm, index)?; + let mapping = Self::map_block(&backing, vm, index, geometry)?; addresses.push(mapping.iova(), GFP_KERNEL)?; mappings.push(mapping, GFP_KERNEL)?; } Ok(( Self { + geometry, addresses, _primary: backing, extensions: KVec::new(), @@ -423,17 +484,18 @@ impl Tvb { if target <= self.blocks() { return Ok(None); } - if target > cfg::TVB_MAX_BLOCKS { + if target > self.geometry.max_blocks() { return Err(ERANGE); } let added = target - self.blocks(); self.extensions.reserve(1, GFP_KERNEL)?; self.addresses.reserve(added, GFP_KERNEL)?; - let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + let backing = KernelObject::backing(dev, Self::backing_bytes(self.geometry, added)?, + CpuMap::WriteCombined)?; let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; for relative in 0..added { - let mapping = Self::map_block(&backing, vm, relative)?; + let mapping = Self::map_block(&backing, vm, relative, self.geometry)?; addresses.push(mapping.iova(), GFP_KERNEL)?; mappings.push(mapping, GFP_KERNEL)?; } @@ -449,10 +511,17 @@ use super::object::Allocator; use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use kernel::sync::Arc; +/// Bytes per scene of an M5 metrics entry (the M5 graph layout's metric stride). +const M5_METRIC_STRIDE: usize = 64; + /// First-free metrics entries are shared by all parameter-buffer graphs. /// An entry returns only when the graph and its firmware references retire. pub(crate) struct MetricsIds { used: [AtomicU64; 2], + scene_stride: usize, + /// Bytes from one entry to the next. An M5 entry starts its own pages, so a + /// client VM maps the entries of its own renders and no others. + entry_pitch: usize, } impl MetricsIds { pub(crate) fn new() -> Self { @@ -461,29 +530,86 @@ impl MetricsIds { AtomicU64::new(0), AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), ], + scene_stride: size_of::(), + entry_pitch: size_of::() * fw::SCENES, } } + /// The allocator of the firmware's metrics entries: G17P's fixed entries, + /// or M5's `M5_METRIC_STRIDE`-byte scene entries. + pub(crate) fn for_firmware(m5: bool) -> Result { + if m5 { + Self::with_stride(M5_METRIC_STRIDE) + } else { + Ok(Self::new()) + } + } + /// The M5 metrics entries: `scene_stride` bytes per scene, bounded by the + /// backing each lease is taken from. + pub(crate) fn with_stride(scene_stride: usize) -> Result { + if scene_stride < 4 || scene_stride % 4 != 0 { return Err(EINVAL); } + let entry_pitch = scene_stride.checked_mul(fw::SCENES) + .and_then(|bytes| bytes.checked_next_multiple_of(mmu::UAT_PGSZ)) + .ok_or(EOVERFLOW)?; + Ok(Self { + used: [AtomicU64::new(0), AtomicU64::new(0)], + scene_stride, + entry_pitch, + }) + } + /// Backing required for every entry owned by this allocator. + pub(crate) fn storage_bytes(&self) -> Result { + self.entry_pitch.checked_mul(self.used.len() * u64::BITS as usize) + .ok_or(EOVERFLOW) + } } pub(crate) struct MetricsLease { - _backing: Arc, + backing: Arc, allocator: Arc, slot: usize, addresses: fw::MetricsAddresses, + scene_stride: usize, + /// M5: the client VM's view of this entry's pages alone. + client: Option, } impl MetricsLease { + /// M5: maps this entry's pages, and no other entry, into the client VM, + /// where the GPU writes the entry while the render runs. + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + if self.client.is_some() { + return Err(EINVAL); + } + let pitch = self.allocator.entry_pitch; + let offset = self.slot.checked_mul(pitch).ok_or(EOVERFLOW)?; + let client = self.backing.map_range(vm, offset..offset + pitch, BlockGeometry::M5_COMPACT, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 0)?; + self.addresses.client = client.iova(); + self.client = Some(client); + Ok(()) + } /// Clears only the selected retired metrics entry before reusing it. pub(crate) fn new( ids: &Arc, object: &Arc, bases: fw::MetricsAddresses, ) -> Result { + let scene_stride = ids.scene_stride; + let bytes = ids.entry_pitch; + // The fixed G17P entries are bounded by their reserved bits; an M5 + // allocator by the backing the lease is taken from. + let slots = if scene_stride == size_of::() { ids.used.len() * 64 } else { + // The M5 backing covers every entry of its allocator. + if object.size() < ids.storage_bytes()? { + return Err(EINVAL); + } + (object.size() / bytes).min(ids.used.len() * 64) + }; let mut selected = None; for (bank, used) in ids.used.iter().enumerate() { let mut value = used.load(Ordering::Acquire); loop { let bit = value.trailing_ones(); - if bit == 64 { + if bit == 64 || bank * 64 + bit as usize >= slots { break; } match used.compare_exchange_weak( @@ -507,21 +633,24 @@ impl MetricsLease { // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { - _backing: object.clone(), + backing: object.clone(), allocator: ids.clone(), slot, addresses: fw::MetricsAddresses::default(), + scene_stride, + client: None, }; - let offset = slot * fw::METRICS_STRIDE; + let offset = slot * bytes; lease.addresses = fw::MetricsAddresses { firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, }; - let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); + let words = object.pointer(offset, bytes)?.cast::(); object.word(offset)?; - for index in 0..fw::SCENES { - // SAFETY: The complete naturally aligned metrics entry is inside - // the retained object; the exclusive lease excludes other owners. + for index in 0..bytes / size_of::() { + // SAFETY: The complete naturally aligned metrics entry, with the rest of + // its pages on M5, is inside the retained object; the exclusive lease + // excludes other owners. unsafe { words.add(index).write_volatile(0) }; } Ok(lease) @@ -529,6 +658,8 @@ impl MetricsLease { } impl Drop for MetricsLease { fn drop(&mut self) { + // The client's view is unmapped before another render can lease the entry. + drop(self.client.take()); self.allocator.used[self.slot / 64] .fetch_and(!(1u64 << (self.slot % 64)), Ordering::Release); } @@ -548,9 +679,12 @@ pub(crate) struct ManagerArgs { pub(crate) struct Manager { _page_list: mmu::KernelMapping, graph: KernelObject, + layout: layout::Layout, tvb: Tvb, metrics: MetricsLease, - scenes: Scenes, + /// Boxed: the per-scene owner table would otherwise make every `Manager` value large, and + /// one is built and moved through the frames of the submit path that creates a pair. + scenes: KBox, page_list_client: u64, scratch: u64, buffer_id: u8, @@ -616,41 +750,56 @@ impl Manager { metrics: MetricsLease, args: ManagerArgs, ) -> Result { + let geometry = tvb.geometry; + let neo = geometry == BlockGeometry::NEO; + let layout = if neo { layout::Layout::NEO } else { layout::Layout::m5(geometry)? }; + if metrics.scene_stride != layout.metric_stride { + return Err(EINVAL); + } let mut graph = alloc.kernel( - fw::SIZE, + layout.bytes, cfg::TVB_PAGE, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; - let page_list = compact_alias( - &graph, - vm, - source, - cfg::TVB_PAGE, - mmu::PROT_GPU_FW_SHARED_RW, - mmu::UAT_PGSZ, - )?; + let source = layout.pages..layout.pages + layout.page_bytes; + let page_list = if neo { + compact_alias( + &graph, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )? + } else { + graph.map_range(vm, source, geometry.base..geometry.end, + cfg::TVB_PAGE, mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ)? + }; let address = graph.gpu_va(); - graph.initialize::(0, |view| { - view.init(&fw::GraphArgs { - fw_va: address, - page_list_client: page_list.iova(), - metrics: metrics.addresses, - blocks: tvb.addresses(), - scratch: args.scratch, - discard: args.discard, - buffer_id: args.buffer_id.into(), - generation: args.generation, - }) - })?; + let graph_args = fw::GraphArgs { + fw_va: address, + page_list_client: page_list.iova(), + metrics: metrics.addresses, + blocks: tvb.addresses(), + scratch: args.scratch, + discard: args.discard, + buffer_id: args.buffer_id.into(), + generation: args.generation, + }; + if neo { + graph.initialize::(0, |view| view.init(&graph_args))?; + } else { + layout.initialize(geometry, &graph_args, graph.bytes_mut())?; + } fence(Ordering::SeqCst); - let mut scenes = Scenes::new(); + let mut scenes = KBox::new(Scenes::with_slots(layout.scratch_slots)?, GFP_KERNEL)?; scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; let page_list_client = page_list.iova(); Ok(Self { _page_list: page_list, graph, + layout, tvb, metrics, scenes, @@ -677,6 +826,13 @@ impl Manager { pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } + pub(crate) fn page_count(&self) -> Result { + self.blocks().checked_mul(self.tvb.geometry.pages()).ok_or(EOVERFLOW)? + .try_into().map_err(|_| EOVERFLOW) + } + pub(crate) fn firmware_target(&self) -> usize { + self.blocks().saturating_add(fw::GROW_BLOCKS).min(self.tvb.geometry.max_blocks()) + } /// Read-only diagnostic while firmware is halted. In particular this /// does not reset the high-water counter as automatic_target() does. pub(crate) fn report_recovery(&self) { @@ -689,8 +845,8 @@ impl Manager { read(core::mem::offset_of!(fw::State, block_capacity)), read(core::mem::offset_of!(fw::State, producer)), read(core::mem::offset_of!(fw::State, consumer)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), + read(self.layout.control + core::mem::offset_of!(fw::Control, max_pages)), + read(self.layout.control + core::mem::offset_of!(fw::Control, reset)), self.initialized); } pub(crate) fn state_va(&self) -> u64 { @@ -700,20 +856,33 @@ impl Manager { self.page_list_client } pub(crate) fn trailer_va(&self) -> u64 { - self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 + self.graph.gpu_va() + self.layout.scenes as u64 + (fw::SCENES * size_of::()) as u64 } pub(crate) fn scene_va(&self, index: usize) -> Result { if index >= fw::SCENES { return Err(EINVAL); } - Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) + Ok(self.graph.gpu_va() + self.layout.scenes as u64 + (index * size_of::()) as u64) } pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses.client) + if self.tvb.geometry == BlockGeometry::NEO { + return fw::scene_registers(index, self.scratch, self.metrics.addresses.client); + } + if index >= fw::SCENES { return Err(EINVAL); } + let scratch = self.scratch.checked_sub(self.tvb.geometry.base).ok_or(EINVAL)? + .checked_add((index % self.layout.scratch_slots * self.layout.scratch_stride) as u64) + .ok_or(EOVERFLOW)?; + let metric = self.metrics.addresses.client.checked_sub(self.tvb.geometry.base) + .ok_or(EINVAL)? + .checked_add((index * self.layout.metric_stride) as u64).ok_or(EOVERFLOW)?; + if scratch > u32::MAX as u64 || metric > u32::MAX as u64 || metric & 3 != 0 { + return Err(EINVAL); + } + Ok((scratch, metric | 1)) } pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { let scene = self.scenes.select(ordinal)?; - let offset = fw::SCENE_TABLE + scene * size_of::(); + let offset = self.layout.scenes + scene * size_of::(); let used = self .graph .word(offset + core::mem::offset_of!(fw::Scene, used))?; @@ -738,7 +907,7 @@ impl Manager { self.scenes.release(ordinal) } pub(crate) fn prepare_submission(&mut self) -> Result> { - let word = self.graph.word(fw::COUNTER)?; + let word = self.graph.word(self.layout.counter)?; Ok(Submitted { word, next: word.load(Ordering::Relaxed).wrapping_add(1), @@ -749,10 +918,10 @@ impl Manager { fn sample_max_pages(&mut self) -> Result { let max_pages = self .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; + .word(self.layout.control + core::mem::offset_of!(fw::Control, max_pages))?; let reset = self .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; + .word(self.layout.control + core::mem::offset_of!(fw::Control, reset))?; fence(Ordering::Acquire); let pages = max_pages.load(Ordering::Relaxed); fence(Ordering::Release); @@ -764,7 +933,8 @@ impl Manager { pub(crate) fn automatic_target(&mut self) -> Result> { self.sample_max_pages()?; let pages = core::mem::take(&mut self.shrink.growth_max); - Ok(automatic_growth_target(pages as usize, self.blocks())) + Ok(automatic_target_for(pages as usize, self.blocks(), self.tvb.geometry.pages(), + self.layout.normal_blocks)) } /// Records the blocks a pass layout requires, the floor of any shrink. /// A pass is being prepared at `now`; the heap counts as busy. @@ -782,12 +952,13 @@ impl Manager { self.tvb.primary_blocks().max(self.shrink.required).max(recent) } /// Whether the heap holds growth beyond its primary backing. + /// M5 heaps are never shrunk. pub(crate) fn grown(&self) -> bool { - !self.tvb.extensions.is_empty() + self.tvb.geometry == BlockGeometry::NEO && !self.tvb.extensions.is_empty() } /// Whether the heap holds growth that a shrink could give back. pub(crate) fn shrinkable(&self) -> bool { - self.tvb.extensions.last().is_some_and(|extension| { + self.tvb.geometry == BlockGeometry::NEO && self.tvb.extensions.last().is_some_and(|extension| { self.blocks() - Tvb::extension_blocks(extension) >= self.shrink_floor() }) } @@ -836,9 +1007,9 @@ impl Manager { } fn cursors(&self) -> Result { Ok(Cursors { - total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), - committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), - read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), + total: self.graph.word(self.layout.control)?.load(Ordering::Relaxed), + committed: self.graph.word(self.layout.control + 4)?.load(Ordering::Relaxed), + read: self.graph.word(self.layout.control + 8)?.load(Ordering::Relaxed), }) } } @@ -861,12 +1032,14 @@ impl Manager { /// Validate every retained sparse block and its reserved trailing guard in /// one VM snapshot before constructing more aliases. pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { - if self.blocks() < cfg::TVB_INITIAL_PAGES.len() - || self.blocks() > cfg::TVB_MAX_BLOCKS + let geometry = self.tvb.geometry; + let minimum = if geometry == BlockGeometry::NEO { cfg::TVB_INITIAL_PAGES.len() } else { 1 }; + if self.blocks() < minimum + || self.blocks() > geometry.max_blocks() || !vm.covers_sparse_blocks_with_guards( self.tvb.addresses(), - cfg::TVB_BLOCK_SIZE as u64, - cfg::TVB_BLOCK_STRIDE, + geometry.block_bytes as u64, + geometry.stride as u64, ) { return Err(EFAULT); @@ -893,23 +1066,23 @@ impl Manager { return Ok(None); }; let before = self.cursors()?; - let plan = Growth::new(before, target - previous)?; + let plan = Growth::with_capacity(before, target - previous, self.tvb.geometry.capacity)?; let firmware_request = descriptor.is_none(); let pages = target - .checked_mul(cfg::TVB_PAGES_PER_BLOCK) + .checked_mul(self.tvb.geometry.pages()) .ok_or(EOVERFLOW)? as u32; let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; for (index, &address) in growth.addresses.iter().enumerate() { - let first = fw::compact_page(address)?; + let first = self.tvb.geometry.compact_page(address)?; first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .checked_add(self.tvb.geometry.pages() as u32 - 1) .ok_or(EOVERFLOW)?; added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; } let rebuilt_count = if firmware_request { 0 } else { - plan.available() * cfg::TVB_PAGES_PER_BLOCK + plan.available() * self.tvb.geometry.pages() }; let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; if !firmware_request { @@ -920,15 +1093,15 @@ impl Manager { let first = if ordinal < plan.retained() { let slot = plan.available_slot(ordinal)?; self.graph - .word(fw::BLOCK_TABLE + slot * size_of::())? + .word(self.layout.blocks + slot * size_of::())? .load(Ordering::Relaxed) } else { added[ordinal - plan.retained()].1 }; first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .checked_add(self.tvb.geometry.pages() as u32 - 1) .ok_or(EOVERFLOW)?; - for page in 0..cfg::TVB_PAGES_PER_BLOCK { + for page in 0..self.tvb.geometry.pages() { rebuilt.push(first + page as u32, GFP_KERNEL)?; } } @@ -962,7 +1135,7 @@ impl Manager { return Err(EBUSY); } let firmware_request = candidate.descriptor.is_none(); - let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + let base = self.graph.pointer(0, self.layout.bytes)?.cast::(); // Bounds, actual alignment and every reserved host entry are checked // before transferring aliases or consuming the caller-owned plan. self.graph.word(0)?; @@ -972,7 +1145,8 @@ impl Manager { { return Err(EIO); } - if !candidate.before.permits(self.cursors()?, firmware_request) { + if !candidate.before.permits_capacity(self.cursors()?, firmware_request, + self.tvb.geometry.capacity) { return Err(EAGAIN); } // The callback retains its input on error. The caller can therefore @@ -1013,13 +1187,13 @@ impl Manager { unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; }; for (slot, first) in added.into_iter() { - let at = fw::BLOCK_TABLE + slot * size_of::(); + let at = self.layout.blocks + slot * size_of::(); write(at, first); write(at + 4, 0); } if descriptor.is_some() { for (entry, value) in rebuilt.into_iter().enumerate() { - write(fw::PAGE_LIST + entry * 4, value); + write(self.layout.pages + entry * 4, value); } write(core::mem::offset_of!(fw::State, pages), pages); write(core::mem::offset_of!(fw::State, producer), plan.target()); @@ -1027,9 +1201,9 @@ impl Manager { write(core::mem::offset_of!(fw::State, last_page), pages - 1); } append_reserved(&mut self.tvb.extensions, extension); - write(fw::CONTROL, plan.target()); + write(self.layout.control, plan.target()); fence(Ordering::SeqCst); - write(fw::CONTROL + 4, plan.target()); + write(self.layout.control + 4, plan.target()); fence(Ordering::SeqCst); Ok(descriptor) } @@ -1069,7 +1243,7 @@ impl Manager { let before = self.cursors()?; let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; - if !before.valid() || before.total != before.committed || retained != blocks { + if !before.valid(cfg::TVB_BLOCK_SLOTS) || before.total != before.committed || retained != blocks { return Ok(None); } // Allocation failure only postpones the shrink to a later pass. @@ -1158,8 +1332,10 @@ impl BufferIds { } impl MetricsIds { pub(crate) fn used(&self) -> u32 { + let reserved = if self.scene_stride == size_of::() { 128 - fw::METRICS_SLOTS as u32 } + else { 0 }; self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() - - (128 - fw::METRICS_SLOTS as u32) + - reserved } } diff --git a/drivers/gpu/drm/asahi/g17/buffer/geometry.rs b/drivers/gpu/drm/asahi/g17/buffer/geometry.rs new file mode 100644 index 0000000000000..c871f470c1a15 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/geometry.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer backing geometry. Wire records carry each block's address, +//! so allocation stride and the reserved trailing guard are host policy. + +use crate::hw::t8140::resources as cfg; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct BlockGeometry { + pub(crate) base: u64, + pub(crate) end: u64, + pub(crate) block_bytes: usize, + pub(crate) backing_stride: usize, + pub(crate) stride: usize, + pub(crate) capacity: usize, +} + +impl BlockGeometry { + pub(crate) const PAGE: usize = 0x8000; + /// M5 parameter resources use 32-bit offsets from this compact base. + pub(crate) const M5_COMPACT: core::ops::Range = 0x10_0000_0000..0x11_0000_0000; + pub(crate) const NEO: Self = Self { + base: cfg::USER_BASE, + end: cfg::USER_END, + block_bytes: cfg::TVB_BLOCK_SIZE, + backing_stride: cfg::TVB_BLOCK_STRIDE as usize, + stride: cfg::TVB_BLOCK_STRIDE as usize, + capacity: cfg::TVB_BLOCK_SLOTS, + }; + + pub(crate) fn m5(memory_bytes: u64, initial_bytes: u64) -> Result { + let capacity = crate::g17::fw::m5::pool::parameter_capacity( + memory_bytes, initial_bytes, + ).map_err(|_| EINVAL)?; + // Retain one unallocated inventory slot, as in the shared ring manager. + if capacity < 2 || capacity.checked_mul(16).is_none_or(|n| n > 0x3f_ffff) { + return Err(EINVAL); + } + Ok(Self { + base: Self::M5_COMPACT.start, + end: Self::M5_COMPACT.end, + block_bytes: 0x80000, + backing_stride: 0x80000, + stride: 0x88000, + capacity, + }) + } + + pub(crate) const fn pages(self) -> usize { self.block_bytes / Self::PAGE } + pub(crate) const fn max_blocks(self) -> usize { self.capacity - 1 } + + pub(crate) fn validate(self) -> Result { + if self.capacity < 2 || self.capacity > u32::MAX as usize + || self.base >= self.end || self.base % Self::PAGE as u64 != 0 + || self.block_bytes == 0 || self.block_bytes % Self::PAGE != 0 + || self.backing_stride < self.block_bytes || self.backing_stride % Self::PAGE != 0 + || self.stride <= self.block_bytes || self.stride % Self::PAGE != 0 { + return Err(EINVAL); + } + Ok(()) + } + + pub(crate) fn backing_bytes(self, blocks: usize) -> Result { + self.validate()?; + if blocks == 0 || blocks > self.max_blocks() { return Err(EINVAL); } + (blocks - 1).checked_mul(self.backing_stride) + .and_then(|n| n.checked_add(self.block_bytes)).ok_or(EOVERFLOW) + } + + pub(crate) fn compact_page(self, address: u64) -> Result { + let relative = address.checked_sub(self.base).ok_or(EINVAL)?; + if relative % Self::PAGE as u64 != 0 + || address.checked_add(self.block_bytes as u64).ok_or(EOVERFLOW)? > self.end { + return Err(EINVAL); + } + (relative / Self::PAGE as u64).try_into().map_err(|_| EOVERFLOW) + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/layout.rs b/drivers/gpu/drm/asahi/g17/buffer/layout.rs new file mode 100644 index 0000000000000..ca4ad54750658 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/layout.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Wire geometry for the shared parameter-buffer owner and growth machinery. + +use super::{fw, BlockGeometry}; +use kernel::prelude::*; + +#[derive(Clone, Copy)] +pub(super) struct Layout { + pub(super) pages: usize, + pub(super) page_bytes: usize, + pub(super) control: usize, + pub(super) counter: usize, + pub(super) scenes: usize, + pub(super) blocks: usize, + pub(super) bytes: usize, + pub(super) scratch_slots: usize, + pub(super) scratch_stride: usize, + pub(super) metric_stride: usize, + pub(super) normal_blocks: usize, +} + +fn align(value: usize, alignment: usize) -> Result { + value.checked_add(alignment - 1).map(|n| n & !(alignment - 1)).ok_or(EOVERFLOW) +} + +impl Layout { + pub(super) const NEO: Self = Self { + pages: fw::PAGE_LIST, page_bytes: fw::PAGE_LIST_BYTES, + control: fw::CONTROL, counter: fw::COUNTER, scenes: fw::SCENE_TABLE, + blocks: fw::BLOCK_TABLE, bytes: fw::SIZE, + scratch_slots: fw::SCRATCH_SLOTS, scratch_stride: fw::SCRATCH_STRIDE as usize, + metric_stride: 4, normal_blocks: fw::NORMAL_MAX_BLOCKS, + }; + + pub(super) fn m5(geometry: BlockGeometry) -> Result { + geometry.validate()?; + let blocks = 0x4000; + let pages = align(geometry.capacity.checked_mul(8) + .and_then(|n| n.checked_add(blocks)).ok_or(EOVERFLOW)?, 0x4000)?; + let page_bytes = align(geometry.capacity.checked_mul(geometry.pages()) + .and_then(|n| n.checked_mul(4)).ok_or(EOVERFLOW)?, 0x4000)?; + let scratch_stride = crate::g17::fw::m5::pool::parameter_scratch_stride( + geometry.capacity, 0).map_err(|_| EINVAL)?; + Ok(Self { + pages, page_bytes, control: 0x300, counter: 0x200, + scenes: 0x400, blocks, bytes: pages.checked_add(page_bytes).ok_or(EOVERFLOW)?, + scratch_slots: fw::SCENES, scratch_stride, metric_stride: 64, + normal_blocks: geometry.capacity / 3, + }) + } + + pub(super) fn initialize(self, geometry: BlockGeometry, args: &fw::GraphArgs<'_>, + out: &mut [u8]) -> Result { + geometry.validate()?; + if out.len() < self.bytes || args.generation == 0 || args.blocks.is_empty() + || args.blocks.len() > geometry.max_blocks() { return Err(EINVAL); } + let scratch = args.scratch.checked_sub(geometry.base).ok_or(EINVAL)?; + let discard = args.discard.checked_sub(geometry.base).ok_or(EINVAL)?; + let scratch_end = scratch.checked_add((self.scratch_slots * self.scratch_stride) as u64) + .ok_or(EOVERFLOW)?; + if scratch_end > u32::MAX as u64 || discard > u32::MAX as u64 { return Err(EINVAL); } + let pages = u32::try_from(args.blocks.len().checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + let maximum = u32::try_from(geometry.capacity.checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + let normal = u32::try_from(self.normal_blocks.checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + // Every object is unpublished here. Keep all unused words zero on reuse. + out[..self.bytes].fill(0); + fn put(out: &mut [u8], at: usize, value: [u8; N]) -> Result { + out.get_mut(at..at.checked_add(N).ok_or(EOVERFLOW)?).ok_or(EINVAL)? + .copy_from_slice(&value); + Ok(()) + } + let address = |offset: usize| args.fw_va.checked_add(offset as u64).ok_or(EOVERFLOW); + for (index, &block) in args.blocks.iter().enumerate() { + let first = geometry.compact_page(block)?; + put(out, self.blocks + index * 8, first.to_le_bytes())?; + for page in 0..geometry.pages() { + put(out, self.pages + (index * geometry.pages() + page) * 4, + first.checked_add(page as u32).ok_or(EOVERFLOW)?.to_le_bytes())?; + } + } + for index in 0..fw::SCENES { + let at = self.scenes + index * size_of::(); + let metric = (index * self.metric_stride) as u64; + put(out, at, args.metrics.client.checked_add(metric).ok_or(EOVERFLOW)?.to_le_bytes())?; + put(out, at + 8, args.metrics.firmware.checked_add(metric).ok_or(EOVERFLOW)?.to_le_bytes())?; + put(out, at + 0x28, (scratch + (index % self.scratch_slots * self.scratch_stride) as u64).to_le_bytes())?; + put(out, at + 0x40, address(self.control + 0x40)?.to_le_bytes())?; + } + put(out, self.scenes + fw::FIRST_SCENE * size_of::() + 0x4c, args.generation.to_le_bytes())?; + for (at, value) in [ + (0xc, args.buffer_id), (0x30, u32::try_from(self.page_bytes)?), (0x34, pages), + (0x38, u32::try_from(geometry.capacity)?), (0x3c, args.blocks.len() as u32), + (0x54, pages - 1), (0x58, u32::try_from(geometry.block_bytes)?), + (0x7c, maximum), (0x80, normal), (self.control, args.blocks.len() as u32), + (self.control + 4, args.blocks.len() as u32), (self.control + 0x60, 1), + ] { put(out, at, value.to_le_bytes())?; } + for (at, value) in [ + (0x20, address(self.pages)?), (0x28, args.page_list_client), + (0x44, address(self.blocks)?), (0x4c, address(self.control)?), + (0x64, address(self.counter)?), (0x84, discard), + ] { put(out, at, value.to_le_bytes())?; } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index bd96ecc9d460e..8561053a336f2 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -39,6 +39,7 @@ const GTP6_SIZE: usize = GTP_SLOTS /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { + pub(crate) tilemap_size: usize, pub(crate) layer_offset: usize, pub(crate) heap_offset: usize, pub(crate) used: usize, @@ -49,6 +50,8 @@ pub(crate) struct Layout { gtp: [Option; 8], /// Bytes of the TPC object the command uses. tpc_end: usize, + /// The M5 tiler slices of the TPC object and their cluster controls. + m5: Option<(crate::g17::fw::m5::target::TargetSlices, [u64; 3])>, } impl Layout { pub(crate) fn new( @@ -134,6 +137,7 @@ impl Layout { tpc_end = position; } Ok(Self { + tilemap_size, layer_offset, heap_offset, used, @@ -141,9 +145,34 @@ impl Layout { tvb_blocks, gtp, tpc_end, + m5: None, }) } + pub(crate) fn with_m5_slices(mut self, pass: &RenderPass, clusters: u32, groups: u32, + segments: u32, parameter_bytes: usize) -> Result { + let slices = crate::g17::fw::m5::target::TargetSlices::new( + self.tilemap_size, self.tpc_size, pass.layers, groups, segments, parameter_bytes) + .ok_or(EINVAL)?; + // The last word is the part's minimum parameter-buffer size, whatever + // the group count. + let control = crate::g17::fw::m5::target::cluster_control(groups, + crate::g17::fw::m5::target::min_parameter_buffer_pages(clusters)) + .ok_or(EINVAL)?; + self.m5 = Some((slices, control)); + Ok(self) + } + pub(crate) fn is_m5(&self) -> bool { self.m5.is_some() } + fn check_capacity(&self, stride: usize) -> Result { + if self.m5.is_none() && stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + stride.checked_mul(DEPTH).ok_or(EOVERFLOW)?; + Ok(()) + } pub(crate) fn tpc_capacity(&self) -> Result { + if let Some((slices, _)) = self.m5 { + return aligned(slices.bytes, mmu::UAT_PGSZ); + } // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object // also contains GTP sub-buffers, notably 64 times the tile map. Applying // the cache limit to their combined extent rejects valid large targets. @@ -157,7 +186,8 @@ impl Layout { Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { - Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) + let bytes = aligned(self.used, mmu::UAT_PGSZ)?; + Ok(if self.m5.is_some() { bytes } else { bytes.max(cfg::TARGET_RECORD_MIN) }) } } @@ -167,10 +197,15 @@ struct Object { backing: KernelObject, } impl Object { - fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize) -> Result { + fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize, m5: bool) -> Result { let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; let prot = mmu::PROT_GPU_FW_SHARED_RW; - let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; + let mapping = if m5 { + backing.map_range(vm, 0..size, super::BlockGeometry::M5_COMPACT, + cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)? + } else { + compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)? + }; Ok(Self { address: mapping.iova(), _mapping: mapping, @@ -196,6 +231,7 @@ pub(crate) struct Target { stride: usize, slots: [Option; DEPTH], retired: KVec, + m5: bool, } impl Target { /// Keep this allocation frame separate from the rest of the render graph. @@ -203,11 +239,10 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } - let records = Object::new(dev, vm, stride * DEPTH)?; - let tpc = Object::new(dev, vm, tpc)?; + layout.check_capacity(stride)?; + let m5 = layout.m5.is_some(); + let records = Object::new(dev, vm, stride * DEPTH, m5)?; + let tpc = Object::new(dev, vm, tpc, m5)?; KBox::new( Self { records, @@ -215,6 +250,7 @@ impl Target { stride, slots: [None; DEPTH], retired: KVec::new(), + m5, }, GFP_KERNEL, ) @@ -224,6 +260,27 @@ impl Target { self.tpc.address } + /// The M5 tiling words naming the TPC object's slices of `layout`. + pub(crate) fn m5_extra(&self, layout: &Layout) + -> Result { + if !self.m5 { return Err(EINVAL); } + let (slices, control) = layout.m5.ok_or(EINVAL)?; + if slices.bytes > self.tpc.backing.size() { return Err(ERANGE); } + let base = self.tpc.address.checked_sub(super::BlockGeometry::M5_COMPACT.start) + .ok_or(EINVAL)?; + let mut pointers = [0; 8]; + for index in 0..8 { + if slices.sizes[index] != 0 { + pointers[index] = base.checked_add(slices.offsets[index] as u64).ok_or(EOVERFLOW)?; + } + } + Ok(crate::g17::fw::m5::render_registers::TaExtra { + gtp1: pointers[1], gtp2: pointers[2], gtp4: pointers[4], gtp5: pointers[5], + gtp6: pointers[6], gtp7: pointers[7], segment_control: control[0], segment_index: control[1], + min_free_tvb_pages: control[2], scs_bracket: 0, matrix_bracket: 0, + }) + } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty /// or absent one. The TPC object holds them after the TPC. pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { @@ -243,6 +300,7 @@ impl Target { /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { let slot = ordinal as usize % DEPTH; + if self.m5 != layout.m5.is_some() { return Err(EINVAL); } if self.slots[slot].is_some_and(|owner| owner != ordinal) { return Err(EBUSY); } @@ -285,15 +343,14 @@ impl Target { ) -> Result { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; + if self.m5 != layout.m5.is_some() { return Err(EINVAL); } + layout.check_capacity(stride)?; let mut grew = false; if tpc > self.tpc.backing.size() { - Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; + Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners, self.m5)?; grew = true; } if stride > self.stride { - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } Self::replace( &mut self.retired, &mut self.records, @@ -301,6 +358,7 @@ impl Target { vm, stride * DEPTH, owners, + self.m5, )?; self.stride = stride; self.slots = [None; DEPTH]; @@ -315,12 +373,13 @@ impl Target { vm: &mmu::Vm, size: usize, owners: &[Option; DEPTH], + m5: bool, ) -> Result { let retain = owners.iter().any(Option::is_some); if retain { retired.reserve(1, GFP_KERNEL)?; } - let new = Object::new(dev, vm, size)?; + let new = Object::new(dev, vm, size, m5)?; let previous = core::mem::replace(old, new); if retain { super::append_reserved( diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index da02c6bf21da9..d65ed18b6c786 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -58,19 +58,20 @@ enum ReportRing { } impl ReportRing { - fn geometry(self) -> (usize, usize, u32, u32) { + /// State and entry offsets, capacity and the record tags the family allows. + fn geometry(self, init: &InitData) -> (usize, usize, u32, u64) { match self { Self::Trace => ( status::TRACE_STATE as usize, status::TRACE_RING as usize, TRACE_SLOTS, - TRACE_TYPES, + u64::from(TRACE_TYPES), ), Self::Statistics => ( status::STATS_STATE as usize, status::STATS_RING as usize, STATISTICS_SLOTS, - STATISTICS_TYPES, + init.statistics_types(), ), } } @@ -243,7 +244,7 @@ impl Rings { accept_statistics: bool, statistics_cutoff: &mut Option, ) -> Result { - let (state, entries, capacity, allowed_types) = ring.geometry(); + let (state, entries, capacity, allowed_types) = ring.geometry(init); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; fence(Ordering::Acquire); @@ -272,12 +273,12 @@ impl Rings { let words = init.status_read_words::(role, offset)?; super::trace::report(role as u8, ring as u8, &words); let tag = words[0]; - if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + if tag >= u64::BITS || allowed_types & (1u64 << tag) == 0 { return Err(EIO); } if matches!(ring, ReportRing::Statistics) { let admitted = accept_statistics && statistics_cutoff.is_none(); - reports.record(role, words, host_ns, admitted); + reports.record(role, words, host_ns, allowed_types, admitted); if admitted { telemetry.record(&words, host_ns, max_age_ns); } @@ -318,7 +319,7 @@ impl Rings { init.publish_control( producer, record.bytes(), - record.opcode().asserts_gpu_power(), + init.control_asserts_gpu_power(record.opcode()), next, )?; CONTROL_PUBLISHED.fetch_add(1, Ordering::AcqRel); @@ -385,7 +386,7 @@ impl Events { let words = init.status_read_words(role, offset)?; super::trace::report(role as u8, 2, &words); let record = EventRecord::from_words(words); - let event = record.decode_for(self.graph); + let event = record.decode_for(self.graph, init.is_m5()); fence(Ordering::SeqCst); consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index c65f7de6ac915..140d0ba9d3bac 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -10,7 +10,8 @@ use super::fw::{ }; use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; -use kernel::{prelude::*, uapi}; +use kernel::{prelude::*, sync::Arc, uapi}; +use super::render_resources::{ComputePrograms, Programs}; const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); @@ -29,14 +30,22 @@ const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; const MAX_DIMENSION: u16 = 16384; const MAX_LAYERS: u16 = 2048; -const MAX_TILE_BUFFER: u64 = 32768; +pub(crate) const MAX_TILE_BUFFER: u64 = 32768; +// Eight RGBA8 render targets take 64 bytes a pixel across a 32x32 tile. +pub(crate) const M5_MAX_TILE_BUFFER: u64 = 65536; const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; +const STAGE_PREFIXES: u32 = crate::file::DRM_ASAHI_RENDER_STAGE_PREFIXES; +const WHOLE_ATTACHMENTS: u32 = crate::file::DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS; +const SHADER_WRITES: u32 = crate::file::DRM_ASAHI_RENDER_SHADER_WRITES; +/// Render flags only the M5 runtime implements. +const M5_RENDER_FLAGS: u32 = SHADER_WRITES | STAGE_PREFIXES | WHOLE_ATTACHMENTS; const RENDER_FLAGS: u32 = - PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; + PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS | SHADER_WRITES + | STAGE_PREFIXES | WHOLE_ATTACHMENTS; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -82,13 +91,85 @@ impl Attachments { } Ok(()) } + + /// An M5 render's lists: the firmware never sees them (the pass's tile stores are encoded from + /// its surfaces), so each range is only the host's ordering hint, its footprint + /// (`Vm::render_footprint`). A range may then cover several mappings and the gaps between them: + /// a coarser footprint only orders more. It must still be a range. + fn validate_hint(&self) -> Result { + for range in self.as_slice() { + if range.size == 0 || range.va.checked_add(range.size).is_none() { + return Err(EINVAL); + } + } + Ok(()) + } + + fn validate_as(&self, space: &impl AddressSpace, rule: AttachmentRule) -> Result { + match rule { + AttachmentRule::Mapped => self.validate(space), + AttachmentRule::Hint => self.validate_hint(), + } + } +} + +/// What a render's attachment lists must prove. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AttachmentRule { + /// Every range is mapped writable: the firmware reads the lists. + Mapped, + /// Every range is a non-empty range that does not wrap: the lists only order renders (M5). + Hint, +} + +/// Where a render command's stage-prefix fields start: after the complete render command. +const RENDER_STAGE_PREFIXES: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render_m5, stage_prefixes); +// The wire form M5 Mesa sends: the 448-byte command, then 32 bytes of prefixes. +const _: () = { + assert!(RENDER_STAGE_PREFIXES == size_of::()); + assert!(size_of::() == 448); + assert!(size_of::() == 480); +}; + +// SAFETY: Every field is an integer or a `repr(C)` UAPI struct of integers. +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render_m5 {} + +/// A render command. With DRM_ASAHI_RENDER_STAGE_PREFIXES it is the complete +/// struct drm_asahi_cmd_render_m5, with its reserved fields zero; without it the +/// stage prefixes are absent or zero. +fn render_payload( + bytes: &[u8], +) -> Result<(uapi::drm_asahi_cmd_render, Option)> { + let full: uapi::drm_asahi_cmd_render_m5 = payload(bytes)?; + let (cmd, prefixes) = (full.legacy, full.stage_prefixes); + let staged = cmd.flags & STAGE_PREFIXES != 0; + let valid = if staged { + bytes.len() == size_of::() + && cmd.flags & FRAGMENT_BARRIERS == 0 + && prefixes.reserved0 == 0 + && prefixes.reserved1 == 0 + && prefixes.reserved2 == 0 + && prefixes.reserved3 == 0 + } else { + !bytes.get(RENDER_STAGE_PREFIXES..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) + }; + if !valid { + return Err(EINVAL); + } + Ok((cmd, staged.then_some(prefixes))) } // Commands stay inline in the submission plan, avoiding an allocation per render pass. #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { - Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2]), - Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), + Render( + uapi::drm_asahi_cmd_render, + Option, + [Attachments; 2], + [RayQueryStage; 2], + Option>, + ), + Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables, Option>), } pub(crate) struct Command { @@ -103,6 +184,8 @@ pub(crate) struct Parser<'a> { bytes: &'a [u8], counts: [u16; 2], attachments: [Attachments; 3], + render_resources: Option>, + compute_resources: Option>, } impl<'a> Parser<'a> { @@ -111,6 +194,8 @@ impl<'a> Parser<'a> { bytes, counts: [0; 2], attachments: [Attachments::EMPTY; 3], + render_resources: None, + compute_resources: None, } } @@ -138,18 +223,26 @@ impl<'a> Parser<'a> { { self.counts[0] += 1; no_timestamp_copies(bytes, RENDER_TIMESTAMP_COPIES)?; + // The ray-query stages follow prior_cdm, and the stage-prefix fields follow them. + let ray_query = bytes + .get(RENDER_BASE_SIZE..bytes.len().min(RENDER_STAGE_PREFIXES)) + .unwrap_or(&[]); + let (cmd, stage_prefixes) = render_payload(bytes)?; Payload::Render( - payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, + cmd, + stage_prefixes, [self.attachments[0], self.attachments[1]], - RayQueryStage::parse_pair(bytes.get(RENDER_BASE_SIZE..).unwrap_or(&[]))?, + RayQueryStage::parse_pair(ray_query)?, + self.render_resources.clone(), ) } else { self.counts[1] += 1; no_timestamp_copies(bytes, COMPUTE_TIMESTAMP_COPIES)?; Payload::Compute( - payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, + payload(bytes)?, self.attachments[2], IntersectionTables::parse(bytes.get(COMPUTE_BASE_SIZE..).unwrap_or(&[]))?, + self.compute_resources.clone(), ) }; return Ok(Some(Command { barriers, payload })); @@ -164,6 +257,25 @@ impl<'a> Parser<'a> { - uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS as usize; self.attachments[slot] = Attachments::parse(bytes)?; } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + self.render_resources = if bytes.is_empty() { None } + else { Some(Arc::new(Programs::parse(bytes)?, GFP_KERNEL)?) }; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2 => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { return Err(EINVAL); } + self.render_resources = if bytes.is_empty() { None } + else { Some(Arc::new(Programs::parse_v2(bytes)?, GFP_KERNEL)?) }; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + self.compute_resources = if bytes.is_empty() { None } + else { Some(Arc::new(ComputePrograms::parse(bytes)?, GFP_KERNEL)?) }; + } _ => return Err(EINVAL), } } @@ -247,7 +359,13 @@ impl UscWindow { } } -fn sampler(space: &impl AddressSpace, heap: u64, count: u32) -> Result { +/// A command's sampler heap. The M5 command set has none: its render and compute +/// programs carry no sampler-heap state, so M5 refuses a heap here, before anything is +/// queued, rather than run the command without it. +fn sampler(space: &impl AddressSpace, heap: u64, count: u32, m5: bool) -> Result { + if m5 && (heap != 0 || count != 0) { + return Err(EOPNOTSUPP); + } if count > MAX_SAMPLERS || (heap == 0) != (count == 0) || heap & (SAMPLER_SIZE - 1) != 0 { return Err(EINVAL); } @@ -311,13 +429,27 @@ pub(crate) enum Validated { long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], + resources: Option>, + plan: Option, attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { long_job: bool, intersection: IntersectionTables, + matrix_ops: bool, + /// The command may copy or reset queries, so it must follow the host's + /// timestamp publication for earlier commands. + query_access: bool, + resources: Option>, + plan: Option, scratch: ScratchRequest, + cdm_auxiliary_allocation: u32, + /// The M5 intersection-table binding (dj's `intersection`; that name is the G17P + /// tables' above). + m5_intersection: super::fw::m5::compute::IntersectionTables, + /// The M5 shader-context scratch request (`scratch` is G17P's). + m5_scratch: super::fw::m5::compute::ScratchRequest, usc_base: u64, cdm_va: u64, cdm_end_va: u64, @@ -343,8 +475,20 @@ impl Validated { impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { - Self::Render(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Render(cmd, ..) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, ..) => [cmd.prior_vdm, cmd.prior_cdm], + } + } + + /// The tiling half's own render and compute prefixes (barrier, exact prior), if the + /// command carries them. + pub(crate) fn tiling_prefixes(&self) -> Option<([u16; 2], [u64; 2])> { + match self { + Self::Render(_, Some(p), ..) => Some(( + [p.ta_vdm_barrier, p.ta_cdm_barrier], + [p.ta_prior_vdm, p.ta_prior_cdm], + )), + _ => None, } } @@ -354,9 +498,14 @@ impl Payload { variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { + // Everything M5 accepts beyond the G17P command set is gated on this. + let m5 = variant.m5_commands; + let max_tile_buffer = if m5 { M5_MAX_TILE_BUFFER } else { MAX_TILE_BUFFER }; + let attachment_rule = if m5 { AttachmentRule::Hint } else { AttachmentRule::Mapped }; match self { - Self::Render(cmd, attachments, ray_query) => { - if cmd.flags & !RENDER_FLAGS != 0 + Self::Render(cmd, _, attachments, ray_query, resources) => { + if cmd.flags & !(if m5 { RENDER_FLAGS } else { RENDER_FLAGS & !M5_RENDER_FLAGS }) != 0 + || (!m5 && resources.is_some()) || !(1..=MAX_DIMENSION).contains(&cmd.width_px) || !(1..=MAX_DIMENSION).contains(&cmd.height_px) || !(1..=MAX_LAYERS).contains(&cmd.layers) @@ -371,13 +520,14 @@ impl Payload { 1 => 0, 2 => 1, 4 => 2, + 8 if m5 => 3, _ => return Err(EINVAL), }; if u64::from(cmd.sample_size_B) * u64::from(cmd.utile_width_px) * u64::from(cmd.utile_height_px) * u64::from(cmd.samples) - > MAX_TILE_BUFFER + > max_tile_buffer || cmd.vdm_ctrl_stream_base == 0 || cmd.vdm_ctrl_stream_base & 3 != 0 || cmd.isp_scissor_base == 0 @@ -400,7 +550,7 @@ impl Payload { } let depth = surface(space, &cmd.depth, cmd.layers)?; let stencil = surface(space, &cmd.stencil, cmd.layers)?; - sampler(space, cmd.sampler_heap, cmd.sampler_count.into())?; + sampler(space, cmd.sampler_heap, cmd.sampler_count.into(), m5)?; if !helper_is_empty(&cmd.vertex_helper) || !helper_is_empty(&cmd.fragment_helper) { return Err(EINVAL); } @@ -408,7 +558,7 @@ impl Payload { window.program(space, program.usc)?; } for list in attachments { - list.validate(space)?; + list.validate_as(space, attachment_rule)?; } let pass = RenderPass { width: cmd.width_px.into(), @@ -427,6 +577,9 @@ impl Payload { depth_bias_is_int: cmd.flags & DBIAS_INT != 0, few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, + stage_prefixes: cmd.flags & STAGE_PREFIXES != 0, + whole_attachments: cmd.flags & WHOLE_ATTACHMENTS != 0, + shader_writes_memory: cmd.flags & SHADER_WRITES != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), @@ -459,7 +612,7 @@ impl Payload { }; for stage in ray_query { if (stage.intersection.enabled() || stage.scratch.enabled()) - && !variant.ray_traversal + && !(variant.ray_traversal && variant.render_ray_query) { return Err(refuse_ray_traversal()); } @@ -469,18 +622,40 @@ impl Payload { if code != 0 { require_range(space, code, 64, Access::Read)?; } } } + // M5's per-stage intersection tables, from the render's programs. + let mut m5_tables = [false; 2]; + if let Some(programs) = resources { + for (index, stage) in programs.stage_scs()?.into_iter().enumerate() { + let intersection = stage.intersection; + if intersection.enabled() { + require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; + let code = intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + m5_tables[index] = true; + } + } + } Ok(Validated::Render { long_job: super::timeout::select(false, - ray_query.map(|stage| stage.intersection.enabled()), false), + [0, 1].map(|i| ray_query[i].intersection.enabled() || m5_tables[i]), false), pass, ray_query: *ray_query, + resources: resources.clone(), + plan: None, attachments: *attachments, timestamps: [cmd.ts_vtx, cmd.ts_frag], }) } - Self::Compute(cmd, attachments, intersection) => { - let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; - if cmd.cdm_ctrl_stream_base & 3 != 0 + Self::Compute(cmd, attachments, intersection, resources) => { + let as_build = if m5 { false } else { super::timeout::as_build(cmd.flags).ok_or(EINVAL)? }; + if (m5 + && (cmd.flags + & !(crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX + | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS) + != 0 + || cmd.reserved != 0)) + || (!m5 && resources.is_some()) + || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -493,21 +668,53 @@ impl Payload { cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, Access::Read, )?; - sampler(space, cmd.sampler_heap, cmd.sampler_count)?; + sampler(space, cmd.sampler_heap, cmd.sampler_count, m5)?; attachments.validate(space)?; - let scratch = ScratchRequest::new(cmd.scs_layout)?; + // M5 shader-context scratch has its own (Gen2) layout; G17P's stays disabled. + let scratch = ScratchRequest::new(if m5 { 0 } else { cmd.scs_layout })?; + let m5_scratch = if m5 { + super::fw::m5::compute::ScratchRequest::new(cmd.scs_layout)? + } else { + super::fw::m5::compute::ScratchRequest::NONE + }; if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { return Err(refuse_ray_traversal()); } + // The auxiliary allocation is the per-USC page count of the dispatch's SCS + // storage, which the pool is sized for (`plan_compute_scs`). Without SCS there + // is no such storage, as render's programs require. + if !m5_scratch.enabled() && cmd.cdm_auxiliary_allocation != 0 { + return Err(EINVAL); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } + let m5_intersection = if m5 { + super::fw::m5::compute::IntersectionTables::new( + cmd.intersection_pool, cmd.intersection_pool_bytes, + cmd.intersection_first, cmd.intersection_count, + cmd.intersection_default_program, cmd.intersection_default_control)? + } else { + super::fw::m5::compute::IntersectionTables::NONE + }; + if m5_intersection.enabled() { + require_range(space, m5_intersection.pool, m5_intersection.pool_bytes, Access::Read)?; + let code = m5_intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } Ok(Validated::Compute { - long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), + long_job: super::timeout::select(intersection.enabled() || m5_intersection.enabled(), + [false; 2], as_build), intersection: *intersection, + matrix_ops: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX != 0, + query_access: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS == 0, + resources: resources.clone(), plan: None, scratch, + m5_scratch, + cdm_auxiliary_allocation: cmd.cdm_auxiliary_allocation, + m5_intersection, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -525,3 +732,42 @@ fn refuse_ray_traversal() -> Error { kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) } + +#[kunit_tests(asahi_g17_attachments)] +mod attachment_tests { + use super::*; + + /// A VM with nothing mapped. + struct Unmapped; + + impl AddressSpace for Unmapped { + fn covers(&self, _address: u64, _size: u64, _access: Access) -> bool { + false + } + } + + fn list(ranges: &[(u64, u64)]) -> Attachments { + let mut list = Attachments::EMPTY; + for (slot, &(va, size)) in list.ranges.iter_mut().zip(ranges) { + *slot = WriteRange { va, size }; + list.count += 1; + } + list + } + + #[test] + fn an_m5_hint_may_span_unmapped_gaps() { + // One range over two attachments and the gap between them, as userspace merges more + // attachments than a list holds. + let merged = list(&[(0x1_0000_0000, 0x40_0000), (0x2_0000_0000, 0x1_0000_0000)]); + assert_eq!(merged.validate_as(&Unmapped, AttachmentRule::Hint), Ok(())); + assert_eq!(merged.validate_as(&Unmapped, AttachmentRule::Mapped), Err(EINVAL)); + } + + #[test] + fn an_m5_hint_is_still_a_range() { + assert_eq!(list(&[(0x1_0000_0000, 0)]).validate_hint(), Err(EINVAL)); + assert_eq!(list(&[(u64::MAX - 0xfff, 0x2000)]).validate_hint(), Err(EINVAL)); + assert_eq!(list(&[]).validate_hint(), Ok(())); + } +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 0f027e7e06828..801a8aca22144 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -254,6 +254,31 @@ impl Drop for PooledTimestamps { /// Republications of one command whose earlier publications firmware dropped unexecuted. pub(crate) const MAX_REPLAYS: u8 = 3; +/// M5 timestamp pairs share a retained page of the physical compute queue. +/// A completion keeps its backing alive through quarantine and replay. +struct M5ComputeTimestamps { + object: Arc, + offset: usize, +} + +impl M5ComputeTimestamps { + fn clear(&self) -> Result { + let start = self.object.dword(self.offset)?; + let end = self.object.dword(self.offset + 8)?; + start.store(0, Ordering::Relaxed); + end.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + Ok(()) + } + + fn words(&self) -> Result<[u64; 2]> { + let words = [self.object.dword(self.offset)?.load(Ordering::Relaxed), + self.object.dword(self.offset + 8)?.load(Ordering::Relaxed)]; + barrier(Ordering::Acquire); + Ok(words) + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -262,6 +287,11 @@ pub(crate) struct Completion { // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, compute: Option, + #[pin] + m5_compute: Mutex>, + /// A compute command. M5 compute binds its timestamps at publication, so + /// `compute` alone does not identify the engine. + is_compute: bool, feed: Option>, usage: Arc, sources: Option>, @@ -320,7 +350,9 @@ impl Completion { } else { HostOutput::Immediate(output) }; - let compute = if compute { + // M5 compute binds its timestamp pair in its physical queue's page at publication. + let is_compute = compute; + let compute = if compute && !context.is_m5() { Some(context.compute_timestamps().take(dev, context.vm())?) } else { None @@ -331,6 +363,8 @@ impl Completion { output, terminal, compute, + m5_compute <- new_mutex!(None, "M5 queue timestamps"), + is_compute, feed, usage, sources, @@ -429,14 +463,33 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) + if let Some(compute) = &self.compute { return Ok(compute.page()?.mapping.iova()); } + let compute = self.m5_compute.lock(); + let compute = compute.as_ref().ok_or(EINVAL)?; + Ok(compute.object.gpu_va() + compute.offset as u64) + } + + /// The queue checked its record-slot retirement before calling this. + /// Clearing and binding precede descriptor publication under the device lock. + pub(crate) fn bind_m5_compute_timestamps(&self, object: &Arc, offset: usize) -> Result { + if self.compute.is_some() || self.signalled.load(Ordering::Acquire) { + return Err(EINVAL); + } + let timestamps = M5ComputeTimestamps { object: object.clone(), offset }; + timestamps.clear()?; + let address = timestamps.object.gpu_va() + offset as u64; + *self.m5_compute.lock() = Some(timestamps); + Ok(address) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let Some(compute) = self.compute.as_ref() else { + return self.m5_compute.lock().as_ref().ok_or(EINVAL)?.words(); + }; + let object = &compute.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -452,6 +505,10 @@ impl Completion { self.compute_words().ok() } + pub(crate) fn signalled(&self) -> bool { + self.signalled.load(Ordering::Acquire) + } + pub(crate) fn begin_replay(&self) -> bool { let accepted = self .replays @@ -489,7 +546,10 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let Some(compute) = self.compute.as_ref() else { + return self.m5_compute.lock().as_ref().ok_or(EINVAL)?.clear(); + }; + let object = &compute.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -656,7 +716,21 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + // M5 compute feeds the refill detector only from its exact execution record + // (RetiredCompute); M5 compute is the compute without pooled timestamps. + let execution = if self.is_compute && self.compute.is_none() { + None + } else { + result.as_ref().ok().map(|ticks| [ticks[0], ticks[1]]) + }; + self.complete_with_compute(result, execution); + } + + /// Compute feedback is separate from public timestamps, which may contain + /// a synthesized retirement envelope when the execution record is absent. + #[track_caller] + fn complete_with_compute(&self, mut result: Result<[u64; 4]>, compute: Option<[u64; 2]>) { if result.is_ok() { if let Err(error) = self.check_source_inputs() { result = Err(error); @@ -671,7 +745,7 @@ impl Completion { if let Ok(ticks) = &result { use crate::usage::Engine; let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; - if self.compute.is_some() { + if self.is_compute { self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); } else { self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); @@ -681,8 +755,10 @@ impl Completion { if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. - if self.compute.is_some() { - feed.note_pass(ticks[0], ticks[1]); + if self.is_compute { + if let Some([start, end]) = compute { + feed.note_pass(start, end); + } } else { feed.note_pass(ticks[2], ticks[3]); } @@ -745,6 +821,8 @@ fn discarded(error: Error) -> bool { /// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. pub(crate) enum Deferred { Retired(Arc, Result<[u64; 4]>), + /// Only the optional interval comes from an exact compute execution record. + RetiredCompute(Arc, [u64; 4], Option<[u64; 2]>), Drained(Arc, Result<[u64; 4]>), FailedOwned(Arc, Error), Unpublished(Arc, Error), @@ -754,6 +832,10 @@ pub(crate) enum Deferred { impl Deferred { pub(crate) fn finish(self) { match self { + Self::RetiredCompute(completion, ticks, execution) => { + completion.complete_with_compute(Ok(ticks), execution); + completion.release_after_retirement(); + } Self::Retired(completion, result) => { completion.complete(result); // An earlier failure may already have signalled this fence while diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 87572f7eb102f..1f953697dd6b7 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -16,6 +16,10 @@ use super::{ use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; + +/// Both of a context's render channels are bound to other pairs whose publications have not +/// all retired: the publisher waits for that retirement. +pub(crate) const RENDER_CHANNEL_PENDING: Error = ERECALLCONFLICT; use super::job::Engine; use kernel::{ new_mutex, @@ -27,6 +31,8 @@ const QOS_IDS: usize = 128; struct QosState { used: u128, + /// Released IDs whose firmware QoS rows still need clearing. + released: u128, /// When each ID was last returned; reuse takes the one returned longest ago. returns: [u64; QOS_IDS], /// Firmware progress a returned ID waits for (see `channel::control_fence`). @@ -47,7 +53,7 @@ impl QosIds { Arc::pin_init( pin_init!(Self { state <- new_mutex!( - QosState { used: 3, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 }, + QosState { used: 3, released: 0, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 }, "G17 QoS IDs" ) }), @@ -65,6 +71,8 @@ impl QosIds { return Err(super::capacity::refused(super::capacity::Site::QosIds)); }; state.used |= 1u128 << id; + // A new owner writes its own scheduler row at its first publication. + state.released &= !(1u128 << id); id }; // Drop returns the ID if the allocation fails. @@ -98,6 +106,7 @@ impl Drop for QosLease { state.returns[id] = state.clock; state.fences[id] = fence; state.used &= !(1u128 << id); + state.released |= 1u128 << id; } } @@ -157,10 +166,35 @@ impl WorkStorage { } } +/// A context's scheduler page. The M5 firmware caches its mapping, so on M5 it +/// comes from the retention pool and stays mapped until firmware stop; elsewhere +/// it is an ordinary allocation. +pub(crate) enum SchedulerStorage { + Ordinary(KernelObject), + Retained(PooledObject), +} +impl core::ops::Deref for SchedulerStorage { + type Target = KernelObject; + fn deref(&self) -> &KernelObject { + match self { + Self::Ordinary(object) => object, + Self::Retained(object) => object, + } + } +} +impl core::ops::DerefMut for SchedulerStorage { + fn deref_mut(&mut self) -> &mut KernelObject { + match self { + Self::Ordinary(object) => object, + Self::Retained(object) => object, + } + } +} + /// Firmware-visible pages of a context, retained by an installed queue whose /// last owner has exited (see [`Context::retain_firmware_pages`]). pub(crate) struct FirmwarePages { - _scheduler: Arc, + _scheduler: Arc, _work: Arc>, /// The queue record still names the context's execution identity and QoS ID. _hold: Arc, @@ -168,11 +202,13 @@ pub(crate) struct FirmwarePages { } pub(crate) struct Context { + /// The GPU runs the M5 firmware's protocol. + m5: bool, execution: Arc, /// This context's own hold on its execution identity, cloned by retained queue records. hold: Arc, status: Arc, - scheduler: Arc, + scheduler: Arc, scheduler_generation: u32, published: AtomicBool, /// A kill of this context was published, through a compute queue or on its render pairs' @@ -221,12 +257,30 @@ impl Context { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let hold = execution.hold(super::channel::control_fence)?; let qos = qos.acquire()?; - let mut scheduler = alloc.kernel( - UAT_PGSZ, - UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; + // On M5 only the firmware reads and writes the scheduler and work-state pages; the GPU + // has no access to them. + let kernel_prot = if alloc.uat.is_m5() { + mmu::PROT_FW_SHARED_RW + } else { + mmu::PROT_GPU_FW_SHARED_RW + }; + let mut scheduler = if alloc.uat.is_m5() { + SchedulerStorage::Retained(pool.allocate( + alloc, + Family::Scheduler, + UAT_PGSZ, + UAT_PGSZ, + kernel_prot, + CpuMap::WriteCombined, + )?) + } else { + SchedulerStorage::Ordinary(alloc.kernel( + UAT_PGSZ, + UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?) + }; scheduler.initialize::(0, |value| { value.init(); Ok(()) @@ -239,7 +293,7 @@ impl Context { Family::WorkState, 2 * UAT_PGSZ, UAT_PGSZ, - mmu::PROT_GPU_FW_SHARED_RW, + kernel_prot, CpuMap::WriteCombined, )?; let work_base = object.gpu_va(); @@ -269,6 +323,7 @@ impl Context { let order = Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { + m5: alloc.uat.is_m5(), execution, hold, status, @@ -327,6 +382,24 @@ impl Context { pub(crate) fn is_current(&self) -> bool { self.execution.is_current() } + /// Claim the context's hardware identity for one published M5 command until it retires. + pub(crate) fn pin(&self) -> Result { + self.execution.pin() + } + pub(crate) fn unpin(&self, pin: mmu::ContextPin) { + self.execution.unpin(pin) + } + /// Claim the queue's QoS slot for one published M5 command. Stage 1: the slot is the + /// context's own, taken at creation. + pub(crate) fn pin_qos(&self) -> Result { + Ok(self.qos_id()) + } + pub(crate) fn unpin_qos(&self, _slot: u8) {} + + pub(crate) fn is_m5(&self) -> bool { + self.m5 + } + pub(crate) fn owner_pid(&self) -> u32 { self.owner_pid } @@ -341,7 +414,7 @@ impl Context { } /// Installed physical channels retain this page after the logical execution /// context and its VM lease have completed their scheduler release. - pub(crate) fn scheduler_owner(&self) -> Arc { + pub(crate) fn scheduler_owner(&self) -> Arc { self.scheduler.clone() } /// Scheduler release and command retirement have both been witnessed. @@ -442,21 +515,44 @@ impl Context { alloc: &Allocator<'_>, ids: &Arc, render_global: &Vm, + plan: Option, ) -> Result> { if !self.is_current() { return Err(EFAULT); } + if alloc.uat.is_m5() != plan.is_some() { + return Err(EINVAL); + } let mut render = self.render.lock(); if let Some(pool) = &*render { - // A pool vacated under memory pressure is re-armed before reuse. - pool.ensure_backing(alloc, render_global, self.vm())?; - return Ok(pool.clone()); + if pool.fits(plan.as_ref().map(|p| p.geometry)) { + // A pool vacated under memory pressure is re-armed before reuse. + // M5 backing is installed by the pair that pins the pool. + if !alloc.uat.is_m5() { + pool.ensure_backing(alloc, render_global, self.vm())?; + } + return Ok(pool.clone()); + } + // M5: existing pairs retain the old pool through their rebind and + // its release acknowledgement. Allocation failure leaves it installed. } let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())?, + match plan { + Some(plan) => RenderPool::m5( + alloc, + super::freelist::RenderId::logical(ids)?, + self.vm(), + plan.geometry, + )?, + None => { + RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())? + } + }, GFP_KERNEL, )?; - *render = Some(pool.clone()); + let previous = render.replace(pool.clone()); + drop(render); + drop(previous); Ok(pool) } /// Drops the pool once its free-list release is consumed and the closed @@ -484,17 +580,20 @@ pub(crate) struct WorkStateLease { } impl WorkStateLease { pub(super) fn status(&self) -> &Arc { self.context.status() } - pub(crate) fn deferred(context: Arc) -> Self { - let ordinal = context.order.lock().assign(); - Self { + /// The lease of a new ioctl, ordered after every earlier one of its context. EBUSY while + /// too many of the context's ioctls are queued behind its oldest waiting one + /// (see `identity::Order`). + pub(crate) fn deferred(context: Arc) -> Result { + let ordinal = context.order.lock().assign().ok_or(EBUSY)?; + Ok(Self { context, word: AtomicU32::new(FREE), engine: AtomicU8::new(0), ordinal, - } + }) } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many - /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: an older + /// ioctl still waits for its node (see `identity::Order`), or no node may be taken now /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); @@ -535,6 +634,15 @@ impl WorkStateLease { pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } + /// The oldest node of this ioctl's context still in use ahead of this ioctl's node, whose + /// draining the firmware's flush of this node waits for; none without a node of its own. + pub(crate) fn older_node(&self) -> Option { + let word = self.word(); + if word == FREE { + return None; + } + self.context.states.older_live(word) + } pub(crate) fn node_va(&self) -> Result { let word = self.word(); if word == FREE { @@ -565,9 +673,35 @@ impl WorkStateLease { .change_count(self.word(), kicks, false) } - pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or_else(|| - super::capacity::refused(super::capacity::Site::RenderWorkSlots)) + /// The render channel of `pair` for a publication. When both channels are bound to other + /// pairs, the least recently used one whose every publication `retired(qids, ordinal)` + /// moves to `pair`; otherwise [`RENDER_CHANNEL_PENDING`]: the caller waits for retirement + /// instead of failing the client. + pub(crate) fn render_channel( + &self, + pair: u8, + retired: impl Fn([u8; 2], u64) -> bool, + ) -> Result { + let (index, previous) = self + .context + .work + .lock() + .render + .bind(pair, retired) + .ok_or(RENDER_CHANNEL_PENDING)?; + if let Some(previous) = previous { + static REBINDS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + let n = REBINDS.fetch_add(1, Ordering::Relaxed) + 1; + if n <= 8 || n.is_power_of_two() { + pr_info!("asahi: render channel {} of context {} moved from pair {} to pair {} (rebind {})\n", + index, self.context.id(), previous, pair, n); + } + } + Ok(index) + } + /// Records a publication through `pair`'s channel once it reached firmware. + pub(crate) fn channel_published(&self, pair: u8, qids: [u8; 2], ordinal: u64) { + self.context.work.lock().render.published(pair, qids, ordinal); } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -609,6 +743,24 @@ impl QosIds { pub(crate) fn used(&self) -> u32 { (self.state.lock().used & !3u128).count_ones() } + /// Clears, with `forget`, the firmware QoS rows of the IDs released and not taken again: + /// they still name the released context's scheduler page. The IDs `forget` returns, whose + /// rows still have work in flight (or all, if it fails), stay released for the next call. + pub(crate) fn forget_released(&self, forget: impl FnOnce(u128) -> Result) { + let ids = { + let mut state = self.state.lock(); + let ids = state.released & !state.used; + state.released = 0; + ids + }; + if ids == 0 { + return; + } + let left = forget(ids).unwrap_or(ids); + let mut state = self.state.lock(); + // An ID taken again meanwhile has a new owner, which writes its own row. + state.released |= left & !state.used; + } } /// Memory a context's work list may lead the firmware to stays allocated: a linked node (or @@ -646,6 +798,44 @@ mod capacity_tests { assert_eq!(ids.state.lock().used, 3); } } + + #[test] + fn a_released_qos_row_still_in_use_is_cleared_later() { + let ids = QosIds::new().unwrap(); + let (first, second) = (ids.acquire().unwrap(), ids.acquire().unwrap()); + let both = (1u128 << first.id) | (1u128 << second.id); + // Occupy fresh IDs so the callback must reuse one of the returned IDs. + let mut others = KVec::with_capacity(QOS_IDS - 4, GFP_KERNEL).unwrap(); + for _ in 0..QOS_IDS - 4 { + others.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + drop((first, second)); + // Work still runs on both rows: neither is cleared, and both stay released. + ids.forget_released(|released| { + assert_eq!(released, both); + Ok(released) + }); + // A failed clear keeps them too. + ids.forget_released(|released| { + assert_eq!(released, both); + Err(EIO) + }); + // One is taken again meanwhile: its new owner writes its own row. + let mut again = None; + ids.forget_released(|released| { + again = Some(ids.acquire().unwrap()); + Ok(released) + }); + let again = again.unwrap(); + assert_ne!(both & (1u128 << again.id), 0); + let rest = both & !(1u128 << again.id); + ids.forget_released(|released| { + assert_eq!(released, rest); + Ok(0) + }); + ids.forget_released(|_| panic!("nothing left to clear")); + drop(again); + } } impl Context { diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 7586f8474e602..0fe045d6f780c 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -77,12 +77,10 @@ impl WorkStates { } } -/// Ordinals past the turn remembered for ioctls that took a node or left early. +/// How far past the oldest waiting ioctl a new ioctl may be ordered: the ordinals past the +/// turn remembered for ioctls that left early. pub(super) const ORDER_WINDOW: u64 = 1024; -/// How far past the oldest waiting ioctl an ioctl may take its node. -pub(super) const RUN_AHEAD: u64 = 8; - /// The order in which a context's ioctls take work-state nodes. /// /// Each engine admits a context's ioctls in submission order, but the engines advance @@ -93,20 +91,21 @@ pub(super) const RUN_AHEAD: u64 = 8; /// engines, at most 36 between scheduling and completion, and the same load runs there without /// failure. /// -/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have -/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; -/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler -/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that -/// without letting one engine get far ahead of the other. +/// So an ioctl takes its node only at its turn, once every older ioctl of its context has taken +/// one or left without one. The firmware sorts a context's nodes by their words and flushes +/// from the head, and a node's word is drawn when the node is taken +/// ([`WorkStates::allocate`]): an ioctl taking its node ahead of an older one would hold a lower +/// word than that ioctl, whose completion stamp would then wait behind a node submitted after +/// it. Taking nodes strictly in turn keeps the words in submission order. Each engine admits +/// from its own scheduler, so the turn costs a scheduler round trip whenever the other engine's +/// ioctl is ready first. pub(super) struct Order { /// The next ordinal to hand out. next: u64, /// The oldest ordinal that has neither taken a node nor left. turn: u64, - /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + /// Ordinals past the turn that left, as bits at ordinal % ORDER_WINDOW. done: [u64; (ORDER_WINDOW / 64) as usize], - /// Set once an ioctl left too far past the turn to remember; admission is unordered after. - unordered: bool, /// An ioctl was refused since the turn last moved: its publisher waits for the next move. refused: bool, } @@ -116,24 +115,28 @@ impl Order { next: 0, turn: 0, done: [0; (ORDER_WINDOW / 64) as usize], - unordered: false, refused: false, } } - /// The ordinal of a new ioctl, in submission order. - pub(super) fn assign(&mut self) -> u64 { + /// The ordinal of a new ioctl, in submission order. None while [`ORDER_WINDOW`] ioctls of + /// the context are past the oldest one still waiting for its node. + pub(super) fn assign(&mut self) -> Option { + if self.next - self.turn >= ORDER_WINDOW { + return None; + } let ordinal = self.next; self.next += 1; - ordinal + Some(ordinal) } - /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the - /// turn next moves, so that move wakes the refused publisher. + /// Whether the ioctl `ordinal` may take its node now: only at its turn. A refusal is + /// remembered until the turn next moves, so that move wakes the refused publisher. pub(super) fn admits(&mut self, ordinal: u64) -> bool { - let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + let admits = ordinal == self.turn; self.refused |= !admits; admits } - /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + /// The ioctl `ordinal` took its node at its turn. Returns whether a refused ioctl may now be + /// admitted. pub(super) fn admit(&mut self, ordinal: u64) -> bool { self.finish(ordinal) } @@ -143,17 +146,14 @@ impl Order { self.finish(ordinal) } fn finish(&mut self, ordinal: u64) -> bool { - if self.unordered || ordinal < self.turn { + if ordinal < self.turn { return false; } if ordinal == self.turn { self.pass(); return core::mem::take(&mut self.refused); } - if ordinal - self.turn >= ORDER_WINDOW { - self.unordered = true; - return core::mem::take(&mut self.refused); - } + // assign() keeps every ordinal within the window past the turn. let bit = ordinal % ORDER_WINDOW; self.done[(bit / 64) as usize] |= 1 << (bit % 64); false @@ -172,19 +172,35 @@ impl Order { } } -/// Two stable logical ordinals, independent of physical pair numbering. +/// Two stable logical ordinals (render channels), independent of physical pair numbering. A +/// context's render work can move between physical pairs (sibling pairs, lending, relocation, +/// QID recycling), so a channel is rebound to a new pair once every publication this context +/// made through it has retired. A context's channel counts its passes up for the whole +/// process, so a rebound channel continues its pass sequence. pub(super) struct RenderSlots { pairs: [Option; 2], origins: [Option; 2], + /// Pass of the channel's last publication on its previous pair. + bases: [u8; 2], + /// QIDs and visibility target (ordinal + 1) of the last publication made through each + /// channel. + last: [Option<([u8; 2], u64)>; 2], + /// Recency of each channel's last publication. + used: [u64; 2], + epoch: u64, } impl RenderSlots { pub(super) const fn new() -> Self { Self { pairs: [None; 2], origins: [None; 2], + bases: [0; 2], + last: [None; 2], + used: [0; 2], + epoch: 0, } } - pub(super) fn get(&mut self, pair: u8) -> Option { + fn get(&mut self, pair: u8) -> Option { if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { return Some(index as u32); } @@ -192,14 +208,61 @@ impl RenderSlots { self.pairs[index] = Some(pair); Some(index as u32) } + /// The channel of `pair`, rebinding the least recently used channel whose publications + /// have all retired (`retired(qids, ordinal)`) when both are bound to other pairs. `None`: + /// no channel can move yet. + pub(super) fn bind( + &mut self, + pair: u8, + retired: impl Fn([u8; 2], u64) -> bool, + ) -> Option<(u32, Option)> { + if let Some(index) = self.get(pair) { + return Some((index, None)); + } + let index = (0..2) + .filter(|index| self.last[*index].is_none_or(|(qids, ordinal)| retired(qids, ordinal))) + .min_by_key(|index| self.used[*index])?; + let previous = self.pairs[index]; + // `last` holds the ordinal after the last publication (its visibility target), so that + // publication's pass is base + (next - origin). + if let (Some(origin), Some((_, next))) = (self.origins[index], self.last[index]) { + self.bases[index] = self.bases[index].wrapping_add(next.wrapping_sub(origin) as u8); + } + self.pairs[index] = Some(pair); + self.origins[index] = None; + self.last[index] = None; + Some((index as u32, previous)) + } pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { - let index = self.get(pair)? as usize; + let index = self.pairs.iter().position(|value| *value == Some(pair))?; let origin = *self.origins[index].get_or_insert(ordinal); - Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) + Some(self.bases[index].wrapping_add(ordinal.checked_sub(origin)?.wrapping_add(1) as u8)) + } + /// Records a publication that reached firmware through `pair`'s channel. + pub(super) fn published(&mut self, pair: u8, qids: [u8; 2], ordinal: u64) -> bool { + let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) else { + return false; + }; + self.epoch = self.epoch.wrapping_add(1); + self.used[index] = self.epoch; + self.last[index] = Some((qids, ordinal)); + true } } impl WorkStates { + /// The oldest node in use that the firmware lists ahead of `word`: a live word older than + /// it by the signed 32-bit difference the firmware sorts by. Its ioctl was submitted + /// earlier ([`Order`]), and until that node drains the firmware withholds every stamp of + /// `word`'s node behind it. + pub(super) fn older_live(&self, word: u32) -> Option { + self.live + .iter() + .map(|live| live.load(Ordering::Acquire)) + .filter(|&live| live != FREE && (live.wrapping_sub(word) as i32) < 0) + .min_by_key(|&live| live.wrapping_sub(word) as i32) + } + pub(super) fn used(&self) -> usize { self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() } @@ -250,65 +313,66 @@ mod tests { } #[test] - fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + fn ioctls_take_nodes_only_at_their_turn() { let mut order = Order::new(); - let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); - let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); - // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. - for &ordinal in &ordinals[..=RUN_AHEAD as usize] { - assert!(order.admits(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // Ioctls ahead of the turn taking their nodes do not move it. - for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { - assert!(!order.admit(ordinal)); + let ordinals: [u64; 4] = core::array::from_fn(|_| order.assign().unwrap()); + // Only the oldest waiting ioctl may take its node; the others are refused. + for &ordinal in &ordinals[1..] { + assert!(!order.admits(ordinal)); } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // The oldest one moves the turn past everything already admitted, and wakes the refused. - assert!(order.admit(oldest)); - assert!(order.admits(last)); - let next = order.assign(); - assert!(order.admits(next)); - assert!(!order.admits(next + RUN_AHEAD + 1)); + assert!(order.admits(ordinals[0])); + // Its admission moves the turn and wakes the refused. + assert!(order.admit(ordinals[0])); + assert!(order.admits(ordinals[1])); + // Nobody was refused since, so the next admission wakes nobody. + assert!(!order.admit(ordinals[1])); + assert!(!order.admits(ordinals[3])); + assert!(order.admits(ordinals[2])); + assert!(order.admit(ordinals[2])); + assert!(order.admits(ordinals[3])); } #[test] fn ioctls_leaving_without_a_node_pass_the_turn() { let mut order = Order::new(); - let (a, b, c) = (order.assign(), order.assign(), order.assign()); - let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); - let beyond = later[RUN_AHEAD as usize - 1]; - // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + let (a, b, c, d) = ( + order.assign().unwrap(), + order.assign().unwrap(), + order.assign().unwrap(), + order.assign().unwrap(), + ); + // b and c fail before their turn; while a waits, d waits too. assert!(!order.leave(b)); assert!(!order.leave(c)); - assert!(!order.admits(beyond)); + assert!(!order.admits(d)); // Once a takes its node, the turn passes b and c as well, and the refused one may retry. assert!(order.admit(a)); - assert!(order.admits(beyond)); - // With nobody refused since, moving the turn wakes nobody; an ordinal already passed - // leaves without effect. - assert!(!order.leave(later[0])); + assert!(order.admits(d)); + // An ordinal already passed leaves without effect. assert!(!order.leave(a)); } #[test] - fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + fn no_ioctl_is_ordered_past_the_window() { let mut order = Order::new(); - let first = order.assign(); + let first = order.assign().unwrap(); let mut last = first; - for _ in 0..ORDER_WINDOW { - last = order.assign(); + for _ in 1..ORDER_WINDOW { + last = order.assign().unwrap(); } - assert!(!order.admits(last)); - assert!(order.leave(last)); - assert!(order.admits(last - 1)); + assert!(order.assign().is_none()); + // An ioctl leaving far ahead is remembered within the window. + assert!(!order.leave(last)); + assert!(order.admits(first)); + assert!(!order.admit(first)); + assert!(order.assign().is_some()); } #[test] fn single_engine_admission_wakes_nobody() { let mut order = Order::new(); - for _ in 0..3 * RUN_AHEAD { - let ordinal = order.assign(); + for _ in 0..24 { + let ordinal = order.assign().unwrap(); assert!(order.admits(ordinal)); assert!(!order.admit(ordinal)); } diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index a175e58392379..50df8107bbac9 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -2,6 +2,9 @@ //! Event service ordering under the device mutex. A restart ends its role's batch; a worker //! joins preparation leases and services it on a separate turn before continuing the rings. +//! On M5 only the event rings are consumed here: the firmware's trace and statistics reports +//! are drained after the completions this service retires have been signalled +//! (`Firmware::drain_reports`), so no report queued ahead of a completion delays its fence. use super::{ fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, @@ -15,6 +18,8 @@ pub(super) trait Transport { fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); + /// The reports are drained after the pass's completions are signalled (M5). + fn reports_after_signal(&self) -> bool; fn drain( &mut self, role: Role, @@ -91,11 +96,13 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for &role in host.roles() { - host.trace(role); - } - for &role in host.roles() { - host.statistics(role); + if !host.reports_after_signal() { + for &role in host.roles() { + host.trace(role); + } + for &role in host.roles() { + host.statistics(role); + } } let mut masks = [0; 2]; if !host.recovery().pending() { diff --git a/drivers/gpu/drm/asahi/g17/fans.c b/drivers/gpu/drm/asahi/g17/fans.c new file mode 100644 index 0000000000000..6178a62068608 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fans.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The M5 thermal governor's sensor and fan calls (thermal.rs). The thermal + * core and the SMC hwmon cell have stubs where they are not built; Rust + * reaches them through these. + */ +#include +#include +#include +#include "fans.h" + +int asahi_zone_temp(const char *name, int *temp) +{ + struct thermal_zone_device *zone = thermal_zone_get_zone_by_name(name); + + if (IS_ERR_OR_NULL(zone)) + return zone ? PTR_ERR(zone) : -ENODEV; + return thermal_zone_get_temp(zone, temp); +} + +int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm) +{ + return macsmc_hwmon_fans_request(dev, permille, min_step_rpm); +} diff --git a/drivers/gpu/drm/asahi/g17/fans.h b/drivers/gpu/drm/asahi/g17/fans.h new file mode 100644 index 0000000000000..4bc18d80d3fe0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fans.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _ASAHI_G17_FANS_H +#define _ASAHI_G17_FANS_H +struct device; +int asahi_zone_temp(const char *name, int *temp); +int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm); +#endif diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index bd1228a82e26f..a96f67ae6b8f9 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -9,6 +9,9 @@ //! Policy creation and the initial snapshot take the policy rwsem before the //! requests mutex. Removal callbacks and the worker take only requests; //! notifier unregistration joins callbacks without holding that mutex. +//! M5 requests full CPU capacity only while refill pressure keeps the detector +//! active. Sustained demand can renew its deadline indefinitely; withdrawal +//! occurs on a later worker poll, not at an exact eight-millisecond boundary. use crate::hw::t8140::qos::CLOCK_HZ; use core::{ @@ -27,6 +30,7 @@ use core::ptr; use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; +const _: () = assert!(CLOCK_HZ == crate::hw::t8140::resources::COMMAND_TIMESTAMP_HZ); const STREAK_ON: u32 = 8; const STREAK_MAX: u32 = 64; const LINGER_NS: u64 = 8_000_000; @@ -76,7 +80,7 @@ impl Detector { } fn note(&self, start: u64, end: u64, now: u64) { - if end <= start { + if start == 0 || end <= start { return; } let previous = self.last_end.fetch_max(end, Ordering::AcqRel); @@ -141,6 +145,8 @@ struct Requests { policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], maxima: [u32; MAX_POLICIES], applied: [u32; MAX_POLICIES], + /// Full CPU capacity for M5 refill pressure; other profiles retain their floor. + floor_percent: u64, } // SAFETY: Each nonnull policy owns a kobject reference. The device requests @@ -150,12 +156,21 @@ unsafe impl Send for Requests {} #[cfg(CONFIG_CPU_FREQ)] impl Requests { - fn new() -> Self { + fn new(m5: bool) -> Self { Self { requests: Opaque::zeroed(), policies: [ptr::null_mut(); MAX_POLICIES], maxima: [0; MAX_POLICIES], applied: [0; MAX_POLICIES], + floor_percent: if m5 { 100 } else { FLOOR_PERCENT }, + } + } + + fn target_floor(maximum: u32, floor_percent: u64, enabled: bool) -> u32 { + if enabled { + (u64::from(maximum) * floor_percent / 100) as u32 + } else { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 } } @@ -206,11 +221,7 @@ impl Requests { if this.policies[index].is_null() { continue; } - let value = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE - }; + let value = Self::target_floor(this.maxima[index], *this.floor_percent, enabled); if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; // the policy reference keeps its constraints alive through REMOVE. @@ -296,12 +307,14 @@ pub(crate) struct Feed { } impl Feed { - pub(crate) fn new() -> Result> { + pub(crate) fn new(m5: bool) -> Result> { + #[cfg(not(CONFIG_CPU_FREQ))] + let _ = m5; let feed = Arc::pin_init( pin_init!(Self { detector: Detector::new(), #[cfg(CONFIG_CPU_FREQ)] - requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + requests <- new_mutex!(Requests::new(m5), "G17 CPU feed"), #[cfg(CONFIG_CPU_FREQ)] notifier <- pin_init!(PolicyNotifier { block: Opaque::new(bindings::notifier_block { @@ -422,6 +435,8 @@ impl Feed { Monotonic::ktime_get() as u64 } + /// Firmware execution intervals in the device's 24 MHz timestamp domain. + /// Missing, unexecuted and wrapped intervals must not affect the detector. pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 0b8819bdbd396..38955eb33271d 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,7 +7,7 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, @@ -63,9 +63,44 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { pub(crate) struct ExportFence { source: Option>, scoped: bool, + /// A waiter enabled signalling on this ioctl output fence before it signalled ([`waiters`]). + waited: AtomicBool, } static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); +/// Output fences a waiter is waiting on: signalling enabled and not yet signalled. +static WAITERS: AtomicU32 = AtomicU32::new(0); +/// Waits begun on output fences, ever. +static WAITS: AtomicU64 = AtomicU64::new(0); + +/// The output fences a waiter is blocked on now, and the waits begun so far: the work a host +/// thread waits for, which the M5 performance effort follows ([`super::power::Effort`]). +pub(crate) fn waiters() -> (u32, u64) { + (WAITERS.load(Ordering::Acquire), WAITS.load(Ordering::Acquire)) +} + +/// The first waiter on an unsignalled output fence (the fence core calls this once, under the +/// fence's lock). +fn waited(flag: &AtomicBool) { + if !flag.swap(true, Ordering::AcqRel) { + WAITERS.fetch_add(1, Ordering::AcqRel); + WAITS.fetch_add(1, Ordering::AcqRel); + } +} + +/// The fence signalled or went away: a waiter on it no longer waits. +fn unwaited(flag: &AtomicBool) { + if flag.swap(false, Ordering::AcqRel) { + WAITERS.fetch_sub(1, Ordering::AcqRel); + } +} + +impl Drop for ExportFence { + fn drop(&mut self) { + unwaited(&self.waited); + } +} + #[vtable] impl FenceOps for ExportFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -74,6 +109,10 @@ impl FenceOps for ExportFence { fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") } + fn enable_signaling(self: &FenceObject) -> bool { + waited(&self.waited); + true + } } #[inline(never)] @@ -83,6 +122,7 @@ fn export_fallback(contexts: &FenceContexts) -> Result> ExportFence { source: None, scoped: false, + waited: AtomicBool::new(false), }, )?; // SAFETY: Unique, initialized and still private. Only this constructor @@ -159,7 +199,7 @@ pub(crate) struct Submission { impl Submission { fn new(context: Arc, firmware_source: bool) -> Result> { - let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + let work = Arc::new(WorkStateLease::deferred(context)?, GFP_KERNEL)?; Self::with_work(Some(work), firmware_source) } fn with_work(work: Option>, firmware_source: bool) -> Result> { @@ -238,6 +278,7 @@ impl Submission { } else { fence.signal(); } + unwaited(&fence.waited); } } } @@ -510,6 +551,7 @@ impl Drop for Outputs { ); } fallback.signal(); + unwaited(&fallback.waited); } } } diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 9015dc82e4076..3d1449bee2147 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -11,31 +11,206 @@ use super::fw::freelist::*; use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; -use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; +use core::sync::atomic::{fence, AtomicBool, AtomicU16, AtomicU64, Ordering}; use kernel::{ new_mutex, prelude::*, sync::{Arc, Mutex}, }; +mod render_ids; + +const NO_RENDER_ID: u16 = u16::MAX; +static NEXT_RENDER_OWNER: AtomicU64 = AtomicU64::new(1); + +struct RenderIdCell { + owner: u64, + slot: AtomicU16, + generation: AtomicU64, + closed: AtomicBool, + storage: Arc, +} +impl RenderIdCell { + fn has_storage(&self) -> bool { + self.storage.backing.lock().is_some() || self.storage.spare.lock().is_some() + } +} + +/// Detached after the device's final free-list release. Drop outside the +/// device and identity locks because mappings may take memory-management locks. +#[must_use] +pub(crate) struct ReclaimedRenderId { + _backing: Option, + spare: Option, +} +impl ReclaimedRenderId { + /// Caller is outside the device lock. Remove the previous client's alias + /// before handing physical storage to another logical pool. + fn into_pages(self) -> Option { + self._backing.map(RenderBacking::into_pages).or(self.spare) + } +} + +struct RenderIdState { + leases: render_ids::State, + cells: [Option>; render_ids::COUNT], +} + /// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. /// First-free allocation keeps the shared firmware namespace deterministic. +#[pin_data] pub(crate) struct RenderIds { used: [AtomicU64; 2], + available: Arc, + #[pin] + state: Mutex, } impl RenderIds { - pub(crate) const fn new() -> Self { - Self { + #[inline(never)] + pub(crate) fn new() -> Result> { + Arc::pin_init(try_pin_init!(Self { used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], + available: super::availability::Availability::new()?, + state <- new_mutex!(RenderIdState { + leases: render_ids::State::new(), + cells: [const { None }; render_ids::COUNT], + }, "G17 render identities"), + }), GFP_KERNEL) + } + + pub(crate) fn availability(&self) -> &Arc { + &self.available + } + + fn pin(self: &Arc, cell: &Arc, assigned: impl FnOnce()) -> Result { + let mut state = self.state.lock(); + let slot = cell.slot.load(Ordering::Acquire); + if slot != NO_RENDER_ID { + let lease = render_ids::Lease { + slot, owner: cell.owner, generation: cell.generation.load(Ordering::Acquire), + }; + match state.leases.pin(lease) { + Ok(_) => return Ok(RenderIdPin { allocator: self.clone(), cell: cell.clone(), lease }), + Err(render_ids::Error::Overflow) => return Err(EOVERFLOW), + Err(_) => {} + } + } + let claim = state.leases.claim(cell.owner, |_| false).map_err(|error| match error { + render_ids::Error::Busy | render_ids::Error::PendingRelease => ENOSPC, + render_ids::Error::Overflow => EOVERFLOW, + _ => EIO, + })?; + let index = usize::from(claim.lease.slot - render_ids::FIRST); + state.cells[index] = Some(cell.clone()); + assigned(); + cell.generation.store(claim.lease.generation, Ordering::Relaxed); + cell.slot.store(claim.lease.slot, Ordering::Release); + Ok(RenderIdPin { allocator: self.clone(), cell: cell.clone(), lease: claim.lease }) + } + + fn close(&self, cell: &Arc) { + cell.closed.store(true, Ordering::Release); + let mut state = self.state.lock(); + let slot = cell.slot.load(Ordering::Acquire); + if slot == NO_RENDER_ID { return; } + let index = usize::from(slot - render_ids::FIRST); + if state.cells[index].as_ref().is_some_and(|owner| Arc::ptr_eq(owner, cell)) + && state.leases.remove(cell.owner, |_| true).is_ok() { + let old = state.cells[index].take(); + cell.slot.store(NO_RENDER_ID, Ordering::Release); + drop(state); + drop(old); + self.available.notify(); } } + + /// Caller holds the device mutex and supplies the exact final release proof. + pub(crate) fn reclaim_one(&self, released: impl FnMut(u16) -> bool) -> Option { + self.reclaim(false, released) + } + /// Reuse backing for a pool that already has an ID, without evicting an + /// empty metadata-only owner. Caller supplies the same release proof. + pub(crate) fn reclaim_backing(&self, released: impl FnMut(u16) -> bool) -> Option { + self.reclaim(true, released) + } + fn reclaim(&self, only_storage: bool, mut released: impl FnMut(u16) -> bool) -> Option { + let mut state = self.state.lock(); + let RenderIdState { leases, cells } = &mut *state; + let mut select = |require_storage: bool| leases.reclaim_one(|lease| { + cells[usize::from(lease.slot - render_ids::FIRST)].as_ref().is_some_and(|cell| + cell.owner == lease.owner + && cell.slot.load(Ordering::Acquire) == lease.slot + && cell.generation.load(Ordering::Acquire) == lease.generation + && (!require_storage || cell.has_storage())) + && released(lease.slot) + }); + let lease = select(true).or_else(|| if only_storage { None } else { select(false) })?; + let old = state.cells[usize::from(lease.slot - render_ids::FIRST)].take(); + let (backing, spare) = if let Some(cell) = old.as_ref() { + let backing = cell.storage.backing.lock().take(); + let spare = cell.storage.spare.lock().take(); + cell.storage.page_list.store(0, Ordering::Release); + cell.slot.store(NO_RENDER_ID, Ordering::Release); + (backing, spare) + } else { (None, None) }; + drop(state); + drop(old); + self.available.notify(); + Some(ReclaimedRenderId { _backing: backing, spare }) + } +} + +pub(super) struct RenderIdPin { + allocator: Arc, + cell: Arc, + lease: render_ids::Lease, +} +impl Drop for RenderIdPin { + fn drop(&mut self) { + let available = self.allocator.state.lock().leases.unpin(self.lease).unwrap_or(false); + if self.cell.closed.load(Ordering::Acquire) { self.allocator.close(&self.cell); } + if available { self.allocator.available.notify(); } + } +} + +enum RenderIdKind { + Fixed(u16), + Rebindable(Arc), } -struct RenderId { - slot: u16, +pub(super) struct RenderId { + kind: RenderIdKind, allocator: Arc, } impl RenderId { - fn new(allocator: &Arc) -> Result { + pub(super) fn slot(&self) -> u16 { + match &self.kind { + RenderIdKind::Fixed(slot) => *slot, + RenderIdKind::Rebindable(cell) => cell.slot.load(Ordering::Acquire), + } + } + pub(super) fn logical(allocator: &Arc) -> Result { + let owner = NEXT_RENDER_OWNER.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |next| next.checked_add(1)).map_err(|_| EOVERFLOW)?; + let storage = RenderStorage::new(None)?; + Ok(Self { kind: RenderIdKind::Rebindable(Arc::new(RenderIdCell { + owner, slot: AtomicU16::new(NO_RENDER_ID), generation: AtomicU64::new(0), + closed: AtomicBool::new(false), + storage, + }, GFP_KERNEL)?), allocator: allocator.clone() }) + } + fn storage(&self) -> Option> { + match &self.kind { + RenderIdKind::Rebindable(cell) => Some(cell.storage.clone()), + RenderIdKind::Fixed(_) => None, + } + } + fn pin(&self, assigned: impl FnOnce()) -> Result> { + match &self.kind { + RenderIdKind::Fixed(_) => Ok(None), + RenderIdKind::Rebindable(cell) => self.allocator.pin(cell, assigned).map(Some), + } + } + pub(super) fn new(allocator: &Arc) -> Result { for (bank, used) in allocator.used.iter().enumerate() { let mut value = used.load(Ordering::Acquire); loop { @@ -51,7 +226,7 @@ impl RenderId { ) { Ok(_) => { return Ok(Self { - slot: 129 + bank as u16 * 64 + bit as u16, + kind: RenderIdKind::Fixed(129 + bank as u16 * 64 + bit as u16), allocator: allocator.clone(), }) } @@ -64,11 +239,39 @@ impl RenderId { } impl Drop for RenderId { fn drop(&mut self) { - let bit = self.slot as usize - 129; - self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + match &self.kind { + RenderIdKind::Fixed(slot) => { + let bit = *slot as usize - 129; + self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + } + RenderIdKind::Rebindable(cell) => self.allocator.close(cell), + } } } +/// What one command needs of its compute queue's USC pool, in the family's +/// terms: G17P pools count blocks, M5 pools have the command plan's geometry +/// (`None`: the default M5 pool). Derived once per command +/// (`Backend::compute_capacity`); every admission check and allocation takes it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ComputeCapacity { + Blocks(usize), + M5(Option), +} + +/// Profile-specific descriptor data for a fresh physical pool. The runtime +/// installs this only before publication, or after an acknowledged release. +pub(crate) enum PoolDescriptor { + Neo { slot: u16, page_list: u64, blocks: usize }, + M5 { slot: u16, words: [u64; 8] }, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PageLayout { + Neo { blocks: usize, runs: RunLayout }, + M5(super::fw::m5::pool::PoolGeometry), +} + /// Free-list backing is by far the largest user of the shared lower GPU VA /// window, and placing it populates every page before the window is searched. /// After a refused placement, builders waiting for capacity would allocate and @@ -125,8 +328,10 @@ fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { pub(crate) struct Pages { backing: KernelObject, - blocks: usize, - layout: RunLayout, + layout: PageLayout, + /// The charge of an M5 backing sized by a client's request, released with it. It is the + /// backing's only count: the backing itself is not counted as kernel memory. + charge: Option, } impl Drop for Pages { fn drop(&mut self) { @@ -134,7 +339,7 @@ impl Drop for Pages { } } impl Pages { - fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { + fn new(alloc: &Allocator<'_>, blocks: usize, runs: RunLayout) -> Result { let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; let backing = place_lower(size, || { alloc.lower( @@ -144,39 +349,148 @@ impl Pages { CpuMap::WriteCombined, ) })?; - Self::from_backing(backing, blocks, layout) + Self::from_backing(backing, blocks, runs) } - fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { + fn from_backing(mut backing: KernelObject, blocks: usize, runs: RunLayout) -> Result { let base = backing.gpu_va(); - backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; + backing.initialize::(0, |tables| tables.init(base, blocks, runs))?; Ok(Self { backing, - blocks, - layout, + layout: PageLayout::Neo { blocks, runs }, + charge: None, }) } - /// Fresh compute backing at the admitted capacity, built off-lock. - pub(crate) fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { + /// Fresh compute backing at the admitted capacity, built off-lock, in the + /// capacity's family format. + /// An M5 backing is charged to `client` when one is given (see [`Self::m5`]). + pub(crate) fn for_compute(alloc: &Allocator<'_>, capacity: ComputeCapacity, + client: Option<&Arc>) -> Result { + match capacity { + ComputeCapacity::Blocks(blocks) => Self::compute_sized(alloc, blocks), + ComputeCapacity::M5(geometry) => Self::compute_m5(alloc, geometry, client), + } + } + fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { if blocks < COMPUTE_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } Self::new(alloc, blocks, RunLayout::Compute) } - pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { + /// Fresh zeroed M5 compute backing, built without the device mutex. M5 keeps + /// its directory, block list and initial eight render blocks in one retained + /// allocation. + fn compute_m5(alloc: &Allocator<'_>, required: Option, + client: Option<&Arc>) -> Result { + use super::fw::m5::pool; + let geometry = match required { Some(geometry) => geometry, + None => pool::PoolGeometry::g17s(8).map_err(|_| EINVAL)?, }; + Self::m5(alloc, geometry, client) + } + /// M5 pool sizes follow the client's program records, up to the board's whole inventory, + /// so the backing is admitted against `client`'s and the device's limits before it is + /// allocated, as a user object of that size would be, and counted as that client's held + /// memory instead of kernel memory. A refusal fails with ENOMEM. + fn m5(alloc: &Allocator<'_>, geometry: super::fw::m5::pool::PoolGeometry, + client: Option<&Arc>) -> Result { + let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; + let size = geometry.blocks.checked_mul(geometry.block_bytes()) + .and_then(|n| n.checked_add(tables)).ok_or(EOVERFLOW)?; + let (backing, charge) = match client { + Some(client) => { + let commit = crate::memacct::LEDGER.admit(client, size as u64)?; + let backing = alloc.lower_user_charged(size, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW)?; + let charge = crate::memacct::Charge::user(commit, None); + charge.set_held(true); + (backing, Some(charge)) + } + None => (alloc.lower(size, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined)?, None), + }; + let mut pages = Self { backing, layout: PageLayout::M5(geometry), charge }; + pages.populate_m5()?; + Ok(pages) + } + fn populate_m5(&mut self) -> Result { + use super::fw::m5::pool; + let PageLayout::M5(geometry) = self.layout else { return Err(EINVAL); }; + let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; + let base = self.backing.gpu_va(); + let mut blocks = KVec::with_capacity(geometry.blocks, GFP_KERNEL)?; + for index in 0..geometry.blocks { + blocks.push(base + (tables + index * geometry.block_bytes()) as u64, GFP_KERNEL)?; + } + pool::encode_directory(geometry, &blocks, + &mut self.backing.bytes_mut()[..geometry.directory_bytes]).map_err(|_| EINVAL)?; + pool::encode_freelist(geometry, &blocks, + &mut self.backing.bytes_mut()[geometry.directory_bytes..tables]).map_err(|_| EINVAL)?; + Ok(()) + } + fn reset_for_client(&mut self) -> Result { + // Released storage may contain the previous client's private data. + // Clear it before exposing an alias in a different client VM. + self.backing.bytes_mut().fill(0); + self.populate_m5()?; + fence(Ordering::SeqCst); + Ok(()) + } + /// Aliases the backing into `vm`, whose client becomes its user: a charged backing's + /// charge moves to that client (ENOMEM over its limit), so storage that another client + /// released and this one reuses is no longer counted against the first. + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + if let (Some(charge), Some(client)) = (self.charge.as_mut(), vm.memory_client()) { + charge.transfer(client)?; + } self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - pub(crate) fn page_list_va(&self) -> u64 { - self.backing.gpu_va() + fn fits(&self, required: Option) -> bool { + match (self.layout, required) { + (PageLayout::M5(current), Some(required)) => + current.block_bytes() == required.block_bytes() && current.blocks >= required.blocks, + (_, None) => true, + _ => false, + } + } + /// Whether this backing holds `required`; backing of the other family never does. + pub(crate) fn fits_capacity(&self, required: ComputeCapacity) -> bool { + match (self.layout, required) { + (PageLayout::Neo { blocks, .. }, ComputeCapacity::Blocks(required)) => blocks >= required, + (PageLayout::M5(_), ComputeCapacity::M5(geometry)) => self.fits(geometry), + _ => false, + } } + fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 + self.page_list_va() + match self.layout { + PageLayout::Neo { .. } => PAGE_LIST_SIZE, + PageLayout::M5(g) => g.directory_bytes, + } as u64 + } + fn m5_layout(&self, slot: u16) -> Result { + let PageLayout::M5(geometry) = self.layout else { return Err(EINVAL); }; + Ok(super::fw::m5::pool::PoolLayout { geometry, directory: self.page_list_va(), + freelist: self.run_list_va(), hardware_id: u32::from(slot) }) + } + /// The descriptor row of this backing installed at `slot`. + pub(crate) fn descriptor(&self, slot: u16) -> Result { + match self.layout { + PageLayout::M5(_) => { + let words = super::fw::m5::pool::initial_uma_descriptor(self.m5_layout(slot)?) + .map_err(|_| EINVAL)?; + Ok(PoolDescriptor::M5 { slot, words }) + } + PageLayout::Neo { blocks, .. } => + Ok(PoolDescriptor::Neo { slot, page_list: self.page_list_va(), blocks }), + } } fn repopulate(&self) -> Result { + let PageLayout::Neo { blocks, runs } = self.layout else { return Err(EINVAL); }; let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); PoolTables::for_each_word( self.page_list_va(), - self.blocks, - self.layout, + blocks, + runs, |index, value| { // SAFETY: The page-aligned backing was validated for the complete // tables. The enumerator bounds every index inside those tables; @@ -194,11 +508,17 @@ static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); /// both aliases before the backing. pub(crate) struct RenderBacking { _client: mmu::KernelMapping, - _global: mmu::KernelMapping, + _global: Option, pages: Pages, } impl RenderBacking { + fn into_pages(self) -> Pages { + let Self { _client, _global, pages } = self; + drop(_client); + drop(_global); + pages + } /// Fresh populated backing, built without the device mutex. pub(crate) fn new( alloc: &Allocator<'_>, @@ -222,10 +542,37 @@ impl RenderBacking { let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, - _global: global, + _global: Some(global), pages, }) } + /// Fresh M5 backing, aliased into the client VM only: M5 firmware uses + /// the canonical and client mappings of these pages. + fn m5(alloc: &Allocator<'_>, client: &mmu::Vm, + geometry: super::fw::m5::pool::PoolGeometry) -> Result { + let mut pages = Pages::m5(alloc, geometry, client.memory_client())?; + let client = pages.map_client(client)?; + Ok(Self { _client: client, _global: None, pages }) + } +} + +#[pin_data] +struct RenderStorage { + #[pin] + backing: Mutex>, + #[pin] + spare: Mutex>, + page_list: AtomicU64, +} +impl RenderStorage { + fn new(backing: Option) -> Result> { + let address = backing.as_ref().map_or(0, |backing| backing.pages.page_list_va()); + Arc::pin_init(try_pin_init!(Self { + backing <- new_mutex!(backing, "G17 render pool backing"), + spare <- new_mutex!(None, "G17 render pool spare"), + page_list: AtomicU64::new(address), + }), GFP_KERNEL) + } } /// A render pool retained by a logical context and every graph referencing it. @@ -233,23 +580,39 @@ impl RenderBacking { /// consumed and the client VM is evicted; a fresh backing is installed before /// the pool is staged again. pub(crate) struct RenderPool { - blocks: usize, - backing: Arc>>, - /// Page-list VA of the installed backing; zero while vacant. - page_list: AtomicU64, + storage: Arc, /// Counts installed backings, so a re-armed backing is a new free-list /// owner even if its addresses repeat. epoch: AtomicU64, /// Residency of the client VM the backing is aliased into. client_residency: Option>, + /// The one VM (`mmu::Vm::id`) this pool's backing is aliased into. + client_vm: u64, control: KernelObject, state: KernelObject, served: AtomicBool, serial: u64, id: RenderId, + layout: PageLayout, +} + +/// Keeps both a physical identity and its logical pool alive during use. +/// Drop the identity pin before releasing the retained pool reference. +pub(crate) struct PoolPin { + _identity: RenderIdPin, + _pool: Arc, } impl RenderPool { + /// The caller retains this guard across construction and all firmware use. + pub(crate) fn pin_id(self: &Arc) -> Result> { + let pin = self.id.pin(|| { + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + })?; + Ok(pin.map(|pin| PoolPin { _identity: pin, _pool: self.clone() })) + } + pub(crate) fn new( alloc: &Allocator<'_>, ids: &Arc, @@ -259,6 +622,32 @@ impl RenderPool { ) -> Result { let id = RenderId::new(ids)?; let backing = RenderBacking::new(alloc, render_global, client, blocks)?; + Self::with_backing(alloc, id, client, backing) + } + + /// Explicit initial geometry; allocation policy remains with the caller. + /// M5 firmware uses the canonical and client mappings of these pages. + pub(super) fn m5(alloc: &Allocator<'_>, id: RenderId, client: &mmu::Vm, + geometry: super::fw::m5::pool::PoolGeometry) -> Result { + if !alloc.uat.is_m5() { return Err(EINVAL); } + // Logical metadata is cheap; physical backing is installed only once + // preparation owns both an identity pin and admission capacity. + let storage = id.storage().ok_or(EINVAL)?; + let state = alloc.kernel(CONTROL_SIZE, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let control = alloc.kernel(CONTROL_SIZE, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let serial = RENDER_POOL_SERIAL.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |value| value.checked_add(1)).map_err(|_| EOVERFLOW)? + 1; + Ok(Self { storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + client_vm: client.id(), + control, state, served: AtomicBool::new(true), serial, id, + layout: PageLayout::M5(geometry) }) + } + + fn with_backing(alloc: &Allocator<'_>, id: RenderId, client: &mmu::Vm, + backing: RenderBacking) -> Result { + let layout = backing.pages.layout; let address = backing.pages.page_list_va(); let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( @@ -273,45 +662,51 @@ impl RenderPool { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - control.write( - 0, - FreeListControl::render( - &FreeListArgs { - buffer_slot: u32::from(id.slot), - class: FreeListClass::Two, - page_list_va: address, - run_list_va, - blocks: blocks as u32, - state_va: state.gpu_va(), - }, + match layout { + PageLayout::M5(_) => { + super::fw::m5::pool::encode_g17s_render_state(backing.pages.m5_layout(id.slot())?, + state.gpu_va(), control.bytes_mut()).map_err(|_| EINVAL)?; + } + PageLayout::Neo { blocks, .. } => control.write( 0, - ), - )?; + FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(id.slot()), + class: FreeListClass::Two, + page_list_va: address, + run_list_va, + blocks: blocks as u32, + state_va: state.gpu_va(), + }, + 0, + ), + )?, + } fence(Ordering::SeqCst); let serial = RENDER_POOL_SERIAL .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) .map_err(|_| EOVERFLOW)? + 1; - let backing = Arc::pin_init( - new_mutex!(Some(backing), "G17 render pool backing"), - GFP_KERNEL, - )?; + let storage = RenderStorage::new(Some(backing))?; Ok(Self { - blocks, - backing, - page_list: AtomicU64::new(address), + storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + client_vm: client.id(), control, state, served: AtomicBool::new(false), serial, id, + layout, }) } /// No backing is installed: the pool must be re-armed before staging. pub(crate) fn vacant(&self) -> bool { - self.page_list.load(Ordering::Acquire) == 0 + self.storage.page_list.load(Ordering::Acquire) == 0 + } + pub(crate) fn needs_storage(&self) -> bool { + self.vacant() && self.storage.spare.lock().is_none() } /// Pressure reclaim evicted the client VM, which is idle. pub(crate) fn client_evicted(&self) -> bool { @@ -322,9 +717,9 @@ impl RenderPool { /// next staging requires a re-armed backing. The caller drops the result /// after releasing the device mutex. pub(crate) fn vacate(&self) -> Option { - let backing = self.backing.lock().take(); + let backing = self.storage.backing.lock().take(); if backing.is_some() { - self.page_list.store(0, Ordering::Release); + self.storage.page_list.store(0, Ordering::Release); } backing } @@ -332,7 +727,7 @@ impl RenderPool { /// The next staging rewrites the control header and descriptor row for it /// (its run-list address changes the free-list owner identity). pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { - let mut backing = self.backing.lock(); + let mut backing = self.storage.backing.lock(); if backing.is_some() { return Ok(()); } @@ -340,21 +735,63 @@ impl RenderPool { *backing = Some(fresh); self.epoch.fetch_add(1, Ordering::AcqRel); self.served.store(true, Ordering::Release); - self.page_list.store(address, Ordering::Release); + self.storage.page_list.store(address, Ordering::Release); Ok(()) } + /// Keep released physical storage for the incoming owner, without keeping + /// the old VM alias or assigning backing to every logical context. Its + /// charge moves to the owner's client when the owner maps it + /// ([`Pages::map_client`]). + pub(crate) fn cache_reclaimed(&self, reclaimed: ReclaimedRenderId) { + let pages = reclaimed.into_pages(); + let backing = self.storage.backing.lock(); + let mut spare = self.storage.spare.lock(); + let mut pages = pages; + if backing.is_none() && spare.is_none() + && pages.as_ref().is_some_and(|pages| pages.layout == self.layout) { + *spare = pages.take(); + } + drop(spare); + drop(backing); + drop(pages); + } /// Re-arms a vacant pool with fresh backing built here, without the device - /// mutex. `client` is the VM of the context that owns this pool. + /// mutex. `client` is the VM of the context that owns this pool; EPERM + /// for any other VM. pub(crate) fn ensure_backing( &self, alloc: &Allocator<'_>, render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { - if !self.vacant() { - return Ok(()); + if client.id() != self.client_vm { + return Err(EPERM); } - self.rearm(RenderBacking::new(alloc, render_global, client, self.blocks)?) + let geometry = match self.layout { + PageLayout::M5(geometry) => geometry, + PageLayout::Neo { blocks, .. } => { + if !self.vacant() { + return Ok(()); + } + return self.rearm(RenderBacking::new(alloc, render_global, client, blocks)?); + } + }; + let mut backing = self.storage.backing.lock(); + if backing.is_some() { return Ok(()); } + let spare = self.storage.spare.lock().take(); + let fresh = if let Some(mut pages) = spare { + pages.reset_for_client()?; + let alias = pages.map_client(client)?; + RenderBacking { _client: alias, _global: None, pages } + } else { + RenderBacking::m5(alloc, client, geometry)? + }; + let address = fresh.pages.page_list_va(); + *backing = Some(fresh); + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + self.storage.page_list.store(address, Ordering::Release); + Ok(()) } pub(crate) fn serial(&self) -> u64 { @@ -364,21 +801,26 @@ impl RenderPool { self.epoch.load(Ordering::Acquire) } pub(crate) fn id(&self) -> u16 { - self.id.slot + self.id.slot() } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } /// Zero while vacant. pub(crate) fn page_list_va(&self) -> u64 { - self.page_list.load(Ordering::Acquire) + self.storage.page_list.load(Ordering::Acquire) } pub(crate) fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 + self.page_list_va() + match self.layout { + PageLayout::Neo { .. } => PAGE_LIST_SIZE, + PageLayout::M5(g) => g.directory_bytes, + } as u64 } pub(crate) fn end_va(&self) -> u64 { - self.run_list_va() - + (self.blocks * size_of::()) as u64 + self.run_list_va() + match self.layout { + PageLayout::Neo { blocks, .. } => blocks * size_of::(), + PageLayout::M5(g) => g.blocks * 8, + } as u64 } /// Fresh pools are already populated. Reuse requires the previous release @@ -388,13 +830,34 @@ impl RenderPool { return Ok(()); } let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let blocks = match self.layout { + PageLayout::Neo { blocks, .. } => blocks, + PageLayout::M5(_) => { + let backing = self.storage.backing.lock(); + let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; + if pages.page_list_va() != self.page_list_va() { return Err(EIO); } + let mut bytes = [0; super::fw::m5::pool::STATE_BYTES]; + super::fw::m5::pool::encode_g17s_render_state(pages.m5_layout(self.id())?, + self.state.gpu_va(), &mut bytes).map_err(|_| EINVAL)?; + bytes[super::fw::m5::pool::COMPLETED..super::fw::m5::pool::COMPLETED + 8] + .copy_from_slice(&submitted.to_le_bytes()); + let raw = self.control.pointer(0, bytes.len())?; + for (index, byte) in bytes.into_iter().enumerate() { + // SAFETY: The whole retained range was checked and the caller + // observed final release before installing fresh pool backing. + unsafe { raw.add(index).write_volatile(byte) }; + } + fence(Ordering::SeqCst); + return Ok(()); + } + }; let header = FreeListControl::render( &FreeListArgs { buffer_slot: u32::from(self.id()), class: FreeListClass::Two, page_list_va: self.page_list_va(), run_list_va: self.run_list_va(), - blocks: self.blocks as u32, + blocks: blocks as u32, state_va: self.state.gpu_va(), }, submitted, @@ -409,7 +872,7 @@ impl RenderPool { .write_unaligned(header) }; let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - let backing = self.backing.lock(); + let backing = self.storage.backing.lock(); let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; if pages.page_list_va() != self.page_list_va() { return Err(EIO); @@ -428,7 +891,9 @@ impl RenderPool { /// counter, so retain the byte accesses required by that interface. pub(crate) fn counters(&self) -> Result<[u64; 2]> { let submitted = self.state.dword(0)?.load(Ordering::Relaxed); - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let offset = if self.is_m5() { super::fw::m5::pool::COMPLETED } + else { FreeListControl::COMPLETED }; + let completed = self.control.pointer(offset, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { // SAFETY: The checked eight-byte range stays mapped; byte accesses @@ -438,6 +903,47 @@ impl RenderPool { Ok([submitted, u64::from_le_bytes(bytes)]) } + pub(crate) fn fits(&self, geometry: Option) -> bool { + match (self.layout, geometry) { + (PageLayout::M5(current), Some(required)) => + current.block_bytes() == required.block_bytes() && current.blocks >= required.blocks, + (PageLayout::Neo { .. }, None) => true, + _ => false, + } + } + + pub(crate) fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } + + fn m5_state(&self) -> Result { + if !self.is_m5() { return Err(EINVAL); } + let mut state = super::fw::m5::pool::PoolState([0; super::fw::m5::pool::STATE_BYTES]); + let raw = self.control.pointer(0, state.0.len())?; + fence(Ordering::Acquire); + for (index, byte) in state.0.iter_mut().enumerate() { + // SAFETY: The entire retained header range was checked above. + *byte = unsafe { raw.add(index).read_volatile() }; + } + Ok(state) + } + + pub(crate) fn m5_descriptor(&self) -> Result<[u64; 8]> { Ok(self.m5_state()?.descriptor()) } + + pub(crate) fn store_m5_descriptor(&self, words: [u64; 8]) -> Result { + let mut state = self.m5_state()?; + if !state.matches(&words) { return Err(EIO); } + state.restore(words); + for offset in super::fw::m5::pool::SAVED_FIELDS { + let raw = self.control.pointer(offset, 4)?; + for index in 0..4 { + // SAFETY: Each mutable field was checked; the caller owns the + // retired descriptor, and immutable header bytes stay intact. + unsafe { raw.add(index).write_volatile(state.0[offset + index]) }; + } + } + fence(Ordering::SeqCst); + Ok(()) + } + /// Snapshot the unaligned shared fields before loading their descriptor. pub(crate) fn page_pool_state(&self) -> Result { let raw = self.control.pointer(0x14, 0x1c)?; @@ -511,23 +1017,39 @@ impl Submitted<'_> { /// Idle retained queues may give their USC backing back (`pages` is `None`) /// once the firmware consumed the pool's release; the control and state /// objects the queue record names stay until processor stop. +/// +/// A backing serves one VM: its blocks hold the stack and scratch memory of +/// that VM's work, so a view of it in another VM is refused +/// ([`Self::map_client`]). Another VM gets the queue only after the backing +/// was given back and fresh, zeroed backing installed ([`Self::rearm`]). pub(crate) struct ComputePool { pages: Option, + /// The VM (`mmu::Vm::id`) the backing serves, from its first client view + /// until it is given back; zero while no client has had a view of it. + client_vm: u64, control: KernelObject, state: KernelObject, id: u16, + m5: bool, } impl ComputePool { - pub(crate) fn fits(&self, blocks: usize) -> bool { - self.pages.as_ref().is_some_and(|pages| pages.blocks >= blocks) + /// Capacity check for admission and growth. + pub(crate) fn fits_capacity(&self, required: ComputeCapacity) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits_capacity(required)) } - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { + /// A pool at the admitted capacity, in the capacity's family format. + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, required: ComputeCapacity, + client: Option<&Arc>) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } + let pages = Pages::for_compute(alloc, required, client)?; + let m5 = pages.is_m5(); + Self::with_pages(alloc, qid, m5, pages) + } + fn with_pages(alloc: &Allocator<'_>, qid: u8, m5: bool, pages: Pages) -> Result { let id = u16::from(qid) + 1; - let pages = Pages::compute_sized(alloc, blocks)?; let mut control = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -540,38 +1062,64 @@ impl ComputePool { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - control.write( - 0, - FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: blocks as u32, - state_va: state.gpu_va(), - }), - )?; + Self::initialize_control(&mut control, &pages, id, state.gpu_va(), [0; 8])?; fence(Ordering::SeqCst); - Ok(Self { - pages: Some(pages), - control, - state, - id, - }) + Ok(Self { pages: Some(pages), client_vm: 0, control, state, id, m5 }) + } + fn initialize_control(control: &mut KernelObject, pages: &Pages, id: u16, + submitted: u64, completed: [u8; 8]) -> Result { + match pages.layout { + PageLayout::M5(_) => { + use super::fw::m5::pool; + pool::encode_g17s_compute_state(pages.m5_layout(id)?, submitted, + &mut control.bytes_mut()[..pool::STATE_BYTES]).map_err(|_| EINVAL)?; + control.bytes_mut()[pool::COMPLETED..pool::COMPLETED + 8].copy_from_slice(&completed); + Ok(()) + } + PageLayout::Neo { blocks, .. } => { + let mut header = FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(id), class: FreeListClass::One, + page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), + blocks: blocks as u32, state_va: submitted, + }); + header.unk_54[..8].copy_from_slice(&completed); + control.write(0, header) + } + } + } + pub(crate) fn descriptor(&self) -> Result { + self.pages.as_ref().ok_or(EAGAIN)?.descriptor(self.id) + } + pub(crate) fn completed_submissions(&self) -> Result { + Ok(u64::from_le_bytes(self.completed_bytes()?)) + } + fn completed_bytes(&self) -> Result<[u8; 8]> { + let offset = if self.m5 { super::fw::m5::pool::COMPLETED } else { FreeListControl::COMPLETED }; + let pointer = self.control.pointer(offset, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked packed counter remains mapped. Firmware may + // update it; callers use it only with the queue's completion proof. + *byte = unsafe { pointer.add(index).read_volatile() }; + } + Ok(bytes) } pub(crate) fn id(&self) -> u16 { self.id } - /// EAGAIN while the pool is vacant: the owner must re-arm it first. - pub(crate) fn page_list_va(&self) -> Result { - self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) - } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + pub(crate) fn fits(&self, required: Option) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits(required)) + } pub(crate) fn populated(&self) -> bool { self.pages.is_some() } + /// Whether a view of the backing may be given to `vm`. + pub(crate) fn serves(&self, vm: &mmu::Vm) -> bool { + self.client_vm == 0 || self.client_vm == vm.id() + } /// The release record for this pool's buffer slot under the free-list /// generation the queue last published. pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { @@ -580,6 +1128,7 @@ impl ComputePool { /// The caller has observed the firmware consume this pool's release and /// frees the returned backing after releasing the device mutex. pub(crate) fn take_pages(&mut self) -> Option { + self.client_vm = 0; self.pages.take() } /// Rewrites the shared control header for fresh backing. The caller then @@ -596,39 +1145,29 @@ impl ComputePool { } /// Preserve both sets of pages on failure. The caller owns the consumed /// release and returns old pages for destruction outside the device mutex. + /// The grown backing's alias is in the VM the old backing served, which it + /// keeps serving. pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; core::mem::swap(&mut self.pages, pages); Ok(()) } fn write_pages(&mut self, pages: &Pages) -> Result { - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; - let mut bytes = [0; 8]; - for (index, byte) in bytes.iter_mut().enumerate() { - // SAFETY: The checked eight-byte range stays mapped; byte accesses - // match the unaligned packed counter the firmware writes. - *byte = unsafe { completed.add(index).read_volatile() }; - } - let mut header = FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(self.id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: pages.blocks as u32, - state_va: self.state.gpu_va(), - }); - header.unk_54[..8].copy_from_slice(&bytes); - self.control.write(0, header)?; + if pages.is_m5() != self.m5 { return Err(EINVAL); } + let completed = self.completed_bytes()?; + Self::initialize_control(&mut self.control, pages, self.id, self.state.gpu_va(), completed)?; fence(Ordering::SeqCst); Ok(()) } + /// EPERM for a VM other than the one the backing serves. pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let pages = self.pages.as_mut().ok_or(EAGAIN)?; - let address = pages.page_list_va(); - pages - .backing - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + if !self.serves(vm) { + return Err(EPERM); + } + let alias = self.pages.as_mut().ok_or(EAGAIN)?.map_client(vm)?; + self.client_vm = vm.id(); + Ok(alias) } pub(crate) fn reserve_submission(&self) -> Result { let counter = self.state.dword(0)?; @@ -826,6 +1365,11 @@ impl Lease { self.phase = LeasePhase::ReleaseSent(cursor); Ok(()) } + /// Only sent releases and inherited release witnesses consume cursors. + pub(crate) fn needs_release_observation(&self) -> bool { + self.previous_release.is_some() || matches!(self.phase, LeasePhase::ReleaseSent(_)) + } + /// Observes all older release records passed by one consumer snapshot. pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { if let Some(cursor) = self.previous_release { @@ -867,7 +1411,7 @@ mod capacity_tests { #[test] fn rejected_pool_construction_returns_all_render_ids() { - let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + let ids = RenderIds::new().unwrap(); for _ in 0..8 { let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); for _ in 0..127 { diff --git a/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs b/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs new file mode 100644 index 0000000000000..e750b01bd91c7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pure physical-ID lease rules. The caller serializes this state and supplies +//! a firmware release proof before a slot is reassigned to another pool. + +pub(super) const FIRST: u16 = 129; +pub(super) const COUNT: usize = 127; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Lease { + pub(super) slot: u16, + pub(super) owner: u64, + pub(super) generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Claim { + pub(super) lease: Lease, + pub(super) displaced: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Error { + BadOwner, + Busy, + PendingRelease, + StaleLease, + Overflow, +} + +#[derive(Clone, Copy)] +struct Slot { + owner: u64, + generation: u64, + pins: u32, + last_use: u64, +} + +impl Slot { + const EMPTY: Self = Self { owner: 0, generation: 0, pins: 0, last_use: 0 }; + + fn lease(self, index: usize) -> Lease { + Lease { slot: FIRST + index as u16, owner: self.owner, generation: self.generation } + } +} + +pub(super) struct State { + slots: [Slot; COUNT], + clock: u64, +} + +impl State { + pub(super) const fn new() -> Self { Self { slots: [Slot::EMPTY; COUNT], clock: 0 } } + + /// Constant-time cached-owner path; a recycled generation cannot be pinned. + pub(super) fn pin(&mut self, lease: Lease) -> Result { + let index = lease.slot.checked_sub(FIRST).ok_or(Error::StaleLease)? as usize; + let slot = self.slots.get_mut(index).ok_or(Error::StaleLease)?; + if slot.owner != lease.owner || slot.generation != lease.generation || slot.owner == 0 { + return Err(Error::StaleLease); + } + let clock = self.clock.checked_add(1).ok_or(Error::Overflow)?; + let pins = slot.pins.checked_add(1).ok_or(Error::Overflow)?; + slot.pins = pins; + slot.last_use = clock; + self.clock = clock; + Ok(lease) + } + + /// A successful claim is a preparation pin. It must remain pinned through + /// any out-of-lock construction and every firmware reference, not just + /// through publication. Existing-owner reacquisition does not reassign it. + pub(super) fn claim(&mut self, owner: u64, mut released: impl FnMut(Lease) -> bool) + -> Result + { + if owner == 0 { return Err(Error::BadOwner); } + let next_clock = self.clock.checked_add(1).ok_or(Error::Overflow)?; + if let Some(index) = self.slots.iter().position(|s| s.owner == owner) { + let slot = &mut self.slots[index]; + let pins = slot.pins.checked_add(1).ok_or(Error::Overflow)?; + slot.pins = pins; + slot.last_use = next_clock; + self.clock = next_clock; + return Ok(Claim { lease: slot.lease(index), displaced: None }); + } + let mut candidate = self.slots.iter().position(|s| s.owner == 0); + let mut displaced = None; + if candidate.is_none() { + // One bounded scan, with no allocation and at most one release + // query per idle slot. Normal cached-owner pinning is O(1). + let mut idle = false; + for (index, slot) in self.slots.iter().enumerate() { + if slot.pins != 0 { continue; } + idle = true; + if candidate.is_some_and(|best| self.slots[best].last_use <= slot.last_use) { continue; } + let old = slot.lease(index); + if released(old) { + candidate = Some(index); + displaced = Some(old); + } + } + if candidate.is_none() { + return Err(if idle { Error::PendingRelease } else { Error::Busy }); + } + } + let index = candidate.ok_or(Error::Busy)?; + let generation = self.slots[index].generation.checked_add(1).ok_or(Error::Overflow)?; + self.slots[index] = Slot { owner, generation, pins: 1, last_use: next_clock }; + self.clock = next_clock; + Ok(Claim { lease: self.slots[index].lease(index), displaced }) + } + + /// An old owner or old generation must never release the successor's pin. + pub(super) fn unpin(&mut self, lease: Lease) -> Result { + let index = lease.slot.checked_sub(FIRST).ok_or(Error::StaleLease)? as usize; + let slot = self.slots.get_mut(index).ok_or(Error::StaleLease)?; + if slot.owner != lease.owner || slot.generation != lease.generation || slot.pins == 0 { + return Err(Error::StaleLease); + } + slot.pins -= 1; + Ok(slot.pins == 0) + } + + /// Pool destruction may also need a pending release to finish. The caller + /// retains its memory when this refuses; dropping it is not a release proof. + pub(super) fn remove(&mut self, owner: u64, released: impl FnOnce(Lease) -> bool) + -> Result<(), Error> + { + if owner == 0 { return Err(Error::BadOwner); } + let Some(index) = self.slots.iter().position(|s| s.owner == owner) else { return Ok(()); }; + let slot = self.slots[index]; + if slot.pins != 0 { return Err(Error::Busy); } + if !released(slot.lease(index)) { return Err(Error::PendingRelease); } + self.slots[index].owner = 0; + Ok(()) + } + + /// Return one cached, unpinned registration only after its release proof. + pub(super) fn reclaim_one(&mut self, mut released: impl FnMut(Lease) -> bool) -> Option { + let index = (0..COUNT) + .filter(|&i| self.slots[i].owner != 0 && self.slots[i].pins == 0) + .filter(|&i| released(self.slots[i].lease(i))) + .min_by_key(|&i| self.slots[i].last_use)?; + let old = self.slots[index].lease(index); + self.slots[index].owner = 0; + Some(old) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn full() -> (State, Vec) { + let mut state = State::new(); + let leases = (1..=COUNT as u64).map(|owner| + state.claim(owner, |_| panic!("free slot must not need release proof")).unwrap().lease + ).collect(); + (state, leases) + } + + #[test] fn bounded_physical_namespace() { + let (state, leases) = full(); + assert_eq!(leases.iter().map(|l| l.slot).collect::>(), (129..=255).collect::>()); + assert_eq!(state.slots.len(), COUNT); + } + + #[test] fn all_preparation_pins_prevent_reassignment() { + let (mut state, _) = full(); + assert_eq!(state.claim(1000, |_| panic!("pinned slot must not be offered")), Err(Error::Busy)); + } + + #[test] fn unpinned_is_not_firmware_released() { + let (mut state, leases) = full(); + for lease in &leases { state.unpin(*lease).unwrap(); } + assert_eq!(state.claim(1000, |_| false), Err(Error::PendingRelease)); + for (i, lease) in leases.iter().enumerate() { assert_eq!(state.slots[i].lease(i), *lease); } + } + + #[test] fn released_slot_can_serve_another_logical_owner() { + let (mut state, leases) = full(); + state.unpin(leases[13]).unwrap(); + let claim = state.claim(1000, |old| old == leases[13]).unwrap(); + assert_eq!(claim.displaced, Some(leases[13])); + assert_eq!(claim.lease.slot, leases[13].slot); + assert!(claim.lease.generation > leases[13].generation); + assert_eq!(state.unpin(leases[13]), Err(Error::StaleLease)); + assert!(state.unpin(claim.lease).unwrap()); + } + + #[test] fn same_owner_reuses_without_release_proof_or_generation_change() { + let mut state = State::new(); + let a = state.claim(1, |_| false).unwrap().lease; + let b = state.claim(1, |_| panic!("same owner is not reassigned")).unwrap().lease; + assert_eq!(a, b); + assert!(!state.unpin(a).unwrap()); + assert!(state.unpin(b).unwrap()); + assert_eq!(state.unpin(a), Err(Error::StaleLease)); + } + + #[test] fn stale_generation_cannot_unpin_returned_owner() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + let other = state.claim(1000, |_| true).unwrap().lease; + state.unpin(other).unwrap(); + let returned = state.claim(leases[0].owner, |_| true).unwrap().lease; + assert_eq!(returned.slot, leases[0].slot); + assert_eq!(state.unpin(leases[0]), Err(Error::StaleLease)); + assert!(state.unpin(returned).unwrap()); + } + + #[test] fn thousands_of_logical_owners_do_not_expand_the_namespace() { + let mut state = State::new(); + for owner in 1..=4096 { + let lease = state.claim(owner, |_| true).unwrap().lease; + assert!((129..=255).contains(&lease.slot)); + assert!(state.unpin(lease).unwrap()); + } + assert_eq!(state.slots.iter().filter(|s| s.owner != 0).count(), COUNT); + } + + #[test] fn removal_waits_for_both_pins_and_firmware() { + let mut state = State::new(); + let lease = state.claim(1, |_| false).unwrap().lease; + assert_eq!(state.remove(1, |_| true), Err(Error::Busy)); + state.unpin(lease).unwrap(); + assert_eq!(state.remove(1, |_| false), Err(Error::PendingRelease)); + state.remove(1, |old| old == lease).unwrap(); + assert_eq!(state.unpin(lease), Err(Error::StaleLease)); + } + + #[test] fn cached_pin_preserves_lease_and_blocks_reclaim() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + assert_eq!(state.pin(leases[0]), Ok(leases[0])); + assert_eq!(state.claim(1000, |_| true), Err(Error::Busy)); + } + + #[test] fn stale_cached_pin_cannot_name_new_owner() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + let replacement = state.claim(1000, |_| true).unwrap().lease; + assert_eq!(state.pin(leases[0]), Err(Error::StaleLease)); + assert!(state.unpin(replacement).unwrap()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 26c6adcf34c83..2d145f4511a2d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -166,10 +166,10 @@ impl EventRecord { } /// Type 7 is a retirement witness only on the firmware whose layout and - /// visibility semantics are qualified. 25G83 must keep consuming it as an - /// unknown event until its own fault contract is established. - pub(crate) fn decode_for(&self, graph: super::variant::InitGraph) -> Result { - if graph == super::variant::InitGraph::V25G83 && self.tag() == Self::COMPUTE_FAULT { + /// visibility semantics are qualified. 25G83 and M5 must keep consuming it as an + /// unknown event until their own fault contract is established. + pub(crate) fn decode_for(&self, graph: super::variant::InitGraph, m5: bool) -> Result { + if (m5 || graph == super::variant::InitGraph::V25G83) && self.tag() == Self::COMPUTE_FAULT { return Ok(Event::Unknown(Self::COMPUTE_FAULT)); } self.decode() @@ -273,7 +273,9 @@ impl ControlRecord { match self { Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, - Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, + Self::SchedulerStateRelease(record) => if record.opcode == DeviceControlOp::M5SchedulerStateRelease as u32 { + DeviceControlOp::M5SchedulerStateRelease + } else { DeviceControlOp::SchedulerStateRelease }, Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, Self::ContextKill(_) => DeviceControlOp::ContextKill, } @@ -305,6 +307,8 @@ pub(crate) enum DeviceControlOp { IdlePowerOff = 0x0a, /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). SchedulerStateRelease = 0x14, + /// M5 encoding of the same scheduler-state release. + M5SchedulerStateRelease = 0x13, /// Release a free list's buffer slot ([`FreeListRelease`]). FreeListRelease = 0x2e, /// Stop the queues named by a closed logical scheduler. @@ -574,10 +578,11 @@ mod tests { fn v25g83_does_not_accept_an_unqualified_fault_witness() { use super::super::variant::InitGraph; let fault = record(&[7, 1, 2, 0x39, 0x300]); - assert_eq!(fault.decode_for(InitGraph::G17P), + assert_eq!(fault.decode_for(InitGraph::G17P, false), Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); - assert_eq!(fault.decode_for(InitGraph::V25G83), Ok(Event::Unknown(7))); - assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83), + assert_eq!(fault.decode_for(InitGraph::G17P, true), Ok(Event::Unknown(7))); + assert_eq!(fault.decode_for(InitGraph::V25G83, false), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83, false), Ok(Event::Unknown(7))); } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 91b3a2f58334c..76f4f96bd6ea6 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -180,6 +180,16 @@ pub(crate) struct OperandState { } impl OperandState { + /// Size of one preemption bank, the region the M5 program exchanges. + pub(crate) const fn bank_bytes(&self) -> u32 { + self.r0a5c9 as u32 + } + + /// Offset of the per-cluster tails, the M5 program's auxiliary area. + pub(crate) const fn tail_offset(&self) -> u32 { + self.r140a8 as u32 + } + /// One cluster of six cores (A18 Pro). pub(crate) const ONE_BY_SIX: OperandState = OperandState { r10229: 0x12800, @@ -209,6 +219,17 @@ impl OperandState { r0a5c9: 0x3c000, size: 0x80000, }; + /// Eight clusters of ten cores (M5 Ultra): bank stride 0x78400, tails 0x4000. The size is + /// the M5 compute queues' operand buffer, the tails' end rounded up to 16 KiB. + pub(crate) const EIGHT_BY_TEN: OperandState = OperandState { + r10229: 0xf0800, + r140a8: 0xf4800, + r10099: 0x78405, + r10091: 0xf0400, + r0a5c1: 0x0005, + r0a5c9: 0x78000, + size: 0xf8000, + }; } /// Checkpoints into the main register array, as entry counts: after the diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs index 86fdc5db75a8f..56c1f2324e2a1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -13,7 +13,7 @@ pub(crate) const RENDER_BLOCKS: usize = 44; pub(crate) const COMPUTE_BLOCKS: usize = 21; /// Render reserves eight run words per block; compute packs runs consecutively. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum RunLayout { Render, Compute, diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 2a035524bdffb..8dd18ac550a46 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -369,8 +369,16 @@ pub(crate) struct KickArgs<'a> { /// queue. A queue's first kick names `timestamp.prev()` on render queues /// and [`KickTimestamp::ZERO`] on compute queues. pub(crate) parent: KickTimestamp, + /// The parent's own predecessor on the same queue, a further implicit dependency after the + /// parent: with the G17D forms a fragment kick names it while its queue remembers a previous + /// kick ([`super::super::kick::Queue::ancestor`]); `None` elsewhere. + pub(crate) ancestor: Option, /// Kicks of other queues this kick must wait for. pub(crate) barriers: &'a [KickDependency], + /// Two-bit completion classes for each explicit dependency followed by + /// the implicit parent and the ancestor if any. Interfaces without typed + /// dependencies use `None`. + pub(crate) dependency_types: Option<&'a [u8]>, /// GPU address of the work descriptor (firmware alias). pub(crate) descriptor_va: u64, /// GPU address of the queue record. @@ -383,12 +391,23 @@ pub(crate) struct KickArgs<'a> { pub(crate) event_mask: [u64; 4], /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], - /// The compute register program includes a shader-context scratch request. + /// The stage's register program includes a shader-context scratch (SCS) + /// bracket (header bit 61; compute, TA and 3D alike). pub(crate) scs_enabled: bool, + /// The stage's register program includes a matrix bracket (bit 63). + pub(crate) compute_matrix: bool, /// Priority class of the queue. pub(crate) priority: u8, } +/// Per-stage (TA, 3D) kick header flags of a render, derived from the same +/// state that emits the SCS and matrix register brackets. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct StageKickFlags { + pub(crate) scratch: [bool; 2], + pub(crate) matrix: [bool; 2], +} + /// One kick ring entry. /// /// Live state: none. The host writes the whole entry before announcing it and @@ -397,7 +416,8 @@ pub(crate) struct KickArgs<'a> { #[derive(Copy, Clone, Debug)] pub(crate) struct KickEntry { /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. + /// 58, queue priority in bits 60:59, SCS bracket in the stage's register + /// program in bit 61 and matrix bracket in bit 63. pub(crate) header: u64, /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. pub(crate) mcache: u64, @@ -418,7 +438,10 @@ pub(crate) struct KickEntry { pub(crate) register_arrays: [u64; 4], pub(crate) unk_170: u64, pub(crate) unk_178: u64, - pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], + /// Two bits per valid dependency, in the same order as `dependencies`. + /// Zero on interfaces without typed dependencies. + pub(crate) dependency_types: u64, + pub(crate) pad_188: [u8; KICK_ENTRY_STRIDE - 0x188], } static_assert!(size_of::() == KICK_ENTRY_STRIDE); @@ -426,6 +449,7 @@ static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); +static_assert!(core::mem::offset_of!(KickEntry, dependency_types) == 0x180); static_assert!(size_of::() == 0xc); static_assert!(size_of::() == 0x10); static_assert!(size_of::() == 0x720); @@ -438,6 +462,7 @@ impl KickEntry { const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; const SCS_ENABLED: u64 = 1 << 61; + const COMPUTE_MATRIX: u64 = 1 << 63; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -449,8 +474,10 @@ impl KickEntry { /// Builds the entry described by `args`. pub(crate) fn new(args: &KickArgs<'_>) -> Result { let barriers = args.barriers.len(); + let count = barriers + 1 + usize::from(args.ancestor.is_some()); if args.qid > QID_MAX || barriers > KICK_BARRIERS_MAX + || count > KICK_DEPENDENCIES || args.qos.class > Self::QOS_CLASS_MAX || args.priority >= Policy::PRIORITIES { @@ -463,6 +490,9 @@ impl KickEntry { if args.scs_enabled { header |= Self::SCS_ENABLED; } + if args.compute_matrix { + header |= Self::COMPUTE_MATRIX; + } let mcache = match args.mcache { Some(table) => { header |= Self::MCACHE_PRESENT; @@ -475,12 +505,21 @@ impl KickEntry { for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { *slot = barrier.word(); } - dependencies[barriers] = KickDependency { - qid: args.qid, - timestamp: args.parent, + // The implicit dependencies on this queue follow the barriers: the parent, then the + // ancestor if any (`count` fits the array). + let implicit = [Some(args.parent), args.ancestor].into_iter().flatten(); + for (slot, timestamp) in dependencies[barriers..].iter_mut().zip(implicit) { + *slot = KickDependency { qid: args.qid, timestamp }.word(); + } + let mut dependency_types = 0; + if let Some(types) = args.dependency_types { + if types.len() != count || types.iter().any(|kind| *kind > 3) { + return Err(EINVAL); + } + for (index, kind) in types.iter().enumerate() { + dependency_types |= u64::from(*kind) << (2 * index); + } } - .word(); - let count = barriers + 1; let mask = if count == KICK_DEPENDENCIES { u32::MAX } else { @@ -502,7 +541,8 @@ impl KickEntry { register_arrays: args.register_arrays.map(RegisterArrayBinding::word), unk_170: 0, unk_178: Self::UNK_178, - pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + dependency_types, + pad_188: [0; KICK_ENTRY_STRIDE - 0x188], }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs new file mode 100644 index 0000000000000..d44b8b06c2bf9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 compute register programs. The operand and preemption layouts differ +//! from the other G17 profile; sharing a register count does not imply a shared +//! register program. M5 scratch has its own layout and two control writes. + +use kernel::prelude::*; +use super::super::{kick::RegisterWrite, offset_va, work_key}; +use super::initdata::Board; + +pub(crate) const MAIN_COUNT: usize = 40; +pub(crate) const ALTERNATE_COUNT: usize = 4; +pub(crate) const PREEMPT_BYTES: usize = 0x4000; +pub(crate) const CHECKPOINTS: [u16; 4] = [37, 39, 39, 40]; + +/// A dispatch control word for `board`: above four clusters it carries the cluster-topology term, +/// `0x9c` in the form for the firmware feature the 27.0 profile leaves clear (`0x1c` is the +/// feature-set form). The term is an OR. Compute and render dispatch words both take it. +pub(crate) fn dispatch_control(board: &Board, base: u64) -> u64 { + topology_dispatch_control(base, board.clusters) +} + +/// [`dispatch_control`] for `clusters`. +pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { + if clusters > 4 { base | 0x9c } else { base } +} + +/// Dependency class of a pipelined compute dependency: the class the kick-queue encoder supplies +/// for compute work (tiling's is also 2, fragment's 3). +pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2; + +/// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on +/// `board`. With the G17D forms it is derived as the kick-queue encoder derives it: the compute +/// class only for a command whose pipeline dependency type is 1, which only a late-evaluated wait +/// sets, and class 1 otherwise, which holds the kick until its parent has completed. No DRM +/// compute kick is a late-evaluated wait. Otherwise it is the pipelined class, with which a kick +/// may start while its parent still runs. +pub(crate) fn parent_dependency(board: &Board, pipelined: bool) -> u8 { + if pipelined || !board.g17d_forms() { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } +} + +/// The completion ordinal of a compute command on `board`. With the G17D forms it is the count of +/// compute commands its hardware queue published before it (zero for a new queue), wrapping as a +/// u32; render commands count separately. Otherwise it is 0 on every kick. +pub(crate) fn completion_ordinal(board: &Board, published_before: u32) -> u32 { + if board.g17d_forms() { published_before } else { 0 } +} + +/// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. +const POOL_EVENT_CONTEXTS: u32 = 64; +/// The engine of a compute pool in the event space: the firmware's record of each compute pool +/// names engine 2 beside the pool's context. +const POOL_EVENT_ENGINE: u32 = 2; +/// The entry-signal gate: event 254, which the firmware clears in the kick a current-kick +/// notification names once it has handled that record. +const ENTRY_GATE_EVENT: u32 = 254; + +/// The event mask of a compute kick of context `context_id` on `board`, published with its +/// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its +/// own context's pool event (64 x 2 + the context), which the firmware holds while it processes +/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. +/// Otherwise it is the fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). +pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> { + if !board.g17d_forms() { + return Ok([0, 0, 8, 0]); + } + if context_id >= POOL_EVENT_CONTEXTS { + return Err(EINVAL); + } + let mut mask = [0u64; 4]; + for event in [POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id, ENTRY_GATE_EVENT] { + mask[(event / 64) as usize] |= 1 << (event % 64); + } + Ok(mask) +} + +/// The descriptor words of a DRM compute kick that follow the part, not the command: the +/// preemption-state offset, the dispatch control base (before [`dispatch_control`]'s topology +/// term) and the auxiliary control's first write. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct KickTuple { + pub(crate) preempt_state_offset: u32, + pub(crate) dispatch_base: u64, + pub(crate) auxiliary_control: u64, +} + +/// The tuple of J714 compute. +pub(crate) const NORMAL_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1b80, dispatch_base: 0x14c0_8a60, auxiliary_control: 1 }; +/// The tuple the arithmetic qualification runs with on every part. The G17D runs DRM compute with +/// it: with [`NORMAL_TUPLE`] its kicks never complete. +pub(crate) const QUALIFIED_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 }; + +/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). +pub(crate) fn kick_tuple(board: &Board) -> KickTuple { + if board.g17d_forms() { QUALIFIED_TUPLE } else { NORMAL_TUPLE } +} + +const _: () = { + assert!(topology_dispatch_control(0x14c0_8a60, 8) == 0x14c0_8afc); + assert!(topology_dispatch_control(0x10c0_8860, 8) == 0x10c0_88fc); + assert!(topology_dispatch_control(0x14c0_8a60, 4) == 0x14c0_8a60); +}; + +/// The SCS scratch allocation. Pages are 4 KiB USC pool pages, independent +/// of the host's CPU page size. Layout sections and control registers are +/// specific to M5; the other G17 profile uses a different fixed region. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct ScratchRequest { + layout: u64, + driver_end: u32, + pages: u16, +} + +impl ScratchRequest { + pub(crate) const NONE: Self = Self { layout: 0, driver_end: 0, pages: 0 }; + + pub(crate) fn new(layout: u64) -> Result { + let cliques = (layout >> 24) & 0xff; + if cliques == 0 { return Ok(Self::NONE); } + let depth = layout & 0xffff; + let levels = (layout >> 16) & 0xff; + let user = (layout >> 32) & 0xffff; + let driver = layout >> 48; + let threads = cliques * 32; + // Four fixed sections followed by per-thread stacks, driver/user + // payloads and per-level state. All inputs are bounded bitfields. + let driver_end = 0x68000 + threads * (depth * 8 + driver); + // The register naming the end of the driver section holds a 24-bit + // byte offset. Reject truncation even if the total page count fits. + if driver_end > 0xffffff { return Err(EINVAL); } + let bytes = driver_end + threads * user + + cliques * 256 * (levels + u64::from(levels > 1)); + let pages = (bytes + 0x1fff) >> 12; + Ok(Self { layout, driver_end: driver_end as u32, + pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + } + + pub(crate) fn enabled(self) -> bool { self.pages != 0 } + pub(crate) fn layout(self) -> u64 { self.layout } + pub(crate) fn pages(self) -> u16 { self.pages } + pub(crate) fn size_word(self) -> u64 { + let pages = u64::from(self.pages); + pages | (pages.div_ceil(256) << 16) + } + pub(crate) fn window_word(self) -> u64 { + if self.enabled() { (1 << 24) | (((self.layout >> 24) & 0xff) << 16) } + else { 0xff0000 } + } + pub(crate) fn stack_word(self) -> u64 { + if self.enabled() { self.layout & 0xfff8 } else { 8 } + } + pub(crate) fn driver_word(self) -> u64 { + if !self.enabled() { return 0; } + let levels = (self.layout >> 16) & 0xff; + u64::from(self.driver_end) | ((levels + u64::from(levels > 1)) << 24) + } + fn checkpoints(self) -> [u16; 4] { + CHECKPOINTS.map(|count| count + if self.enabled() { 2 } else { 0 }) + } +} + +/// Per-command intersection table bindings. The pool contains 128-byte +/// blocks of eight 16-byte program entries. Addresses are UAT user addresses; +/// ownership and readable ranges are checked by command validation. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct IntersectionTables { + pub(crate) pool: u64, + pub(crate) pool_bytes: u64, + first: u32, + count: u32, + pub(crate) default_program: u64, + default_control: u64, +} + +impl IntersectionTables { + pub(crate) const NONE: Self = Self { + pool: 0, pool_bytes: 0, first: 0, count: 0, + default_program: 0, default_control: 0, + }; + + pub(crate) fn new(pool: u64, pool_bytes: u64, first: u32, count: u32, + default_program: u64, default_control: u64) -> Result { + if pool == 0 { + if pool_bytes != 0 || first != 0 || count != 0 + || default_program != 0 || default_control != 0 { + return Err(EINVAL); + } + return Ok(Self::NONE); + } + // Resource sizes are encoded in 16 KiB pages. Table lengths + // have 18 block-count bits; truncating either field changes addressing. + let end = pool.checked_add(pool_bytes).ok_or(EINVAL)?; + let blocks_end = u64::from(first) + u64::from(count); + let code = default_program & 0x0000_ffff_ffff_ffff; + if pool & 0x3fff != 0 || pool_bytes == 0 || pool_bytes & 0x3fff != 0 + || end > (1u64 << 48) || (pool_bytes >> 14) >= (1u64 << 25) + || count == 0 || count >= (1 << 18) || blocks_end * 128 > pool_bytes + || code & 63 != 0 + || default_control & !0x0000_7fff_0000_3fff != 0 + || (code == 0 && (default_program != 0 || default_control != 0)) + || (code != 0 && default_control & (1u64 << 40) == 0) { + return Err(EINVAL); + } + Ok(Self { pool, pool_bytes, first, count, default_program, default_control }) + } + + pub(crate) fn enabled(self) -> bool { self.pool != 0 } + + pub(crate) fn words(self) -> [u64; 6] { + if !self.enabled() { return [0; 6]; } + [self.pool + u64::from(self.first) * 128 | (1u64 << 51), + 0xc0000000 | (u64::from(self.count) << 7), self.pool, + self.pool_bytes >> 14, self.default_program, self.default_control] + } +} + +/// Addresses and policy owned by a queued compute command. +pub(crate) struct RegisterArgs { + pub(crate) matrix_ops: bool, + pub(crate) scratch: ScratchRequest, + pub(crate) intersection: IntersectionTables, + /// Userspace's opaque auxiliary allocation word, independent of the SCS layout. + pub(crate) auxiliary_allocation: u32, + pub(crate) preempt_va: u64, + pub(crate) cdm_va: u64, + pub(crate) operand_va: u64, + pub(crate) usage_va: u64, + /// Private-memory pool slot, as named by the usage record. + pub(crate) free_list_slot: u32, + pub(crate) context_id: u16, + pub(crate) work_state: u32, + /// Packed predecessor/current object identifiers. + pub(crate) object_ids: u64, + /// Pool target and resource policy for the program's first resource-policy write. + pub(crate) resource_policy: u64, + /// Start of the four eight-byte state words in the preemption buffer. + pub(crate) preempt_state_offset: u32, + /// Size of one operand-state region; the program exchanges two regions. + pub(crate) operand_region_bytes: u32, + pub(crate) operand_aux_offset: u32, + /// Size of the operand-state buffer at `operand_va`. + pub(crate) operand_bytes: usize, + pub(crate) resource_control: u64, + pub(crate) dispatch_control: u64, + pub(crate) auxiliary_control: u64, +} + +impl RegisterArgs { + fn validate(&self) -> Result { + if self.preempt_va & 0x3fff != 0 || self.operand_va & 0x3fff != 0 + || self.cdm_va & 7 != 0 || self.usage_va & 7 != 0 || self.context_id >= 64 { + return Err(EINVAL); + } + offset_va(self.preempt_va, PREEMPT_BYTES as u64)?; + offset_va(self.operand_va, self.operand_bytes as u64)?; + if self.preempt_state_offset & 7 != 0 + || u64::from(self.preempt_state_offset) + 32 > PREEMPT_BYTES as u64 + || self.operand_region_bytes == 0 || self.operand_region_bytes & 0xfff != 0 + || u64::from(self.operand_region_bytes) * 2 + 0x808 > self.operand_bytes as u64 + || self.operand_aux_offset & 7 != 0 + || u64::from(self.operand_aux_offset) + 8 > self.operand_bytes as u64 { + return Err(EINVAL); + } + Ok(()) + } +} + +/// Construct the complete primary program before publishing its descriptor. +pub(crate) fn main(args: &RegisterArgs) -> Result<[RegisterWrite; MAIN_COUNT]> { + args.validate()?; + let preempt = args.preempt_va; + let operand = args.operand_va; + let ids = args.object_ids; + let intersection = args.intersection.words(); + let state = u64::from(args.preempt_state_offset); + let region = u64::from(args.operand_region_bytes); + let key = work_key(args.context_id, args.work_state); + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, state)?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, state + 8)?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, state + 16)?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, state + 24)?), + RegisterWrite::new(0x1a440, 0x154024201), + RegisterWrite::new(0x0d4a1, args.resource_control), + RegisterWrite::new(0x0d411, 0x200000001), + RegisterWrite::new(0x0a599, args.resource_policy), + RegisterWrite::new(0x1a458, args.dispatch_control), + RegisterWrite::new(0x101d9, 0x1c), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), + RegisterWrite::new(0x101d1, args.scratch.size_word()), + RegisterWrite::new(0x0d479, u64::from(args.auxiliary_allocation)), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x1a141, args.scratch.driver_word()), + RegisterWrite::new(0x107a1, args.scratch.window_word() | (u64::from(args.matrix_ops) << 25)), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x0a731, args.auxiliary_control), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, args.free_list_slot.into()), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, region * 2 + 0x800)?), + RegisterWrite::new(0x140a8, offset_va(operand, args.operand_aux_offset.into())?), + RegisterWrite::new(0x10099, offset_va(operand, region + 0x405)?), + RegisterWrite::new(0x10091, offset_va(operand, region * 2 + 0x400)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, 5)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, region)?), + RegisterWrite::new(0x0a731, 0), + RegisterWrite::new(0x1a440, 0x154024209), + RegisterWrite::new(0x0a599, 0x6000400020), +]) +} + +fn program(args: &RegisterArgs) -> Result<([RegisterWrite; MAIN_COUNT + 4], usize)> { + let main = main(args)?; + let mut writes = [RegisterWrite::new(0, 0); MAIN_COUNT + 4]; + let mut count = 0; + if args.scratch.enabled() { + writes[count] = RegisterWrite::new(0x17e1, 1); + count += 1; + } + if args.matrix_ops { + writes[count] = RegisterWrite::new(0x17f1, 1); + count += 1; + } + for (index, register) in main.iter().enumerate() { + if index == 33 { + if args.scratch.enabled() { + writes[count] = RegisterWrite::new(0x17d9, 1); + count += 1; + } + if args.matrix_ops { + writes[count] = RegisterWrite::new(0x17e9, 1); + count += 1; + } + } + writes[count] = *register; + count += 1; + } + Ok((writes, count)) +} + +/// Operand-state exchange used by the alternate program. +pub(crate) fn alternate(args: &RegisterArgs) -> Result<[RegisterWrite; ALTERNATE_COUNT]> { + args.validate()?; + let operand = args.operand_va; + let region = u64::from(args.operand_region_bytes); + Ok([ + RegisterWrite::new(0x10099, offset_va(operand, 5)?), + RegisterWrite::new(0x10091, offset_va(operand, region)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, region + 0x405)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, region * 2 + 0x400)?), + ]) +} + +/// Register-array bindings in a compute kick. The first checkpoint follows +/// the initial operand-region write, before the second auxiliary-control write. +pub(crate) fn bindings(descriptor_va: u64, scratch: ScratchRequest, matrix_ops: bool) + -> Result<[Option; 4]> +{ + use super::super::kick::RegisterArrayBinding; + Ok([ + Some(RegisterArrayBinding::new(offset_va(descriptor_va, 0x40)?, (scratch.checkpoints()[0] + if matrix_ops { 2 } else { 0 }) as u8)?), + Some(RegisterArrayBinding::new(offset_va(descriptor_va, 0x760)?, ALTERNATE_COUNT as u8)?), + None, + None, + ]) +} + +/// The fixed prefix of a compute descriptor. Storage beyond this prefix is +/// owned and initialized by the runtime allocation, not described here. +pub(crate) const DESCRIPTOR_PREFIX_BYTES: usize = 0x1000; + +/// Per-command fields outside the two register programs. Fields whose bit +/// meanings are not established remain explicit opaque values. +pub(crate) struct DescriptorArgs { + pub(crate) registers: RegisterArgs, + pub(crate) descriptor_va: u64, + pub(crate) submit_sequence: u64, + pub(crate) work_state_va: u64, + pub(crate) cdm_end_va: u64, + pub(crate) stamp_va: [u64; 2], + pub(crate) stamp_value: u32, + pub(crate) qid: u32, + pub(crate) kick_timestamp: u64, + pub(crate) completion_ordinal: u32, + pub(crate) status_va: [u64; 2], + pub(crate) timestamp_va: [u64; 2], + pub(crate) shared_control_va: u64, + pub(crate) event_metadata: [u8; 16], + pub(crate) usage_fw_va: u64, + pub(crate) completion_identity: u64, +} + +/// The M5 descriptor has a shorter reserved region before shared-control +/// state. Register-array offsets are shared with G17, tail offsets are not. +#[repr(C, packed)] +pub(crate) struct Descriptor { + tag: u32, + submit_sequence: u64, + context_id: u32, + work_state_va: u64, + checkpoints: [u16; 4], + reserved_20: [u8; 0x20], + registers: super::super::kick::RegisterArray, + alternate: super::super::kick::RegisterArray, + reserved_e80: [u8; 0x58], + preempt_va: u64, + cdm_end_va: u64, + reserved_ee8: [u8; 0x20], + control: u64, + reserved_f10: [u8; 0x10], + predecessor_id: u32, + reserved_f24: u32, + unk_f28: u32, + firmware_timestamps: [u64; 2], + reserved_f3c: u32, + stamp_va: [u64; 2], + stamp_value: u32, + qid: u32, + reserved_f58: u64, + kick_timestamp: u64, + current_id: u64, + completion_ordinal: u32, + reserved_f74: [u8; 8], + status_va: [u64; 2], + timestamp_va: [u64; 2], + reserved_f9c: [u8; 0x10], + shared_control_va: u64, + shared_control_enabled: u8, + event_metadata: [u8; 16], + usage_fw_va: u64, + completion_identity: u64, + reserved_fd5: [u8; 0x2b], +} + +static_assert!(size_of::() == DESCRIPTOR_PREFIX_BYTES); +static_assert!(core::mem::offset_of!(Descriptor, registers) == 0x40); +static_assert!(core::mem::offset_of!(Descriptor, alternate) == 0x760); +static_assert!(core::mem::offset_of!(Descriptor, stamp_va) == 0xf40); +static_assert!(core::mem::offset_of!(Descriptor, completion_ordinal) == 0xf70); +static_assert!(core::mem::offset_of!(Descriptor, shared_control_va) == 0xfac); +static_assert!(core::mem::offset_of!(Descriptor, usage_fw_va) == 0xfc5); +static_assert!(core::mem::offset_of!(Descriptor, completion_identity) == 0xfcd); + +// SAFETY: Every field consists only of integer values and their arrays. +unsafe impl Zeroable for Descriptor {} + +impl Descriptor { + /// Fill the entire prefix; no address or register becomes visible before + /// validation completes. The owner publishes only after this returns. + pub(crate) fn write(&mut self, args: &DescriptorArgs) -> Result { + self.populate::(args) + } + + /// Populate a prefix already cleared by `KernelObject::initialize`. + /// The caller must provide all-zero storage so reserved bytes stay zero. + pub(crate) fn write_zeroed(&mut self, args: &DescriptorArgs) -> Result { + self.populate::(args) + } + + fn populate(&mut self, args: &DescriptorArgs) -> Result { + if args.cdm_end_va < args.registers.cdm_va || args.qid >= 128 { + return Err(EINVAL); + } + let (main, count) = program(&args.registers)?; + let alternate = alternate(&args.registers)?; + let main_va = offset_va(args.descriptor_va, 0x40)?; + let alternate_va = offset_va(args.descriptor_va, 0x760)?; + // Also validate the kick's encodable addresses before mutating storage. + bindings(args.descriptor_va, args.registers.scratch, args.registers.matrix_ops)?; + if CLEAR { + super::super::clear(self); + } + self.tag = 3; + self.submit_sequence = args.submit_sequence; + self.context_id = args.registers.context_id.into(); + self.work_state_va = args.work_state_va; + self.checkpoints = args.registers.scratch.checkpoints().map(|n| n + if args.registers.matrix_ops { 2 } else { 0 }); + super::set_register_array(&mut self.registers, main_va, &main[..count])?; + super::set_register_array(&mut self.alternate, alternate_va, &alternate)?; + self.preempt_va = args.registers.preempt_va; + self.cdm_end_va = args.cdm_end_va; + self.control = 0x154024201; + self.predecessor_id = (args.registers.object_ids >> 32) as u32; + self.unk_f28 = u32::MAX; + self.stamp_va = args.stamp_va; + self.stamp_value = args.stamp_value; + self.qid = args.qid; + self.kick_timestamp = args.kick_timestamp; + self.current_id = args.registers.object_ids as u32 as u64; + self.completion_ordinal = args.completion_ordinal; + self.status_va = args.status_va; + self.timestamp_va = args.timestamp_va; + self.shared_control_va = args.shared_control_va; + self.shared_control_enabled = u8::from(args.shared_control_va != 0); + self.event_metadata = args.event_metadata; + self.usage_fw_va = args.usage_fw_va; + self.completion_identity = args.completion_identity; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs new file mode 100644 index 0000000000000..a7b56f5bab724 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -0,0 +1,2853 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware initialization encodings and allocation layout. +//! Opaque initialized words have no inferred meaning. Runtime ownership is +//! provided by the common G17 KernelObject and GPUVM paths. + +#![allow(dead_code)] + +use core::ops::Range; +use core::result::Result; + +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; + +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; + +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; + +pub(crate) const G17P_QOS_RESOURCE_SIZE: usize = 0xe40; + +pub(crate) const REGION_C_SIZE: usize = 0x1000; + +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; + +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + + J813G17G25G76, + + J873G18G26A428, + + J714G17S26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => COMMON26_INITDATA_VERSION_WORD, + Self::J714G17S26A428 => M5_INITDATA_VERSION_WORD, + } + } + + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => COMMON26_ROOT_SIZE, + Self::J714G17S26A428 => M5_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + COMMON26_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + COMMON26_CONTROL_OPENING_OPCODE_SECONDARY + } + (Self::J714G17S26A428, InstanceRole::Primary) => M5_CONTROL_OPENING_OPCODE_PRIMARY, + (Self::J714G17S26A428, InstanceRole::Secondary) => { + M5_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + pub(crate) const fn control_register_opcode(self) -> Option { + match self { + Self::J873G18G26A428 | Self::J714G17S26A428 => None, + Self::J700G17P | Self::J813G17G25G76 => Some(CONTROL_OPENING_PRIMARY_REGISTER_OPCODE), + } + } + + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => COMMON26_BUNDLE_VIEW_OFFSETS, + Self::J714G17S26A428 => M5_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => COMMON26_BUNDLE_VIEW_EXTENTS, + Self::J714G17S26A428 => M5_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => COMMON26_SHARED_LAYOUT.block_88, + Self::J714G17S26A428 => M5_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + + pub(crate) const fn work_ring_bundle_offsets(self) -> [usize; WORK_CHANNEL_COUNT] { + match self { + Self::J873G18G26A428 => COMMON26_WORK_RING_BUNDLE_OFFSETS, + Self::J714G17S26A428 => M5_WORK_RING_BUNDLE_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => WORK_RING_BUNDLE_OFFSETS, + } + } +} + +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +const ROOT_VERSION: usize = 0x00; + +const ROOT_REGION_A: usize = 0x08; + +const ROOT_MAIN_CONFIG: usize = 0x18; + +const ROOT_REGION_C: usize = 0x20; + +const ROOT_INSTANCE_KIND: usize = 0x28; + +const ROOT_FIXED_WORD: usize = 0x2c; + +const ROOT_PAGE_SIZE: usize = 0x30; + +const ROOT_PAGE_BITS: usize = 0x32; + +const ROOT_LEVEL_COUNT: usize = 0x33; + +const ROOT_LEVELS: usize = 0x34; + +const ROOT_STATUS_A: usize = 0xa8; + +const ROOT_STATUS_B: usize = 0xb0; + +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; + +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; + +pub(crate) const UAT_PAGE_BITS: u8 = 14; + +pub(crate) const UAT_LEVEL_COUNT: usize = 3; + +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; + +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; + +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +const MAIN_HW_DATA: usize = 0x00; + +const MAIN_REPEATED_0: usize = 0x08; + +const MAIN_REPEATED_1: usize = 0x10; + +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; + +const MAIN_CHANNEL_TABLE: usize = 0x20; + +const MAIN_BUNDLE_VIEWS: usize = 0x254; + +const MAIN_REGION_VIEWS: usize = 0x2d0; + +const MAIN_SECONDARY_KIND: usize = 0x300; + +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; + +const MAIN_PRIMARY_MASK: usize = 0x3e0; + +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; + +const MAIN_PRIMARY_SENTINEL_A: usize = 0x32c; + +const MAIN_PRIMARY_SENTINEL_B: usize = 0x344; + +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; + +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; + +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; + +const MAIN_CONTROL_RING: usize = 0x4c0; + +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; + +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +pub(crate) const CONTROL_RECORD_SIZE: usize = 0x40; + +pub(crate) const CONTROL_OPENING_PRIMARY_PRODUCER: u32 = 1; + +pub(crate) const CONTROL_OPENING_SECONDARY_PRODUCER: u32 = 1; + +pub(crate) const CONTROL_OPENING_PRIMARY_REGISTER_OPCODE: u32 = 0x20; + +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; + +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; + +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; + +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; + +pub(crate) const CHANNEL_STATE_PRODUCER: usize = 2; + +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; + +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; + +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; + +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, + 0x080, 0x180, 0x280, + 0x040, 0x140, 0x240, + 0x000, 0x100, 0x200, + 0x300, +]; + +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; + +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; + +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; + +pub(crate) const HW_REGISTER_MAP_OFFSET: usize = 0x640; + +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; + +pub(crate) const HW_REGISTER_APERTURE: Range = 0xffff_fc21_8000_0000..0xffff_fc21_9000_0000; + +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE: usize = 0x20; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; + +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + + Primary = 0, + + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + + pub(crate) region_a: u64, + + pub(crate) region_c: u64, + + pub(crate) hw_data_bundle: u64, + + pub(crate) hw_data_bundle_alloc: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + + pub(crate) status_b: u64, + + pub(crate) secondary_extras: [u64; 2], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + + pub(crate) size: u32, + + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x0000000c), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x0000000f), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairRootSeparation, + PairAliasedObject, + SharedRegionSeparation, + InterfaceMismatch, + HostMappedAllocationsDisabled, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_FIXED_WORD, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +pub(crate) fn derive_channel_table_for_profile( + profile: RootProfile, + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = derive_channel_table(role, state_grid, status_a, main_config, bundle); + if role == InstanceRole::Primary { + let rings = profile.work_ring_bundle_offsets(); + for (index, entry) in table.iter_mut().take(WORK_CHANNEL_COUNT).enumerate() { + entry.ring = bundle + rings[index] as u64; + } + } + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + // J714's firmware takes both primary sentinels too; they are not + // exclusive to the older J700 ABI. + if matches!(profile, RootProfile::J700G17P | RootProfile::J714G17S26A428) { + put_u32(out, MAIN_PRIMARY_SENTINEL_A, 1); + put_u32(out, MAIN_PRIMARY_SENTINEL_B, 0xabcd_abcd); + } + + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + + RootProfile::J873G18G26A428 | RootProfile::J714G17S26A428 => { + COMMON26_MAIN_SECONDARY_KIND + } + }); + if profile == RootProfile::J714G17S26A428 { + + put_u64(out, COMMON26_MAIN_SECONDARY_POLICY_PTR, + bundle + M5_SHARED_LAYOUT.policy as u64); + } else if profile == RootProfile::J873G18G26A428 { + + put_u64(out, COMMON26_MAIN_SECONDARY_POLICY_PTR, + bundle + COMMON26_SHARED_LAYOUT.policy as u64); + } else { + + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + + return encode_common26_region_c(out); + } + if profile == RootProfile::J714G17S26A428 { + + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + + if 1 == 0 { + put_u32(out, 0x9cc, 0); + } + + if 0 != 0 { + out[0xe21] = 1; + } + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +pub(crate) const COMMON26_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; + +pub(crate) const COMMON26_ROOT_SIZE: usize = 0xc8; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum Evidence { + FirmwareProven, + ProducerChecked, + AdtProven, + Inferred, + Guessed, +} + +pub(crate) const ABI_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; + +pub(crate) const ABI_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; + +pub(crate) const ALLOC_HW_DATA: usize = 0; + +pub(crate) const ALLOC_BLOCK_C18: usize = 1; + +pub(crate) const ALLOC_BLOCK_1048: usize = 2; + +pub(crate) const ALLOC_BLOCK_E10: usize = 3; + +pub(crate) const ALLOC_POLICY: usize = 4; + +pub(crate) const ALLOC_BLOCK_68: usize = 5; + +pub(crate) const ALLOC_BLOCK_800: usize = 6; + +pub(crate) const ALLOC_BLOCK_88: usize = 7; + +pub(crate) const ALLOC_DM_RING: usize = 8; + +pub(crate) const ALLOC_MAIN_CONFIG: usize = 9; + +pub(crate) const ALLOC_CONTROL_RING: usize = 10; + +pub(crate) const ALLOC_RING_STATE: usize = 11; + +pub(crate) const ALLOC_STATUS_B: usize = 12; + +pub(crate) const ALLOC_STATUS_CONFIG: usize = 13; + +pub(crate) const ALLOC_STATUS_A: usize = 14; + +pub(crate) const ALLOC_LOG_STATE: usize = 15; + +pub(crate) const ALLOC_RING_4800: usize = 16; + +pub(crate) const ALLOC_BUFFER_28800: usize = 17; + +pub(crate) const ALLOC_BUFFER_79800: usize = 18; + +pub(crate) const ALLOC_RING_9000: usize = 19; + +pub(crate) const ALLOC_SECONDARY_EXTRA_1: usize = 20; + +pub(crate) const ALLOC_POLICY_STATE: usize = 21; + +pub(crate) const ALLOC_REGION_C: usize = 22; + +pub(crate) const ALLOC_AON_PTD: usize = 23; + +pub(crate) const ABI_BRN_TABLE_SIZE: usize = 0; + +pub(crate) const ABI_ALLOCATION_PAGE: usize = 0x4000; + +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; + +pub(crate) const PRIVATE_ALLOCATION_ENTRIES: usize = 38; + +const ROLE_SET_ENTRIES: usize = 10; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[ALLOC_HW_DATA]; + sizes[1] = table[ALLOC_BLOCK_C18]; + sizes[2] = table[ALLOC_BLOCK_1048]; + sizes[3] = table[ALLOC_BLOCK_E10]; + sizes[4] = table[ALLOC_POLICY]; + sizes[5] = table[ALLOC_BLOCK_68]; + sizes[6] = table[ALLOC_BLOCK_800]; + sizes[7] = ABI_BRN_TABLE_SIZE; + sizes[8] = table[ALLOC_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[ALLOC_DM_RING]; + ring += 1; + } + sizes[21] = table[ALLOC_MAIN_CONFIG]; + sizes[22] = table[ALLOC_CONTROL_RING]; + sizes[23] = table[ALLOC_MAIN_CONFIG]; + sizes[24] = table[ALLOC_CONTROL_RING]; + sizes +} + +pub(crate) const fn private_allocation_sizes(table: &[usize; 24]) -> [usize; PRIVATE_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; PRIVATE_ALLOCATION_ENTRIES]; + let mut state = 0; + while state <= WORK_CHANNEL_COUNT { + sizes[state] = table[ALLOC_RING_STATE]; + state += 1; + } + sizes[13] = table[ALLOC_STATUS_B]; + sizes[14] = table[ALLOC_STATUS_CONFIG]; + let role_set = [ + table[ALLOC_STATUS_A], table[ALLOC_RING_STATE], table[ALLOC_LOG_STATE], table[ALLOC_RING_STATE], + table[ALLOC_RING_STATE], table[ALLOC_RING_4800], table[ALLOC_BUFFER_28800], + table[ALLOC_BUFFER_79800], table[ALLOC_RING_9000], table[ALLOC_RING_4800], + ]; + let mut role = 0; + while role < 2 { + let mut entry = 0; + while entry < ROLE_SET_ENTRIES { + sizes[15 + role * ROLE_SET_ENTRIES + entry] = role_set[entry]; + entry += 1; + } + role += 1; + } + sizes[35] = table[ALLOC_RING_STATE]; + sizes[36] = table[ALLOC_SECONDARY_EXTRA_1]; + sizes[37] = table[ALLOC_POLICY_STATE]; + sizes +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RoleSetLayout { + + pub(crate) status_a: usize, + + pub(crate) event_state: usize, + + pub(crate) log_state: usize, + + pub(crate) trace_state: usize, + pub(crate) state_30: usize, + + pub(crate) event_ring: usize, + pub(crate) buffer_28800: usize, + + pub(crate) buffer_79800: usize, + + pub(crate) trace_ring: usize, + pub(crate) buffer_4800: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrivateClusterLayout { + pub(crate) dm_ring_states: [usize; WORK_CHANNEL_COUNT], + + pub(crate) control_state: usize, + + pub(crate) status_b: usize, + + pub(crate) status_config: usize, + pub(crate) roles: [RoleSetLayout; 2], + + pub(crate) secondary_control_state: usize, + + pub(crate) secondary_extra_1: usize, + + pub(crate) policy_state: usize, + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, ABI_ALLOCATION_PAGE), + } +} + +const fn role_set_layout(offsets: &[usize; PRIVATE_ALLOCATION_ENTRIES], first: usize) -> RoleSetLayout { + RoleSetLayout { + status_a: offsets[first], + event_state: offsets[first + 1], + log_state: offsets[first + 2], + trace_state: offsets[first + 3], + state_30: offsets[first + 4], + event_ring: offsets[first + 5], + buffer_28800: offsets[first + 6], + buffer_79800: offsets[first + 7], + trace_ring: offsets[first + 8], + buffer_4800: offsets[first + 9], + } +} + +pub(crate) const fn private_cluster_layout(table: &[usize; 24]) -> PrivateClusterLayout { + let sizes = private_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_ring_states = [0usize; WORK_CHANNEL_COUNT]; + let mut state = 0; + while state < WORK_CHANNEL_COUNT { + dm_ring_states[state] = offsets[state]; + state += 1; + } + let end = offsets[37] + sizes[37]; + PrivateClusterLayout { + dm_ring_states, + control_state: offsets[12], + status_b: offsets[13], + status_config: offsets[14], + roles: [role_set_layout(&offsets, 15), role_set_layout(&offsets, 25)], + secondary_control_state: offsets[35], + secondary_extra_1: offsets[36], + policy_state: offsets[37], + end, + total: round_up_usize(end, ABI_ALLOCATION_PAGE), + } +} + +pub(crate) const WORK_CHANNEL_DM_INDEX: [usize; WORK_CHANNEL_COUNT] = + [3, 7, 11, 2, 6, 10, 1, 5, 9, 0, 4, 8]; + +pub(crate) const fn work_ring_offsets(layout: &SharedClusterLayout) -> [usize; WORK_CHANNEL_COUNT] { + let mut rings = [0usize; WORK_CHANNEL_COUNT]; + let mut channel = 0; + while channel < WORK_CHANNEL_COUNT { + rings[channel] = layout.dm_rings[WORK_CHANNEL_DM_INDEX[channel]]; + channel += 1; + } + rings +} + +pub(crate) const M5_25G76_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_25G76); + +pub(crate) const COMMON26_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_26A428); + +pub(crate) const COMMON26_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428[ALLOC_HW_DATA]; + +pub(crate) const COMMON26_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + COMMON26_SHARED_LAYOUT.block_c18, + COMMON26_SHARED_LAYOUT.block_1048, + COMMON26_SHARED_LAYOUT.block_e10, + COMMON26_SHARED_LAYOUT.block_68, + COMMON26_SHARED_LAYOUT.block_800, +]; + +pub(crate) const COMMON26_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_C18], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_1048], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_E10], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_68], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_800], +]; + +pub(crate) const COMMON26_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&COMMON26_SHARED_LAYOUT); + +pub(crate) const COMMON26_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; + +pub(crate) const COMMON26_MAIN_SECONDARY_KIND: u32 = 0xc; + +pub(crate) const COMMON26_FWCTL_RING_OFFSET: usize = 0x40; + +pub(crate) const COMMON26_STATUS_B_FWCTL_STATE: usize = 0x4ac0; + +pub(crate) const COMMON26_STATUS_B_FWCTL_RING: usize = 0x4ac8; + +pub(crate) const COMMON26_STATUS_B_CONFIG_HEADER: usize = 0xe714; + +pub(crate) const COMMON26_STATUS_B_LINK_WORD: usize = 0xe71c; + +pub(crate) const COMMON26_STATUS_B_LINK_PTR: usize = 0xe720; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LayoutBand { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) delta: isize, + pub(crate) evidence: Evidence, +} + +const fn band(start: usize, end: usize, delta: isize, evidence: Evidence) -> LayoutBand { + LayoutBand { start, end, delta, evidence } +} + +pub(crate) const fn relocate_by_bands(bands: &[LayoutBand], offset: usize) -> Option { + let mut index = 0; + while index < bands.len() { + let band = bands[index]; + if band.start <= offset && offset < band.end { + return Some((offset as isize + band.delta) as usize); + } + index += 1; + } + None +} + +pub(crate) const COMMON26_HW_DATA_BANDS: &[LayoutBand] = &[ + + band(0x000, 0x640, 0, Evidence::Inferred), + + band(0xe88, 0xe90, 0x118, Evidence::ProducerChecked), + + band(0xe90, 0xebc, 0x120, Evidence::FirmwareProven), + + band(0xec0, 0xf5c, 0x11c, Evidence::FirmwareProven), + band(0xf5c, 0xf60, 0x124, Evidence::ProducerChecked), + + band(0xf68, 0xf98, 0x11c, Evidence::FirmwareProven), + band(0xf98, 0xfac, 0x120, Evidence::FirmwareProven), + + band(0xfb0, 0x25f8, 0x11c, Evidence::FirmwareProven), + band(0x25f8, 0x26b4, 0x120, Evidence::FirmwareProven), + band(0x26b4, 0x26f0, 0x128, Evidence::FirmwareProven), + band(0x26f0, M5_25G76_HW_DATA_SIZE, 0x12c, Evidence::FirmwareProven), +]; + +pub(crate) const COMMON26_STATUS_CONFIG_BANDS: &[LayoutBand] = &[ + band(0x000, 0x1dc, 0, Evidence::ProducerChecked), + band(0x4e0, 0x518, 0xc, Evidence::ProducerChecked), + band(0x518, 0x528, 0x14, Evidence::ProducerChecked), + band(0x530, 0x534, 0xc, Evidence::ProducerChecked), + band(0x554, 0x584, 0xc, Evidence::ProducerChecked), + band(0x584, 0x5a8, 0x1c, Evidence::ProducerChecked), + band(0x5cc, 0x628, 0x18, Evidence::ProducerChecked), +]; + +fn write_relocated_words( + words: &[(usize, u32)], + source_base: usize, + bands: &[LayoutBand], + out: &mut [u8], +) -> Result<(), InitdataError> { + for &(offset, value) in words { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + if byte == 0 { + continue; + } + let source = offset + .checked_sub(source_base) + .ok_or(InitdataError::BufferSize)? + + index; + if let Some(target) = relocate_by_bands(bands, source) { + *out.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + } + Ok(()) +} + +pub(crate) const COMMON26_HW_REGISTER_MAP_SLOTS: usize = 60; + +pub(crate) const COMMON26_HW_REGISTER_MAP_END: usize = + HW_REGISTER_MAP_OFFSET + COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE; + +pub(crate) const COMMON26_HW_CHIP_ID_OFFSET: usize = 0xfb0; + +pub(crate) const COMMON26_HW_PERF_STATE_COUNT: usize = 0x10e0; + +pub(crate) const COMMON26_HW_PERF_FREQUENCY: usize = 0x10e4; + +pub(crate) const COMMON26_HW_PERF_CORE_VOLTAGE: usize = 0x1124; + +pub(crate) const COMMON26_HW_PERF_SRAM_VOLTAGE: usize = 0x1524; + +pub(crate) const COMMON26_HW_PERF_ROWS: usize = 16; + +pub(crate) const COMMON26_HW_PERF_ROW_STRIDE: usize = 0x40; + +pub(crate) const COMMON26_HW_PERF_LANES: usize = 16; + +pub(crate) const COMMON26_HW_PERF_SCALE: usize = 0x1964; + +pub(crate) const COMMON26_HW_PERF_RELATIVE_A: usize = 0x19e4; + +pub(crate) const COMMON26_HW_PERF_RELATIVE_B: usize = 0x1a24; + +const M5_HW_AUX_RELATIVE_FREQUENCY: usize = 0x1aa4; +const M5_HW_AUX_PERF_STATE: usize = 0x1cac; +const M5_HW_AUX_PERF_STATE_SIZE: usize = 0x148; + +// Clock-domain 2: MHz, core mV, SRAM mV. The second voltage lane is unused. +const J714_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ + (24, 125, 775), (338, 590, 775), (486, 620, 775), (636, 645, 775), + (796, 685, 775), (888, 700, 775), (988, 745, 775), (1084, 770, 775), + (1182, 795, 795), (1278, 835, 835), (1374, 860, 860), + (1470, 900, 900), (1578, 960, 960), (1620, 990, 990), +]); + +// J716c ADT afr-perf-states (identical to cs-perf-states). SRAM is +// max(775, core), the rule the J714 column follows. +const J716C_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ + (24, 125, 775), (338, 590, 775), (486, 625, 775), (636, 640, 775), + (796, 690, 775), (888, 700, 775), (988, 740, 775), (1084, 765, 775), + (1182, 780, 780), (1278, 830, 830), (1374, 875, 875), + (1470, 910, 910), (1578, 975, 975), (1620, 1020, 1020), +]); + +/// An auxiliary-domain performance state: MHz, then core and SRAM mV of each of its two voltage +/// lanes. A one-lane board leaves the second lane 0. +pub(crate) type M5AuxPerfState = (u32, [u32; 2], [u32; 2]); + +// J775D ADT afr-perf-states / cs-perf-states: one lane per die, mV; SRAM floor 775 mV. +const J775D_AUX_PERF_STATES: [M5AuxPerfState; 14] = [ + (24, [125, 125], [775, 775]), (338, [600, 600], [775, 775]), + (486, [660, 655], [775, 775]), (636, [670, 665], [775, 775]), + (796, [710, 705], [775, 775]), (888, [720, 720], [775, 775]), + (988, [755, 765], [775, 775]), (1084, [800, 795], [800, 795]), + (1182, [810, 820], [810, 820]), (1278, [850, 860], [850, 860]), + (1374, [905, 915], [905, 915]), (1470, [945, 955], [945, 955]), + (1578, [1030, 1040], [1030, 1040]), (1620, [1075, 1085], [1075, 1085]), +]; + +/// One-lane auxiliary tables. +const fn one_lane(states: [(u32, u32, u32); 14]) -> [M5AuxPerfState; 14] { + let mut out = [(0, [0; 2], [0; 2]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state] = (frequency, [core, 0], [sram, 0]); + state += 1; + } + out +} + +// Evaluate the frequency-only curve at compile time; no kernel FPU use. +// Conversion uses Hz before the single-precision multiply and divide. The curve is based at +// `floor`, the lowest state the performance controllers run the GPU at: states at and below it +// read 0. +const fn relative_frequency(states: &[M5PerfState; 14], floor: usize) -> [u32; 16] { + let mut values = [0; 16]; + let base = states[floor].0 as u64 * 1_000_000; + let maximum = states[13].0 as u64 * 1_000_000; + let denominator = (maximum - base) as f32; + let mut state = floor + 1; + while state < 14 { + let frequency = states[state].0 as u64 * 1_000_000; + values[state] = (((frequency - base) as f32 * 100.0) / denominator).to_bits(); + state += 1; + } + values +} + +fn encode_m5_aux_perf_states(board: &Board, out: &mut [u8]) { + let table = &mut out[M5_HW_AUX_PERF_STATE.. + M5_HW_AUX_PERF_STATE + M5_HW_AUX_PERF_STATE_SIZE]; + table.fill(0); + put_u32(table, 0, 13); + put_u32(table, 4, 14); + for (state, &(frequency, core, sram)) in board.aux_perf_states.iter().enumerate() { + put_u32(table, 8 + state * 4, frequency); + for lane in 0..2 { + put_u32(table, 0x48 + state * 8 + lane * 4, core[lane]); + put_u32(table, 0xc8 + state * 8 + lane * 4, sram[lane]); + } + } +} + +pub(crate) const COMMON26_HW_PERF_INDEX_MAP: usize = 0x1ae4; + +pub(crate) const COMMON26_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2734; + +pub(crate) const COMMON26_QOS_POINTER_OFFSET: usize = 0x27cc; + +pub(crate) const COMMON26_HW_FIXED_WORD_OFFSET: usize = 0x2714; + +pub(crate) const COMMON26_HW_FIXED_WORD_VALUE: u32 = 0x96; + +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; + +const COMMON26_PERF_SCALE_VALUE: u32 = 0x3f82_8f5c; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindowSpec { + pub(crate) slot: usize, + pub(crate) phys: u64, + + pub(crate) size_10: u32, + + pub(crate) size_14: u32, + + pub(crate) unk_18: u64, + + pub(crate) flag: u32, +} + +const fn spec(slot: usize, phys: u64, size: u32, unk_18: u64, flag: u32) -> RegisterWindowSpec { + RegisterWindowSpec { slot, phys, size_10: size, size_14: size, unk_18, flag } +} + +pub(crate) const COMMON26_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +pub(crate) fn encode_common26_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + + let shifted = ABI_ALLOCATION_SIZES_25G76[ALLOC_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in COMMON26_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + Ok(()) +} + +// J714 descriptors carry device-specific AON table indices. +const J714_AON_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0079_0001, 0x0000_1401], + [0x6731_5452, 0x00bc_0001, 0x0000_1401], + [0x7230_5452, 0x00ce_0001, 0x0000_1401], +]; + +// J716c's rows tagged RT0g, RT1g and RT0r. +const J716C_AON_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0058_0001, 0x0000_1401], + [0x6731_5452, 0x00a3_0001, 0x0000_1401], + [0x7230_5452, 0x00b8_0001, 0x0000_1401], +]; + +// J775D: both dies' records, g0TR-g3TR and r0TR-r1TR. +const J775D_AON_RECORDS: [[u32; 3]; 6] = [ + [0x6730_5452, 0x019f_0001, 0x0000_1401], + [0x6731_5452, 0x01bb_0001, 0x0000_1401], + [0x6732_5452, 0x00a7_0001, 0x0000_1401], + [0x6733_5452, 0x00c3_0001, 0x0000_1401], + [0x7230_5452, 0x0113_0001, 0x0000_1401], + [0x7231_5452, 0x001b_0001, 0x0000_1401], +]; + +pub(crate) fn encode_common26_aon_ptd(board: &Board, out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_AON_PTD_SIZE || board.aon_records.len() * 12 > out.len() { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, record) in board.aon_records.iter().enumerate() { + for (word, value) in record.iter().enumerate() { + put_u32(out, index * 12 + word * 4, *value); + } + } + Ok(()) +} + +pub(crate) const COMMON26_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; + +pub(crate) const COMMON26_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +pub(crate) const M5_CONTROL_OPENING_OPCODE_PRIMARY: u32 = COMMON26_CONTROL_OPENING_OPCODE_PRIMARY; + +pub(crate) const M5_CONTROL_OPENING_OPCODE_SECONDARY: u32 = + COMMON26_CONTROL_OPENING_OPCODE_SECONDARY; + +pub(crate) const M5_INITDATA_VERSION_WORD: u64 = 0x0c8e_0339_8728_04c0; + +pub(crate) const M5_ROOT_SIZE: usize = 0xc8; + +pub(crate) const ABI_ALLOCATION_SIZES_26A428_M5: [usize; 24] = [ + 0x2850, 0x0c18, 0x1048, 0x0e10, 0x126b0, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x0780, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x00a8, 0x0008, 0x1ca0, 0x00e4, +]; + +pub(crate) const M5_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_26A428_M5); + +pub(crate) const M5_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&ABI_ALLOCATION_SIZES_26A428_M5); + +pub(crate) const M5_SHARED_CLUSTER_SIZE: usize = M5_SHARED_LAYOUT.total; + +pub(crate) const M5_PRIVATE_CLUSTER_SIZE: usize = M5_PRIVATE_LAYOUT.total; + +pub(crate) const M5_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_HW_DATA]; + +pub(crate) const M5_KICK_SHARED_SIZE: usize = 0x2000; +pub(crate) const M5_KICK_SHARED_POINTER: usize = 0x27dc; + +pub(crate) fn set_m5_kick_shared(out: &mut [u8], address: u64) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(address)?; + let end = address.checked_add(M5_KICK_SHARED_SIZE as u64 - 1) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(end)?; + if address & (UAT_PAGE_SIZE as u64 - 1) != 0 { + return Err(InitdataError::AddressOutOfRange); + } + put_u64(out, M5_KICK_SHARED_POINTER, address); + Ok(()) +} + +pub(crate) const M5_POLICY_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_POLICY]; + +pub(crate) const M5_STATUS_CONFIG_SIZE: usize = + ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_STATUS_CONFIG]; + +pub(crate) const M5_REGION_C_ALLOCATION: usize = + ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_REGION_C]; + +pub(crate) const M5_AON_PTD_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_AON_PTD]; + +pub(crate) const M5_SHARED_PREFIX_SIZE: usize = M5_SHARED_LAYOUT.brn_table; + +pub(crate) const M5_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + M5_SHARED_LAYOUT.block_c18, + M5_SHARED_LAYOUT.block_1048, + M5_SHARED_LAYOUT.block_e10, + M5_SHARED_LAYOUT.block_68, + M5_SHARED_LAYOUT.block_800, +]; + +pub(crate) const M5_BUNDLE_VIEW_EXTENTS: [usize; 5] = COMMON26_BUNDLE_VIEW_EXTENTS; + +pub(crate) const M5_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&M5_SHARED_LAYOUT); + +pub(crate) const M5_MAIN_BUNDLE_OFFSETS: [usize; 2] = M5_SHARED_LAYOUT.main_config; + +pub(crate) const M5_CONTROL_RING_BUNDLE_OFFSETS: [usize; 2] = + M5_SHARED_LAYOUT.control_ring; + +pub(crate) const M5_PRIVATE_STATUS_A: [usize; 2] = [ + M5_PRIVATE_LAYOUT.roles[0].status_a, + M5_PRIVATE_LAYOUT.roles[1].status_a, +]; + +pub(crate) const M5_PRIVATE_STATE_GRIDS: [usize; 2] = [ + M5_PRIVATE_LAYOUT.dm_ring_states[0], + M5_PRIVATE_LAYOUT.secondary_control_state - WORK_STATE_GRID_OFFSETS[CONTROL_CHANNEL_INDEX], +]; + +pub(crate) const M5_STATUS_CONFIG_IN_STATUS_B: usize = + M5_PRIVATE_LAYOUT.status_config - M5_PRIVATE_LAYOUT.status_b; + +pub(crate) const M5_STATUS_B_SPAN: usize = + M5_PRIVATE_LAYOUT.roles[0].status_a - M5_PRIVATE_LAYOUT.status_b; + +pub(crate) const M5_RECOVERY_STATE: usize = 0x4ae0; +/// The firmware's progress word in the primary status B, beside the recovery state: the +/// firmware moves it as it works, and a reader that finds it changed since its previous read +/// counts the GPU as progressing. +pub(crate) const M5_PROGRESS: usize = 0x4b00; +pub(crate) const M5_HOST_RECOVERY: usize = 0x4af0; +pub(crate) const M5_RECOVERY_INFO: usize = 0x4298; +pub(crate) const M5_RECOVERY_SLOT_MASK: usize = 0x4ab8; + +pub(crate) const M5_STATUS_CONFIG_LINK_WORD: usize = 0x774; + +pub(crate) const M5_STATUS_CONFIG_LINK_PTR: usize = 0x778; + +pub(crate) const M5_HW_DATA_INSERT: usize = 0x267c; + +pub(crate) const M5_HW_DATA_INSERT_LEN: usize = 8; + +/// What the M5 graph's HW data and power policy depend on per board. Every board is a T6050 +/// running the same firmware build; [`crate::g17::Config::m5`] names the board's. +pub(crate) struct Board { + /// GPU clusters. + pub(crate) clusters: u32, + /// Cores of all clusters. + pub(crate) gpu_cores: u32, + /// The core type and firmware revision the HW data names. + core_revision: (u32, u32), + /// Main-domain performance states. + pub(crate) perf_states: &'static [M5PerfState; 14], + /// Voltage tables in [`Board::perf_states`]. + perf_tables: usize, + /// Auxiliary-domain performance states. + aux_perf_states: &'static [M5AuxPerfState; 14], + /// The relative-frequency curve of [`Board::perf_states`]. + relative_frequency: [u32; 16], + /// Rows of the board's AON table the firmware reads. + aon_records: &'static [[u32; 3]], + /// Register windows the firmware maps, in device-VA order. + register_windows: &'static [M5RegisterWindowSpec], + /// The smallest parameter buffer, bytes. Renders carry it in pages too + /// (`target::min_parameter_buffer_pages()`). + min_parameter_buffer: u32, + /// Render block size and private-memory address inventory of the board's + /// render pools, bytes. + pub(crate) render_pool: (usize, u64), + /// The leakage calibration words [`Board::controller_defaults`] are valid for, or `None` + /// where they hold for every part of the board. + pub(crate) calibration: Option<[u32; 3]>, + /// The policy controllers' configuration, where the board has one. + pub(crate) controller_defaults: Option Result<(), InitdataError>>, + /// Region C's idle standby timer where the board's differs from the image's 1500 us, us. + pub(crate) idle_standby_us: Option, + /// The active memory cache controller mask, where it differs from the inherited 0xf. + mcc_mask: Option, + /// The two topology words (16, and the core count), where the board sets them. + topology_words: Option<(u32, u32)>, + /// The hotspot at which the thermal governor's fan curve starts, millidegrees C. + pub(crate) fan_curve_start_mc: i32, + /// Whether the host encodings are the G17D ones measured on the M5 Ultra + /// ([`Board::g17d_forms`]). + g17d_forms: bool, +} + +/// M5 Pro: two clusters of ten cores. +pub(crate) const J714: Board = Board { + clusters: 2, + gpu_cores: 20, + core_revision: (0x18, 4), + perf_states: &J714_GPU_PERF_STATES, + perf_tables: 2, + aux_perf_states: &J714_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J714_GPU_PERF_STATES, 1), + aon_records: &J714_AON_RECORDS, + register_windows: &J714_REGISTER_WINDOWS, + min_parameter_buffer: 0x0020_0000, + render_pool: (0x40_0000, 1 << 32), + calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), + controller_defaults: Some(encode_m5_policy_controller_defaults), + idle_standby_us: None, + mcc_mask: None, + topology_words: None, + fan_curve_start_mc: 84_000, + g17d_forms: false, +}; + +/// M5 Max: four clusters of ten cores. +pub(crate) const J716C: Board = Board { + clusters: 4, + gpu_cores: 40, + core_revision: (0x19, 4), + perf_states: &J716C_GPU_PERF_STATES, + perf_tables: 4, + aux_perf_states: &J716C_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J716C_GPU_PERF_STATES, 1), + aon_records: &J716C_AON_RECORDS, + register_windows: &J716C_REGISTER_WINDOWS, + min_parameter_buffer: 0x0030_0000, + render_pool: (0x80_0000, 1 << 33), + // Its own controller configuration, for a part with this calibration. + calibration: Some([0x7e4a4902, 0x8c4f3362, 0x96be56e5]), + controller_defaults: Some(encode_m5_policy_controller_defaults_j716c), + idle_standby_us: None, + // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. + mcc_mask: Some(0xff), + topology_words: Some((16, 40)), + fan_curve_start_mc: 84_000, + g17d_forms: false, +}; + +/// J775D's performance controllers run from P3. +const J775D_PERF_FLOOR: usize = 3; + +/// M5 Ultra: two dies of four clusters of ten cores (G17D). Values from the J775D ADT. +pub(crate) const J775D: Board = Board { + clusters: 8, + gpu_cores: 80, + // Firmware revision 2 for raw revision 0x11. + core_revision: (0x1a, 2), + perf_states: &J775D_GPU_PERF_STATES, + // One voltage table per cluster. + perf_tables: 8, + aux_perf_states: &J775D_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J775D_GPU_PERF_STATES, J775D_PERF_FLOOR), + aon_records: &J775D_AON_RECORDS, + register_windows: &J775D_REGISTER_WINDOWS, + // The minimum parameter buffer for eight clusters. + min_parameter_buffer: 0x0050_0000, + // 16 MiB render blocks in a 16 GiB inventory. + render_pool: (0x100_0000, 1 << 34), + // The power model's synthetic leakage inputs hold for every J775D: no calibration words. + calibration: None, + controller_defaults: Some(encode_m5_policy_controller_defaults_j775d), + idle_standby_us: Some(3500), + // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. + mcc_mask: Some(0xff), + topology_words: None, + // The SMC raises the fans in its automatic mode from a 97 C hotspot under a sustained + // full-chip load, with the CPU and GPU still at their top states: the curve starts there, + // above the SMC's own start rather than below it. + fan_curve_start_mc: 97_000, + g17d_forms: true, +}; + +const _: () = { + use super::target::min_parameter_buffer_pages; + assert!(J714.min_parameter_buffer as u64 == min_parameter_buffer_pages(J714.clusters) << 15); + assert!(J716C.min_parameter_buffer as u64 == min_parameter_buffer_pages(J716C.clusters) << 15); + assert!(J775D.min_parameter_buffer as u64 == min_parameter_buffer_pages(J775D.clusters) << 15); +}; + +/// Populate the enabled-USC count from the device's sampled core masks. +/// This count covers the whole device, not one cluster. +pub(crate) fn set_m5_enabled_cores(out: &mut [u8], enabled: u32) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + if enabled == 0 || enabled > 128 { + return Err(InitdataError::InterfaceMismatch); + } + put_u32(out, 0x10a4, enabled); + put_u32(out, 0x268c, enabled); + Ok(()) +} + +pub(crate) const M5_POLICY_STATE_PTR: usize = 0x8254; + +pub(crate) const M5_POLICY_AON_PTD_PTR: usize = 0x11a1c; + +pub(crate) const J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS: u32 = 200_000; + +pub(crate) const J714_GPU_PERF_BASE_PSTATE: u32 = 1; + +const J714_POWER_POLICY_MINIMUM_TABLE: usize = 0x1cab; + +const J714_POWER_POLICY_MINIMUM_ENTRIES: usize = 16 * 32; + +pub(crate) const J714_GPU_NUM_PERF_STATES: u32 = 13; + +/// A main-domain performance state: MHz, then core and SRAM mV for each voltage table. HW perf +/// lane n carries table n; lanes past the board's table count carry table 0. +pub(crate) type M5PerfState = (u32, [u32; M5_PERF_LANES], [u32; M5_PERF_LANES]); + +/// Voltage tables an [`M5PerfState`] can hold. +pub(crate) const M5_PERF_LANES: usize = 8; + +/// Two-table states (MHz, two core-voltage tables, two SRAM-voltage tables), the rest 0. +const fn two_tables(states: [(u32, u32, u32, u32, u32); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core0, core1, sram0, sram1) = states[state]; + out[state].0 = frequency; + out[state].1[0] = core0; + out[state].1[1] = core1; + out[state].2[0] = sram0; + out[state].2[1] = sram1; + state += 1; + } + out +} + +// Frequency in MHz, two core-voltage tables, two SRAM-voltage tables (mV). +pub(crate) const J714_GPU_PERF_STATES: [M5PerfState; 14] = two_tables([ + (0, 125, 125, 775, 775), (338, 605, 605, 775, 775), + (486, 620, 635, 775, 775), (636, 655, 655, 775, 775), + (796, 705, 705, 775, 775), (888, 715, 715, 775, 775), + (988, 745, 745, 775, 775), (1084, 770, 770, 775, 775), + (1182, 800, 800, 800, 800), (1278, 840, 840, 840, 840), + (1374, 880, 880, 880, 880), (1470, 925, 925, 925, 925), + (1578, 1000, 1000, 1000, 1000), (1620, 1025, 1040, 1025, 1040), +]); + +/// Four-table states (MHz, four core-voltage tables, four SRAM-voltage tables), the rest 0. +const fn four_tables(states: [(u32, [u32; 4], [u32; 4]); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state].0 = frequency; + let mut table = 0; + while table < 4 { + out[state].1[table] = core[table]; + out[state].2[table] = sram[table]; + table += 1; + } + state += 1; + } + out +} + +// J716c ADT perf-states / perf-states-sram: four tables (0 == 3, 1 == 2). +pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = four_tables([ + (0, [125, 125, 125, 125], [775, 775, 775, 775]), + (338, [635, 635, 635, 635], [775, 775, 775, 775]), + (486, [655, 640, 640, 655], [775, 775, 775, 775]), + (636, [680, 665, 665, 680], [775, 775, 775, 775]), + (796, [730, 715, 715, 730], [775, 775, 775, 775]), + (888, [740, 725, 725, 740], [775, 775, 775, 775]), + (988, [775, 760, 760, 775], [775, 775, 775, 775]), + (1084, [785, 785, 785, 785], [785, 785, 785, 785]), + (1182, [805, 810, 810, 805], [805, 810, 810, 805]), + (1278, [840, 840, 840, 840], [840, 840, 840, 840]), + (1374, [875, 865, 865, 875], [875, 865, 865, 875]), + (1470, [920, 905, 905, 920], [920, 905, 905, 920]), + (1578, [995, 980, 980, 995], [995, 980, 980, 995]), + (1620, [1015, 1015, 1015, 1015], [1015, 1015, 1015, 1015]), +]); + +// J775D ADT sgx perf-states (one table per cluster) and perf-states-sram, mV. +pub(crate) const J775D_GPU_PERF_STATES: [M5PerfState; 14] = [ + (0, [125, 125, 125, 125, 125, 125, 125, 125], [775, 775, 775, 775, 775, 775, 775, 775]), + (338, [615, 615, 615, 615, 615, 610, 610, 615], [775, 775, 775, 775, 775, 775, 775, 775]), + (486, [670, 655, 655, 670, 650, 665, 665, 650], [775, 775, 775, 775, 775, 775, 775, 775]), + (636, [695, 690, 690, 695, 690, 690, 690, 690], [775, 775, 775, 775, 775, 775, 775, 775]), + (796, [760, 745, 745, 760, 745, 740, 740, 745], [775, 775, 775, 775, 775, 775, 775, 775]), + (888, [770, 755, 755, 770, 750, 750, 750, 750], [775, 775, 775, 775, 775, 775, 775, 775]), + (988, [795, 780, 780, 795, 775, 775, 775, 775], [795, 780, 780, 795, 775, 775, 775, 775]), + (1084, [805, 805, 805, 805, 815, 800, 800, 815], [805, 805, 805, 805, 815, 800, 800, 815]), + (1182, [815, 810, 810, 815, 825, 820, 820, 825], [815, 810, 810, 815, 825, 820, 820, 825]), + (1278, [860, 845, 845, 860, 870, 855, 855, 870], [860, 845, 845, 860, 870, 855, 855, 870]), + (1374, [885, 885, 885, 885, 895, 895, 895, 895], [885, 885, 885, 885, 895, 895, 895, 895]), + (1470, [930, 930, 930, 930, 940, 935, 935, 940], [930, 930, 930, 930, 940, 935, 935, 940]), + (1578, [1020, 1005, 1005, 1020, 1030, 1015, 1015, 1030], + [1020, 1005, 1005, 1020, 1030, 1015, 1015, 1030]), + (1620, [1045, 1040, 1040, 1045, 1065, 1050, 1050, 1065], + [1045, 1040, 1040, 1045, 1065, 1050, 1050, 1065]), +]; + +pub(crate) const J714_GPU_BASE: u64 = 0x23_0000_0000; + +#[derive(Copy, Clone)] +pub(crate) struct M5RegisterWindowSpec { + pub(crate) slot: usize, + + pub(crate) instances: &'static [u64], + pub(crate) size: u32, + + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +const fn jw(slot: usize, instances: &'static [u64], size: u32, unk_18: u64, flag: u32) + -> M5RegisterWindowSpec { + M5RegisterWindowSpec { slot, instances, size, unk_18, flag } +} + +const G: u64 = J714_GPU_BASE; + +const J714_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = [ + jw(0, &[0x2_8041_4000], 0x4000, 0, 2), + jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000], 0x18000, 0, 2), + jw(4, &[0x2_8046_8000], 0x4000, 0, 0), + jw(9, &[0x2_8425_0000], 0x4000, 0, 2), + jw(10, &[0x2_8424_0000], 0x4000, 0, 0), + jw(11, &[0x2_8425_8000], 0x4000, 0, 2), + jw(12, &[0x5_0965_c000], 0x4000, 0, 2), + jw(14, &[0x23_8030_0000], 0x8000, 0, 0), + jw(15, &[0x2_8cdc_0000], 0x24000, 0, 2), + jw(17, &[G], 0x21500, 0, 2), + jw(18, &[G + 0xeb_4000], 0x4000, 0, 2), + jw(20, &[0x23_0300_0000, 0x23_0320_0000], 0x40000, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(23, &[G + 0xf0_0000], 0x10000, 0, 2), + jw(24, &[G + 0xec_0000, G + 0xed_0000], 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x20000, 0xd6_0000, 2), + jw(38, &[G + 0xd4_4000], 0x4000, 0xd4_4000, 2), + jw(39, &[G + 0xd4_c000], 0x200, 0xd4_c000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(42, &[G + 0xe4_0000], 0x4000, 0xe4_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(48, &[G + 0xe6_0000], 0x58, 0xe6_0000, 2), + jw(52, &[G + 0xe4_8000], 0x8000, 0, 2), + jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), +]; + +const J714_REGISTER_WINDOW_58: M5RegisterWindowSpec = + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); + +/// J714's windows, ending with slot 58. +const J714_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { + let mut windows = [J714_REGISTER_WINDOW_58; 30]; + let mut index = 0; + while index < J714_REGISTER_WINDOW_SPECS.len() { + windows[index] = J714_REGISTER_WINDOW_SPECS[index]; + index += 1; + } + windows +}; + +/// J714 with all eight AMCCs in slot 3 (J716c amcc-enable-mask 0xff, MCC +/// reg[14..21] + 0x104000) and four instances, one per cluster, in slots 20 and +/// 24, where J714 has two at the same stride. Slot 44 is a fixed pair on every M5 +/// part. +const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { + let mut specs = J714_REGISTER_WINDOW_SPECS; + assert!(specs[1].slot == 3 && specs[11].slot == 20 && specs[14].slot == 24); + specs[1] = jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, + 0x22_2810_4000, 0x22_2a10_4000, 0x22_2c10_4000, 0x22_2e10_4000], + 0x18000, 0, 2); + specs[11] = jw(20, &[0x23_0300_0000, 0x23_0320_0000, 0x23_0340_0000, 0x23_0360_0000], + 0x40000, 0, 2); + specs[14] = jw(24, &[G + 0xec_0000, G + 0xed_0000, G + 0xee_0000, G + 0xef_0000], + 0x10000, 0, 2); + specs +}; + +/// J716c's windows, ending with slot 58. +const J716C_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { + let mut windows = [J714_REGISTER_WINDOW_58; 30]; + let mut index = 0; + while index < J716C_REGISTER_WINDOW_SPECS.len() { + windows[index] = J716C_REGISTER_WINDOW_SPECS[index]; + index += 1; + } + windows +}; + +/// Second die of J775D. +const D: u64 = 0x40_0000_0000; + +/// J775D: the explicit instances of both dies (ADT), in device-VA order. +const J775D_REGISTER_WINDOWS: [M5RegisterWindowSpec; 32] = [ + jw(0, &[0x2_8041_4000], 0x4000, 0, 2), + // One per active memory cache controller. + jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, + 0x22_2810_4000, 0x22_2a10_4000, 0x22_2c10_4000, 0x22_2e10_4000], 0x18000, 0, 2), + jw(4, &[0x2_8046_8000], 0x4000, 0, 0), + jw(9, &[0x2_8425_0000], 0x4000, 0, 2), + jw(10, &[0x2_8424_0000], 0x4000, 0, 0), + jw(11, &[0x2_8425_8000], 0x4000, 0, 2), + jw(12, &[0x5_0965_c000], 0x4000, 0, 2), + jw(13, &[0x45_0965_c000], 0x4000, 0, 2), + jw(14, &[0x23_8030_0000], 0x8000, 0, 0), + jw(15, &[0x2_8cdc_0000], 0x24000, 0, 2), + jw(17, &[G], 0x21500, 0, 2), + jw(18, &[G + 0xeb_4000, G + 0xeb_4000 + D], 0x4000, 0, 2), + jw(20, &[G + 0x300_0000, G + 0x320_0000, G + 0x340_0000, G + 0x360_0000, + G + 0x380_0000, G + 0x3a0_0000, G + 0x3c0_0000, G + 0x3e0_0000, + 0x23_035a_8000, 0x23_037a_8000, 0x23_03da_8000, 0x23_03fa_8000], 0x40000, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(23, &[G + 0xf0_0000, G + 0xf0_0000 + D], 0x10000, 0, 2), + jw(24, &[G + 0xec_0000, G + 0xed_0000, G + 0xee_0000, G + 0xef_0000, + G + 0xec_0000 + D, G + 0xed_0000 + D, G + 0xee_0000 + D, G + 0xef_0000 + D], + 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x20000, 0xd6_0000, 2), + jw(38, &[G + 0xd4_4000], 0x4000, 0xd4_4000, 2), + jw(39, &[G + 0xd4_c000], 0x200, 0xd4_c000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(42, &[G + 0xe4_0000], 0x4000, 0xe4_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(48, &[G + 0xe6_0000], 0x58, 0xe6_0000, 2), + jw(52, &[G + 0xe4_8000], 0x8000, 0, 2), + jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2), + jw(59, &[0x23_0358_0000, 0x23_0378_0000, 0x23_03d8_0000, 0x23_03f8_0000], 0x60000, 0, 2), +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) device_va: u64, + pub(crate) count: usize, + pub(crate) size: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +/// The most register windows of any board. +pub(crate) const M5_MAX_REGISTER_WINDOWS: usize = J775D_REGISTER_WINDOWS.len(); + +impl Board { + /// Whether the board takes the host encodings measured on the M5 Ultra (G17D) where they + /// differ from J714's, as `super::compute` and the render registers read them. + pub(crate) const fn g17d_forms(&self) -> bool { + self.g17d_forms + } + + /// The board's `index`th register window. + pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { + self.register_windows.get(index) + } + + /// The board's register windows: the first [`Board::register_window_count`] entries. + pub(crate) fn register_windows(&self) -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] { + m5_register_windows(self.register_windows) + } + + /// Register windows of the board. + pub(crate) fn register_window_count(&self) -> usize { + self.register_windows.len() + } +} + +fn m5_register_windows(specs: &'static [M5RegisterWindowSpec]) + -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] +{ + let mask = UAT_PAGE_SIZE as u64 - 1; + let mut out = [M5RegisterWindow { slot: 0, device_va: 0, count: 0, size: 0, unk_18: 0, flag: 0 }; + M5_MAX_REGISTER_WINDOWS]; + let mut next: u64 = 0; + for (window, spec) in out.iter_mut().zip(specs) { + let va = next + (spec.instances[0] & mask); + let span = spec.instances.len() as u64 * ((spec.size as u64 + mask) & !mask); + *window = M5RegisterWindow { + slot: spec.slot, + device_va: HW_REGISTER_APERTURE.start + va, + count: spec.instances.len(), + size: spec.size, + unk_18: spec.unk_18, + flag: spec.flag, + }; + let end = (va + span + mask) & !mask; + next = (end + UAT_PAGE_SIZE as u64 + 0x7fff) & !0x7fff; + } + out +} + +pub(crate) fn encode_m5_register_map( + board: &Board, + windows: &[M5RegisterWindow], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, window) in windows.iter().enumerate() { + let spec = board.register_windows.get(index).ok_or(InitdataError::RegisterWindow)?; + if window.slot >= COMMON26_HW_REGISTER_MAP_SLOTS + || !HW_REGISTER_APERTURE.contains(&window.device_va) + { + return Err(InitdataError::RegisterWindow); + } + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, spec.instances[0]); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.count as u32 * window.size); + put_u32(out, base + 0x14, window.size); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + Ok(()) +} + +pub(crate) fn encode_m5_hw_data( + board: &Board, + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + windows: &[M5RegisterWindow], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high + .checked_add(G17P_QOS_RESOURCE_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; M5_25G76_COMPLETION_QUEUE_COUNT * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + let g18 = &mut out[..COMMON26_HW_DATA_SIZE]; + g18.fill(0); + write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, COMMON26_HW_DATA_BANDS, g18)?; + encode_m5_register_map(board, windows, + &mut g18[HW_REGISTER_MAP_OFFSET..COMMON26_HW_REGISTER_MAP_END])?; + put_u32(g18, COMMON26_HW_CHIP_ID_OFFSET, 0x6050); + put_u32(g18, COMMON26_HW_PERF_STATE_COUNT, J714_GPU_NUM_PERF_STATES); + put_u32(g18, COMMON26_HW_FIXED_WORD_OFFSET, COMMON26_HW_FIXED_WORD_VALUE); + + put_u32(g18, 0x0fb4, 1); + // J714's 26A428 HW data carries 1 here. The firmware copies this + // header at initialization. + put_u32(g18, 0x0fb8, 1); + let (core_type, revision) = board.core_revision; + put_u32(g18, 0x10d4, core_type); + put_u32(g18, 0x10d8, revision); + put_u32(g18, 0x10dc, board.gpu_cores); + put_u32(g18, 0x283a, 1); + put_u32(g18, 0x283e, 1); + let states = board.perf_states; + for (state, &(frequency, core, sram)) in states.iter().enumerate() { + put_u32(g18, COMMON26_HW_PERF_FREQUENCY + state * 4, frequency); + put_u32(g18, COMMON26_HW_PERF_SCALE + state * 4, COMMON26_PERF_SCALE_VALUE); + + let relative = frequency * 100 / states[13].0; + put_u32(g18, COMMON26_HW_PERF_RELATIVE_A + state * 4, relative); + for lane in 0..COMMON26_HW_PERF_LANES { + let row = state * COMMON26_HW_PERF_ROW_STRIDE + lane * 4; + let table = if lane < board.perf_tables { lane } else { 0 }; + put_u32(g18, COMMON26_HW_PERF_CORE_VOLTAGE + row, core[table]); + put_u32(g18, COMMON26_HW_PERF_SRAM_VOLTAGE + row, sram[table]); + } + } + for index in 0..COMMON26_HW_PERF_ROWS { + put_u32(g18, COMMON26_HW_PERF_INDEX_MAP + index * 4, index as u32); + put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, board.relative_frequency[index]); + put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, board.relative_frequency[index]); + } + encode_m5_aux_perf_states(board, g18); + let table = COMMON26_KSM_COMPLETION_TABLE_OFFSET; + g18[table..table + completion.len()].copy_from_slice(&completion); + put_u64(g18, COMMON26_QOS_POINTER_OFFSET, qos_high); + + let insert = M5_HW_DATA_INSERT; + let len = M5_HW_DATA_INSERT_LEN; + out.copy_within(insert..COMMON26_HW_DATA_SIZE, insert + len); + out[insert..insert + len].fill(0); + put_u32(out, 0x2660, 0); + put_u32(out, 0x2728, 0); + put_u32(out, 0x272c, 1); + if let Some(mask) = board.mcc_mask { + put_u64(out, 0x10a8, mask); + } + if let Some((word, cores)) = board.topology_words { + put_u32(out, 0x100c, word); + put_u32(out, 0x1048, cores); + } + // The inherited J813 minimum parameter-buffer size is replaced per board. + put_u32(out, 0x26b0, board.min_parameter_buffer); + Ok(()) +} + +pub(crate) const M5_POLICY_MAXIMUM_POWER: usize = 0xa18; + +/// Region C's idle standby timer: how long the idle GPU waits before it goes to standby, us. +pub(crate) fn set_m5_region_idle_standby(out: &mut [u8], microseconds: u32) + -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { return Err(InitdataError::BufferSize); } + if microseconds == 0 { return Err(InitdataError::InterfaceMismatch); } + put_u32(out, 0x7c8, microseconds); + Ok(()) +} + +/// Apply the calibrated maximum shared by the policy and host power input. +pub(crate) fn set_m5_region_maximum_power(out: &mut [u8], maximum: u32) + -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { return Err(InitdataError::BufferSize); } + if maximum == 0 { return Err(InitdataError::InterfaceMismatch); } + put_u32(out, 0x88, maximum); + Ok(()) +} + +pub(crate) fn encode_m5_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { + return Err(InitdataError::BufferSize); + } + out.fill(0); + encode_common26_region_c(&mut out[..REGION_C_SIZE])?; + + out.copy_within(0xe10..0xe78, 0x1c34); + out[0x9e4..0x1c34].fill(0); + out.copy_within(0x7f4..0x9e4, 0x7f8); + put_u32(out, 0x7f4, 1); + + put_u32(out, 0x9e0, 50); + + put_u32(out, 0x08c, J714_GPU_NUM_PERF_STATES * 100); + put_u32(out, 0x090, J714_GPU_NUM_PERF_STATES * 100); + put_u32(out, 0x09c, (J714_GPU_NUM_PERF_STATES + 1) * 100); + put_u32(out, 0x034, 1); + put_u32(out, 0x038, 1); + put_u32(out, 0x7e4, 0x3f19_999a); + put_u32(out, 0x1c30, 250); + const FW_UTIL_TUNABLES: [u8; 32] = [ + 1, 0, 0, 0, 0, 2, 2, 1, 1, 40, 10, 1, 1, 1, 2, 40, + 10, 1, 1, 1, 2, 40, 10, 1, 1, 1, 1, 40, 10, 1, 1, 0, + ]; + out[0x9e8..0xa08].copy_from_slice(&FW_UTIL_TUNABLES); + Ok(()) +} + +pub(crate) fn encode_m5_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + status_b: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_STATUS_B_SPAN { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + required_address(status_b)?; + let config = M5_STATUS_CONFIG_IN_STATUS_B; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, COMMON26_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, COMMON26_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, COMMON26_STATUS_B_CONFIG_HEADER, 1); + put_u32(out, COMMON26_STATUS_B_LINK_WORD, 0); + put_u64(out, COMMON26_STATUS_B_LINK_PTR, + status_b + (config + M5_STATUS_CONFIG_LINK_WORD) as u64); + write_relocated_words(M5_25G76_STATUS_CONFIG_WORDS, 0, COMMON26_STATUS_CONFIG_BANDS, + &mut out[config..config + M5_STATUS_CONFIG_SIZE])?; + put_u32(out, config + M5_STATUS_CONFIG_LINK_WORD, 1); + put_u64(out, config + M5_STATUS_CONFIG_LINK_PTR, + status_b + COMMON26_STATUS_B_LINK_WORD as u64); + Ok(()) +} + +pub(crate) fn encode_m5_private_cluster( + private_base: u64, + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = M5_PRIVATE_LAYOUT; + if out.len() != layout.total { + return Err(InitdataError::BufferSize); + } + required_address(private_base)?; + let private_end = private_base + .checked_add(layout.total as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(private_end - 1)?; + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state + .checked_add(COMMON26_FWCTL_RING_OFFSET as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + if fwctl_state < private_end && private_base < fwctl_ring + CONTROL_RECORD_SIZE as u64 { + return Err(InitdataError::PairAliasedObject); + } + out.fill(0); + let status_b = layout.status_b; + let status_a = layout.roles[0].status_a; + encode_m5_primary_status_b(fwctl_state, fwctl_ring, private_base + status_b as u64, + &mut out[status_b..status_a])?; + for role in layout.roles { + encode_status_block(&mut out[role.status_a..role.status_a + STATUS_BLOCK_SIZE])?; + } + + let extra = layout.secondary_extra_1; + encode_secondary_root_extra_1_for_profile(RootProfile::J873G18G26A428, + &mut out[extra..extra + STATUS_BLOCK_SIZE])?; + put_u32(out, layout.control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_PRIMARY_PRODUCER); + put_u32(out, layout.secondary_control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_SECONDARY_PRODUCER); + Ok(()) +} + +pub(crate) fn encode_m5_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = M5_SHARED_LAYOUT; + if out.len() < M5_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + out[layout.block_c18..M5_SHARED_PREFIX_SIZE].fill(0); + let m5 = M5_25G76_SHARED_LAYOUT; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.block_1048..m5.block_e10).contains(&offset) { + put_u32(out, layout.block_1048 + (offset - m5.block_1048), value); + } + } + let policy = &mut out[layout.policy..layout.policy + M5_POLICY_SIZE]; + put_u32(policy, 0x04, J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS); + put_u32(policy, 0x08, J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS); + + for (offset, value) in [ + (0x10, 4), (0x14, 0x3f80_0000), (0x2c, 1), (0x30, 1), + (0x40, J714_GPU_PERF_BASE_PSTATE * 100), (0x44, 1), + (0x48, J714_GPU_NUM_PERF_STATES * 100), (0x50, 100), + (0x60, 4), (0x64, 0x3f80_0000), (0x6c, 1), (0x70, 1), + (0x80, J714_GPU_PERF_BASE_PSTATE * 100), (0x84, 1), + (0x88, J714_GPU_NUM_PERF_STATES * 100), (0x90, 100), + + (0x9c, 1), (0xa0, 1), (0xa4, 1), (0xa8, 1), + ] { + put_u32(policy, offset, value); + } + + for index in 0..J714_POWER_POLICY_MINIMUM_ENTRIES { + put_u16(policy, J714_POWER_POLICY_MINIMUM_TABLE + index * 2, i16::MAX as u16); + } + + for (offset, value) in [ + (0xb58, 1), + (0xb60, 0x3f4c_cccd), + (0xb64, 0x3f7e_b852), + (0xb68, 0x3e4c_cccd), + (0xb6c, 0x3ba3_d70a), + (0xb70, 0x3f85_0e56), + (0xb74, 0x3f85_0e56), + (0xb80, 0x4060_0000), + (0xb84, 0x4060_0000), + ] { + put_u32(policy, offset, value); + } + put_u64(policy, M5_POLICY_STATE_PTR, bindings.state); + put_u64(policy, M5_POLICY_AON_PTD_PTR, bindings.aon_ptd); + Ok(()) +} + +/// Initial controller inputs for J714 with the matched calibration. +pub(crate) fn encode_m5_policy_controller_defaults(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() < M5_SHARED_PREFIX_SIZE { return Err(InitdataError::BufferSize); } + // Both calibrated hardware curves, as J714's firmware expects them. + const RELATIVE_POWER_MAIN: [u32; 16] = [ + 0x00000000, 0x4106f651, 0x413ea3b9, 0x4180c1c6, + 0x41b433fc, 0x41ccaf2d, 0x41f7a8ee, 0x42104b59, + 0x4228d2f7, 0x424956c5, 0x426d29a9, 0x428da278, + 0x42b4e595, 0x42c80000, 0x00000000, 0x00000000, + ]; + for (index, &value) in RELATIVE_POWER_MAIN.iter().enumerate() { put_u32(out, 0x19e4 + index * 4, value); } + const RELATIVE_POWER_AUX: [u32; 16] = [ + 0x00000000, 0x4105a0ed, 0x414a36f5, 0x418b74a8, + 0x41c1510d, 0x41df97fb, 0x420be387, 0x4223193c, + 0x423cc4e6, 0x42604354, 0x427ef922, 0x42957243, + 0x42b6ec22, 0x42c80000, 0x00000000, 0x00000000, + ]; + for (index, &value) in RELATIVE_POWER_AUX.iter().enumerate() { put_u32(out, 0x1a64 + index * 4, value); } + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const INITIAL: &[(usize, u32)] = &[ + (0xd4, 0x3f828f5c), + (0xd8, 0x3f828f5c), + (0xdc, 0x3f828f5c), + (0xe0, 0x3f828f5c), + (0xe4, 0x3f828f5c), + (0xe8, 0x3f828f5c), + (0xec, 0x3f828f5c), + (0xf0, 0x3f828f5c), + (0xf4, 0x3f828f5c), + (0xf8, 0x3f828f5c), + (0xfc, 0x3f828f5c), + (0x100, 0x3f828f5c), + (0x104, 0x3f828f5c), + (0x108, 0x3f828f5c), + (0x9cc, 0x271), + (0x9d8, 0x3f7f2e9f), + (0x9e0, 0x3b516154), + (0x9e8, 0x3ca59586), + (0x9f4, 0x47c12200), + (0x9f8, 0x40a90fdb), + (0xa00, 0xbc576554), + (0xa04, 0x28), + (0xa08, J714_GPU_NUM_PERF_STATES * 100), + (0xa0c, J714_GPU_NUM_PERF_STATES * 100), + (M5_POLICY_MAXIMUM_POWER, 0x18244), + (0xa4c, J714_GPU_NUM_PERF_STATES * 100), + (0xa84, 0x8), + (0xa90, 0x3f000000), + (0xa98, 0x3f000000), + (0xaa0, 0x3e0b4396), + (0xaac, 0x47800000), + (0xab0, 0x3e99999a), + (0xabc, 0x28), + (0xac0, J714_GPU_NUM_PERF_STATES * 100), + (0xad0, 0x18244), + (0xadc, 0x10), + (0xae4, 0x5dc00), + (0xb3c, 0x4b), + (0xb44, 0x55), + (0xb48, 0x32), + (0xb4c, 0x6), + (0xb50, 0x1), + (0xb54, 0x6), + (0xb7c, 0x42be0000), + (0xb88, 0x414a1af3), + (0xb8c, 0x64), + (0xb90, J714_GPU_NUM_PERF_STATES * 100), + (0xb94, 0x64), + (0xba0, 0x4a), + (0xba8, 0x1), + (0xbd4, 0x64), + (0xbf4, 0x401), + (0xbf8, 0x6400), + (0xbfc, 0xa00), + (0xc00, 0x1f4000), + (0xc04, 0x12c00), + (0xc08, 0xfa000), + (0xc0c, 0x12c00), + (0xc10, 0x7d000), + (0xc14, 0x12c00), + (0xc18, 0x3e800), + (0xc1c, 0x19000), + (0xc20, 0x100), + (0xc78, 0x1), + (0x22ac, 0x47800000), + (0x22b0, 0x42c80000), + (0x22b4, 0x44a28000), + (0x22b8, 0x654), + (0x22cc, 0x64), + (0x22d4, 0x42c80000), + (0x22d8, J714_GPU_NUM_PERF_STATES * 100), + (0x22e0, 0x3f4ccccd), + (0x22e4, 0x3e4ccccd), + (0x2328, 0x6fcc3fcc), + (0x232c, 0x6fcc3fcc), + (0x234c, 0x1f40), + (0x2354, 0x4), + (0x2368, 0x3f800000), + (0x2370, 0x40ad5555), + (0x237c, 0x47800000), + (0x2380, 0x42080000), + (0x238c, 0x28), + (0x2390, J714_GPU_NUM_PERF_STATES * 100), + (0x2394, J714_GPU_NUM_PERF_STATES * 100), + (0x23a0, 0x2af8), + (0x23d4, J714_GPU_NUM_PERF_STATES * 100), + (0x2cec, 0x20), + (0x2cf0, 0x42000006), + (0x2cfc, 0x3), + (0x2d24, 0x1f40), + (0x2d2c, 0x4), + (0x2d40, 0x3f800000), + (0x2d48, 0x40855555), + (0x2d54, 0x47800000), + (0x2d58, 0x42080000), + (0x2d64, 0x64), + (0x2d68, J714_GPU_NUM_PERF_STATES * 100), + (0x2d6c, J714_GPU_NUM_PERF_STATES * 100), + (0x2d78, 0x2bc0), + (0x2dac, J714_GPU_NUM_PERF_STATES * 100), + (0x36c4, 0x20), + (0x36c8, 0x42000006), + (0x3fdc, 0xb8520000), + (0x3fe0, 0x3f7e), + (0x3fe4, 0xd70a0000), + (0x3fe8, 0x3ba3), + (0x3ff0, 0x43c8), + (0x3ffc, 0x4780), + (0x4000, 0x42c8), + (0x4008, 0x384bac8), + (0x400c, 0x5140000), + (0x4010, 0x3e80000), + (0x4018, 0x80000000), + (0x401c, 0x26ac453b), + (0x4050, 0x5140000), + (0x406c, 0xc80000), + (0x4124, J714_GPU_NUM_PERF_STATES * 100), + (0x4138, 0x1), + (0x41cc, 0x10001), + (0x41d0, 0x40000), + (0x41e8, 0x23f0000), + (0x41ec, 0x10000), + (0x41f0, 0x10000), + (0x4208, 0x2400000), + (0x420c, 0x10000), + (0x4210, 0x10000), + (0x4228, 0x2410000), + (0x42d4, 0x640000), + (0x42f8, 0x1), + (0x438c, 0x10001), + (0x4390, 0x40000), + (0x43a8, 0x25b0000), + (0x43ac, 0x10000), + (0x43b0, 0x10000), + (0x43c8, 0x25c0000), + (0x43cc, 0x10000), + (0x43d0, 0x10000), + (0x43e8, 0x25d0000), + (0x4494, 0x640000), + (0x449c, 0x1), + (0x44a4, 0x18244), + (0x44a8, 0x18244), + (0x44ac, 0x18244), + (0x44bc, 0x3f800000), + (0x44c0, 0x4), + (0x44c4, 0x8), + (0x44c8, 0x2ee00), + (0x4598, 0x7c), + (0x45a4, 0x3f77bdef), + (0x45ac, 0x3d042108), + (0x45b4, 0x3cf5c290), + (0x45c0, 0x47800000), + (0x45c4, 0x3f19999a), + (0x45d0, 0x28), + (0x45d4, J714_GPU_NUM_PERF_STATES * 100), + (0x45d8, J714_GPU_NUM_PERF_STATES * 100), + (0x45e0, 0x47c12200), + (0x45e4, 0x18244), + (0x45f0, 0xf8), + (0x45f8, 0x5ad200), + (0x4618, J714_GPU_NUM_PERF_STATES * 100), + (0x4658, 0x1), + (0x4728, 0x32), + (0x472c, 0x1), + (0x4734, 0x3f638e39), + (0x4738, 0x3f2aaaab), + (0x473c, 0x3de38e39), + (0x4740, 0x3eaaaaab), + (0x4744, 0xbecccccd), + (0x4748, 0xbecccccd), + (0x4750, 0x47800000), + (0x4754, 0xc0a00000), + (0x4758, 0xc0a00000), + (0x4760, 0x64), + (0x4764, J714_GPU_NUM_PERF_STATES * 100), + (0x4768, 0x64), + (0x4770, 0x45fa0000), + (0x4774, 0x578), + (0x477c, 0xe), + (0x4780, 0x48), + (0x4784, 0x18), + (0x4788, 0x1a5e00), + (0x4790, 0x8ca00), + (0x479c, 0x47800000), + (0x47c4, 0x1f40), + (0x47c8, 0x4), + (0x47cc, 0x9c4), + (0x47d0, 0x258), + (0x47d4, 0x2), + (0x47d8, 0x4), + (0x47dc, 0x32), + (0x480c, 0x47800000), + (0x481c, 0x28), + (0x4820, J714_GPU_NUM_PERF_STATES * 100), + (0x48ec, 0x1), + (0x48f0, 0x1), + (0x4910, 0xdac), + (0x4914, 0x3e8), + (0x4918, 0xbb8), + (0x49b8, J714_GPU_NUM_PERF_STATES * 100), + (0x49bc, J714_GPU_NUM_PERF_STATES * 100), + (0x49c0, 0x64), + (0x49c4, 0x20), + (0x49cc, 0x1), + (0x49d0, 0x1), + (0x49fc, 0x1f4), + (0x4a98, J714_GPU_NUM_PERF_STATES * 100), + (0x4a9c, J714_GPU_NUM_PERF_STATES * 100), + (0x4aa0, 0x64), + (0x4aa4, 0x20), + (0x4ac8, 0x4), + (0x4ad4, 0x3f780000), + (0x4adc, 0x3d000000), + (0x4ae4, 0x40a00000), + (0x4af0, 0x47800000), + (0x4af4, 0x41700000), + (0x4b00, 0x4b0), + (0x4b04, J714_GPU_NUM_PERF_STATES * 100), + (0x4b08, J714_GPU_NUM_PERF_STATES * 100), + (0x4b14, J714_GPU_NUM_PERF_STATES * 100), + (0x4b48, J714_GPU_NUM_PERF_STATES * 100), + (0x4b94, 0x42c80000), + (0x4ba4, 0x3), + (0x4ba8, 0x6fcc3fcc), + (0x4bac, 0x6fcc3fcc), + (0x4bcc, 0x7d), + (0x4bd0, 0x1), + (0x11a24, 0x67305452), + (0x11a28, 0x67315452), + (0x11a6c, 0x72305452), + (0x121a4, 0x1), + (0x122e0, 0x186a), + (0x121b0, 0x44d5c000), + (0x121b4, 0x44d20000), + (0x12230, 0x43140000), + (0x12234, 0x43140000), + (0x12280, 0x43d60000), + ]; + for &(offset, value) in INITIAL { put_u32(policy, offset, value); } + Ok(()) +} + +/// Initial controller inputs for J716c with its matched calibration: the J714 +/// defaults, including the relative power curves, with this chip's leakage +/// values (clusters 0-3, base, AFR) and its maximum power (196098 mW), and the +/// found masks of its AON PTD records (RT0g and RT1g, then RT0r) next to their +/// tags. +pub(crate) fn encode_m5_policy_controller_defaults_j716c(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_policy_controller_defaults(out)?; + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const MAXIMUM_POWER: u32 = 196_098; + const MAXIMUM_POWER_F32: u32 = 0x483f_8080; + for offset in [M5_POLICY_MAXIMUM_POWER, 0xad0, 0x44a4, 0x44a8, 0x44ac, 0x45e4] { + put_u32(policy, offset, MAXIMUM_POWER); + } + for offset in [0x9f4, 0x45e0] { + put_u32(policy, offset, MAXIMUM_POWER_F32); + } + for (index, value) in [0x4524_9000, 0x451f_9000, 0x451f_9000, 0x4524_9000].into_iter().enumerate() { + put_u32(policy, 0x121b0 + index * 4, value); + } + for index in 0..4 { + put_u32(policy, 0x12230 + index * 4, 0x434d_8000); + } + put_u32(policy, 0x12280, 0x44b1_8000); + put_u32(policy, 0x11a64, 3); + put_u32(policy, 0x11a74, 1); + Ok(()) +} + +/// Controller inputs for J775D: the J714 defaults, with the Ultra's power model over them. The +/// model covers the main and auxiliary operating points of both dies (64 cores, 110 C) with +/// synthetic leakage inputs (1350 per cluster core and per die auxiliary domain, 51 per cluster +/// SRAM), which hold for every J775D: P13 sums to 345262 mW. Then the relative power curves of +/// that model, the controllers' gains and utilization thresholds, and the temperature-sensor +/// channel masks of both dies (160 GPU and 8 auxiliary-domain channels; the hotspot set is both). +pub(crate) fn encode_m5_policy_controller_defaults_j775d(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_policy_controller_defaults(out)?; + const RELATIVE_POWER_MAIN: [u32; 14] = [ + 0x0000_0000, 0x4107_2435, 0x414b_7ecd, 0x4187_dfa6, 0x41be_759c, 0x41d6_f4a2, 0x41ff_340b, + 0x4215_4139, 0x4226_cc22, 0x4247_2db8, 0x426b_3fa4, 0x428b_aed3, 0x42b6_c65b, 0x42c8_0000, + ]; + for (index, &value) in RELATIVE_POWER_MAIN.iter().enumerate() { put_u32(out, 0x19e4 + index * 4, value); } + const RELATIVE_POWER_AUX: [u32; 14] = [ + 0x0000_0000, 0x4106_2d7e, 0x4154_96b9, 0x4189_3d11, 0x41ba_e22e, 0x41d5_33ed, 0x4202_7bcd, + 0x421c_12dd, 0x4230_6915, 0x4250_6a36, 0x427d_e40c, 0x4294_0cc0, 0x42bd_bd8e, 0x42c8_0000, + ]; + for (index, &value) in RELATIVE_POWER_AUX.iter().enumerate() { put_u32(out, 0x1a64 + index * 4, value); } + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const MAXIMUM_POWER: u32 = 345_262; + const MAXIMUM_POWER_F32: u32 = 0x48a8_95c0; + for offset in [M5_POLICY_MAXIMUM_POWER, 0xad0, 0x44a4, 0x44a8, 0x44ac, 0x45e4] { + put_u32(policy, offset, MAXIMUM_POWER); + } + for offset in [0x9f4, 0x45e0] { + put_u32(policy, offset, MAXIMUM_POWER_F32); + } + // Leakage: eight clusters' cores (1350.0) and SRAM (51.0), then each die's auxiliary domain. + for index in 0..8 { + put_u32(policy, 0x121b0 + index * 4, 0x44a8_c000); + put_u32(policy, 0x12230 + index * 4, 0x424c_0000); + } + for index in 0..2 { + put_u32(policy, 0x12280 + index * 4, 0x44a8_c000); + } + // The power controller's integral and proportional gains (0.08, 0.005), the second + // controller's (0.01, 0.28), and the utilization thresholds 75 and 95. + for (offset, value) in [(0xaa0, 0x3da3_d70b), (0xab0, 0x3ba3_d70a), (0x45b4, 0x3c23_d70b), + (0x45c4, 0x3e8f_5c29), (0xb44, 75), (0xb48, 95)] { + put_u32(policy, offset, value); + } + // The performance controllers' floor, P3, as state x 100 in the three floor words (the last + // byte-packed in the boost block); HwData's relative-frequency tables are based at it too. + for offset in [0xb94, 0xbd4, 0xbf9] { + put_u32(policy, offset, J775D_PERF_FLOOR as u32 * 100); + } + // The fast-active boost may continue while the GPU idles for 20 percent of a boost window + // (byte-packed in the boost block) on the Ultra's core type; the default is 10. + policy[0xbfd] = 20; + // The AON temperature records the GPU's and the auxiliary domain's controllers read, with + // each list's found mask: both dies' g0TR-g3TR and r0TR-r1TR, the six records of J775D's + // AON table (encode_common26_aon_ptd). + for (index, tag) in [0x6730_5452, 0x6731_5452, 0x6732_5452, 0x6733_5452].into_iter().enumerate() { + put_u32(policy, 0x11a24 + index * 4, tag); + } + put_u32(policy, 0x11a64, 0xf); + for (index, tag) in [0x7230_5452, 0x7231_5452].into_iter().enumerate() { + put_u32(policy, 0x11a6c + index * 4, tag); + } + put_u32(policy, 0x11a74, 0x3); + const DIE_CHANNELS: u64 = 0x6fcc_3fcc_6fcc_3fcc; + for (offset, words) in [ + (0x2324, [0x6fcc_3fcc_0000_0000, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), + (0x2cfc, [0x0303_0303, 0, 0, 0, 0]), + (0x4ba4, [0x6fcc_3fcc_0303_0303, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), + ] { + for (index, word) in words.into_iter().enumerate() { + put_u64(policy, offset + index * 8, word); + } + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs new file mode 100644 index 0000000000000..5c3eef8c54e22 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware-specific work encodings, using the shared G17 queue primitives. + +pub(crate) mod initdata; +pub(crate) mod power; + +/// M5 adds a statistics record at tag 32, beyond the legacy 32-bit mask. +pub(crate) const STATISTICS_TYPES: u64 = 0x1_5fff_0000; + +/// M5 stores an entry count in the trailer, with its upper half clear. +/// The preceding register storage and self pointer use the shared layout. +pub(crate) fn set_register_array( + array: &mut super::kick::RegisterArray, + address: u64, + writes: &[super::kick::RegisterWrite], +) -> kernel::prelude::Result { + array.set(address, writes)?; + array.size = writes.len() as u32; + Ok(()) +} + +pub(crate) mod pool; +pub(crate) mod pool_budget; +pub(crate) mod queue; +pub(crate) mod compute; + +pub(crate) mod render_demand; + +pub(crate) mod render; +pub(crate) mod render_tail; + +pub(crate) mod render_registers; + +pub(crate) mod render_assembly; + +pub(crate) mod target; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs new file mode 100644 index 0000000000000..b8c52bc634c5f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 private-memory allocation encodings. +#![allow(dead_code)] + +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +use super::initdata::{Board, J714}; + +pub(crate) const BLOCK_BYTES: usize = 0x20_0000; +// Render block size and private-memory address inventory of the M5 +// part, selected at probe (`Board::render_pool`). The inventory sizes the +// directory, not the physical allocation; each pool allocates only its +// requested initial blocks. +static RENDER_BLOCK_BYTES: AtomicUsize = AtomicUsize::new(J714.render_pool.0); +static POOL_BYTES: AtomicU64 = AtomicU64::new(J714.render_pool.1); + +/// Selects the board's render pool before any pool is built. +pub(crate) fn select_render_pool(board: &Board) { + let (block, pool) = board.render_pool; + RENDER_BLOCK_BYTES.store(block, Ordering::Relaxed); + POOL_BYTES.store(pool, Ordering::Relaxed); +} + +pub(crate) fn render_block_bytes() -> usize { + RENDER_BLOCK_BYTES.load(Ordering::Relaxed) +} + +const RING_MAX_PAGES: u64 = 0x3ff800; +pub(crate) const OPERAND_BYTES: usize = 0x2_0000; +pub(crate) const STATE_BYTES: usize = 0x100; +/// Unaligned firmware-completed submission counter. +pub(crate) const COMPLETED: usize = 0x74; +/// Mutable words saved from the descriptor after the pool's final release. +pub(crate) const SAVED_FIELDS: [usize; 8] = [0x28, 0x20, 0x24, 0x2c, 0x3c, 0x48, 0x38, 0x4c]; +const GPU_PAGE: usize = 0x4000; +const PRIVATE_TOP: u64 = 1 << 42; +pub(crate) const SCENE_TABLE_ENTRIES: usize = 0x50; +pub(crate) const SCENE_STRIDE: usize = 0x80; +pub(crate) const SCENE_TRAILER_SIZE: usize = 0x40; + +pub(crate) fn scene_table_end(address: u64) -> Option { + address.checked_add((SCENE_TABLE_ENTRIES * SCENE_STRIDE) as u64) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PoolError { Geometry, Address, Buffer } + +/// Snapshot of the retained pool header. Address and allocation geometry are +/// host-owned; only SAVED_FIELDS may be restored from a retired descriptor. +pub(crate) struct PoolState(pub(crate) [u8; STATE_BYTES]); + +impl PoolState { + fn u32(&self, offset: usize) -> u64 { + u64::from(u32::from_le_bytes(self.0[offset..offset + 4].try_into().unwrap())) + } + fn u64(&self, offset: usize) -> u64 { + u64::from_le_bytes(self.0[offset..offset + 8].try_into().unwrap()) + } + pub(crate) fn descriptor(&self) -> [u64; 8] { + [self.u64(0x14) >> 7 | self.u32(0x28) << 61, + self.u64(0x20), self.u32(0x2c) | self.u32(0x1c) << 32, 0, + self.u64(0x30) >> 6 | self.u32(0x40) << 44, + self.u32(0x3c) | self.u32(0x44) << 31, + self.u32(0x48), self.u32(0x38) | self.u32(0x4c) << 32] + } + pub(crate) fn restore(&mut self, words: [u64; 8]) { + let values = [words[0] >> 61 & 1, words[1] & 0x0fff_ffff, + words[1] >> 32 & 0x0fff_ffff, words[2] & 0x0fff_ffff, + words[5] & 0x0fff_ffff, words[6] & 0xffff, + words[7] & 0x0fff_ffff, words[7] >> 32 & 0xffff]; + for (offset, value) in SAVED_FIELDS.into_iter().zip(values) { + self.0[offset..offset + 4].copy_from_slice(&(value as u32).to_le_bytes()); + } + } + /// Reject a descriptor belonging to different backing before restoring + /// its mutable counters. Reserved bits are not host address bits. + pub(crate) fn matches(&self, words: &[u64; 8]) -> bool { + let own = self.descriptor(); + words[0] & ((1 << 35) - 1) == own[0] & ((1 << 35) - 1) + && words[2] >> 32 == own[2] >> 32 + && words[4] == own[4] + } +} + +/// Parameter-buffer capacity follows the memory budget and initial demand. +/// Capacity is rounded to groups of 33 half-MiB blocks; it reserves list space, +/// not physical backing for every possible block. +pub(crate) fn parameter_capacity(memory_bytes: u64, initial_bytes: u64) -> Result { + const GROUP: u64 = 33 * 0x80000; + let scaled = memory_bytes.checked_mul(5).ok_or(PoolError::Geometry)? >> 2; + let budget = (((u128::from(scaled) * 0x28f5_c28f_5c28_f5c3) >> 64) as u64) >> 2; + let groups = |bytes: u64| bytes.checked_add(GROUP - 1) + .map(|n| n / GROUP).ok_or(PoolError::Geometry); + let groups = groups(budget)?.max(groups(initial_bytes)?); + usize::try_from(groups.checked_mul(33).ok_or(PoolError::Geometry)?) + .map_err(|_| PoolError::Geometry) +} + +/// Per-scene scratch covers the complete parameter-buffer inventory. Optional +/// cluster entries are supplied by the dispatch policy, not the active draw's +/// physical block count. Each entry and the header occupy eight bytes. +pub(crate) fn parameter_scratch_stride(capacity: usize, cluster_entries: usize) + -> Result +{ + if capacity == 0 { return Err(PoolError::Geometry); } + let pages = capacity.checked_mul(16).ok_or(PoolError::Geometry)?; + let entries = pages.checked_add(0x17ff).ok_or(PoolError::Geometry)? / 0x1800; + let entries = entries.checked_add(cluster_entries).ok_or(PoolError::Geometry)?; + entries.checked_mul(8).and_then(|n| n.checked_add(8 + 63)) + .map(|n| n & !63).ok_or(PoolError::Geometry) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PoolGeometry { + pub(crate) blocks: usize, + pub(crate) directory_entries: u32, + pub(crate) pages: u32, + pub(crate) directory_bytes: usize, + pub(crate) freelist_bytes: usize, + g17s_render_blocks: bool, +} + +fn round_page(bytes: usize) -> Result { + bytes.checked_add(GPU_PAGE - 1).map(|n| n & !(GPU_PAGE - 1)).ok_or(PoolError::Geometry) +} + +impl PoolGeometry { + pub(crate) fn g17s(blocks: usize) -> Result { + let bytes = POOL_BYTES.load(Ordering::Relaxed).min(RING_MAX_PAGES << 12) + & !(render_block_bytes() as u64 - 1); + Self::g17s_render(blocks, (bytes >> 12) as u32) + } + pub(crate) fn new(blocks: usize, directory_entries: u32) -> Result { + Self::with_kind(blocks, directory_entries, false) + } + + pub(crate) fn g17s_render(blocks: usize, directory_entries: u32) -> Result { + Self::with_kind(blocks, directory_entries, true) + } + + pub(crate) fn block_bytes(self) -> usize { + if self.g17s_render_blocks { render_block_bytes() } else { BLOCK_BYTES } + } + + fn with_kind(blocks: usize, directory_entries: u32, g17s_render_blocks: bool) -> Result { + let block_bytes = if g17s_render_blocks { render_block_bytes() } else { BLOCK_BYTES }; + let pages = blocks.checked_mul(block_bytes / 0x1000) + .and_then(|n| u32::try_from(n).ok()).ok_or(PoolError::Geometry)?; + if blocks == 0 || pages > directory_entries { return Err(PoolError::Geometry); } + let directory_bytes = round_page((directory_entries as usize).checked_mul(8) + .ok_or(PoolError::Geometry)?)?; + let stride = if g17s_render_blocks { 8 } else { 0x40 }; + let freelist_bytes = round_page(blocks.checked_mul(stride).ok_or(PoolError::Geometry)?)?; + + if freelist_bytes >> 12 >= 1 << 20 { return Err(PoolError::Geometry); } + Ok(Self { blocks, directory_entries, pages, directory_bytes, freelist_bytes, g17s_render_blocks }) + } +} + +fn private_range(address: u64, bytes: usize) -> Result<(), PoolError> { + if address == 0 || address & (GPU_PAGE as u64 - 1) != 0 + || address.checked_add(bytes as u64).is_none_or(|end| end > PRIVATE_TOP) { + Err(PoolError::Address) + } else { Ok(()) } +} + +fn validate_blocks(g: PoolGeometry, blocks: &[u64]) -> Result<(), PoolError> { + if PoolGeometry::with_kind(g.blocks, g.directory_entries, g.g17s_render_blocks)? != g { return Err(PoolError::Geometry); } + if blocks.len() != g.blocks { return Err(PoolError::Geometry); } + for address in blocks { private_range(*address, g.block_bytes())?; } + Ok(()) +} + +pub(crate) fn encode_directory(g: PoolGeometry, blocks: &[u64], out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_blocks(g, blocks)?; + if out.len() < g.directory_bytes { return Err(PoolError::Buffer); } + out[..g.directory_bytes].fill(0); + + let pages_per_block = g.block_bytes() / 0x1000; + for (block, address) in blocks.iter().enumerate() { + for page in 0..pages_per_block { + let offset = (block * pages_per_block + page) * 8; + out[offset..offset + 8].copy_from_slice(&(address + page as u64 * 0x1000).to_le_bytes()); + } + } + Ok(()) +} + +pub(crate) fn encode_freelist(g: PoolGeometry, blocks: &[u64], out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_blocks(g, blocks)?; + if out.len() < g.freelist_bytes { return Err(PoolError::Buffer); } + out[..g.freelist_bytes].fill(0); + for (index, address) in blocks.iter().enumerate() { + let (offset, value) = if g.g17s_render_blocks { (index * 8, *address) } + else { (index * 0x40, address | (1 << 61)) }; + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PoolLayout { + pub(crate) geometry: PoolGeometry, + pub(crate) directory: u64, + pub(crate) freelist: u64, + + pub(crate) hardware_id: u32, +} + +fn validate_layout(p: PoolLayout) -> Result<(), PoolError> { + if PoolGeometry::with_kind(p.geometry.blocks, p.geometry.directory_entries, p.geometry.g17s_render_blocks)? != p.geometry { + return Err(PoolError::Geometry); + } + private_range(p.directory, p.geometry.directory_bytes)?; + private_range(p.freelist, p.geometry.freelist_bytes) +} + +pub(crate) fn initial_uma_descriptor(p: PoolLayout) -> Result<[u64; 8], PoolError> { + validate_layout(p)?; + if p.geometry.g17s_render_blocks { + let blocks = p.geometry.blocks as u64; + return Ok([p.directory >> 7, u64::from(p.geometry.pages) << 32, + u64::from(p.geometry.pages) | (u64::from(p.geometry.directory_entries) << 32), + 0, (p.freelist >> 6) | ((p.geometry.block_bytes() as u64 >> 12) << 44), + blocks, 0, blocks]); + } + Ok([p.directory >> 7, u64::from(p.geometry.pages) << 32, + u64::from(p.geometry.pages) | (u64::from(p.geometry.directory_entries) << 32), + 0, (p.freelist >> 6) | ((p.geometry.freelist_bytes as u64 >> 12) << 44), + 0, 0, 0]) +} + +pub(crate) fn encode_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_layout(p)?; + if submitted == 0 || submitted & 3 != 0 { return Err(PoolError::Address); } + if out.len() < STATE_BYTES { return Err(PoolError::Buffer); } + out[..STATE_BYTES].fill(0); + for (offset, value) in [(0, 1u32), (8, p.hardware_id), (0x10, 2), + (0x1c, p.geometry.directory_entries), (0x24, p.geometry.pages), + (0x2c, p.geometry.pages), (0x40, (p.geometry.freelist_bytes >> 12) as u32), + (0x60, 2), (0x80, 1)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(0x14, p.directory), (0x30, p.freelist), + (0x50, p.freelist), (0x6c, submitted)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + if p.geometry.g17s_render_blocks { + + for (offset, value) in [(0x0c, 2), (0x38, p.geometry.blocks as u32), + (0x3c, p.geometry.blocks as u32), + (0x40, (p.geometry.block_bytes() >> 12) as u32), (0x44, 0), + (0x60, 0), + (0x68, p.geometry.blocks as u32), (0x80, 4)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } + Ok(()) +} + +pub(crate) fn encode_g17s_render_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + encode_state(p, submitted, out)?; + out[0x80..0x84].copy_from_slice(&4u32.to_le_bytes()); + Ok(()) +} + +pub(crate) fn encode_g17s_compute_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + if !p.geometry.g17s_render_blocks { return Err(PoolError::Geometry); } + encode_state(p, submitted, out)?; + out[0x7c..0x80].copy_from_slice(&1u32.to_le_bytes()); + out[0x80..0x84].copy_from_slice(&1u32.to_le_bytes()); + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs new file mode 100644 index 0000000000000..ce0b5dc6129a4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G17S shared render private-memory demand. Counts describe hardware and +//! simultaneous work; byte requirements come from the submitted programs. + +use super::pool::{render_block_bytes, PoolError, PoolGeometry}; + +#[derive(Clone, Copy)] +pub(crate) struct RenderBudget { + active_cores: u32, + constant_pages: u32, + active_kicks: u32, +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct Requirements { + pub(crate) minimum_bytes: u64, + pub(crate) ideal_bytes: u64, + pub(crate) self_reserve_pages: u32, + pub(crate) render_target_pages: u32, + pub(crate) has_spill: bool, +} + +/// Retained high-water marks, excluding per-kick reservations from the two +/// base byte counts. They may increase demand but never replace live demand. +#[derive(Clone, Copy, Default)] +pub(crate) struct History { + pub(crate) minimum_base: u64, + pub(crate) ideal_base: u64, + pub(crate) self_reserve: u64, + pub(crate) render_target: u64, + pub(crate) minimum_pool: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Sizes { + pub(crate) minimum: u64, + pub(crate) ideal: u64, +} + +fn add(a: u64, b: u64) -> Result { + a.checked_add(b).ok_or(PoolError::Geometry) +} +fn mul(a: u64, b: u64) -> Result { + a.checked_mul(b).ok_or(PoolError::Geometry) +} +fn aligned(bytes: u64) -> Result { + Ok(add(bytes, render_block_bytes() as u64 - 1)? + & !(render_block_bytes() as u64 - 1)) +} + +impl RenderBudget { + /// `core_masks` contains the enabled cores of each physical cluster. + /// `ptc_floor` is the configured per-core context count; `active_kicks` + /// includes simultaneously resident tiling and fragment work. + pub(crate) fn new(core_masks: &[u32], ptc_floor: u32, active_kicks: u32) + -> Result + { + let mut active_cores = 0u32; + let mut cluster_cores = 0u32; + for mask in core_masks { + let count = mask.count_ones(); + active_cores = active_cores.checked_add(count).ok_or(PoolError::Geometry)?; + cluster_cores = cluster_cores.max(count); + } + if active_cores == 0 || active_kicks == 0 || ptc_floor == 0 { + return Err(PoolError::Geometry); + } + let constant_pages = cluster_cores.checked_mul(ptc_floor) + .and_then(|n| n.checked_add(32)) + .and_then(|n| n.checked_mul(u32::try_from(core_masks.len()).ok()?)) + .ok_or(PoolError::Geometry)?; + Ok(Self { active_cores, constant_pages, active_kicks }) + } + + /// Compute SCS storage is private to each enabled USC. The auxiliary + /// allocation the dispatch publishes, per USC and at least the eight-page + /// tail, belongs to the compute allocator, in addition to its small/large + /// SCS page inventories and the execution-state spill reservation. As for + /// render, the pool covers what the descriptor tells the hardware. + pub(crate) fn compute_scs_requirements(&self, mut request: Requirements, + pages: u16, auxiliary_pages: u32) -> Result + { + if pages != 0 { + let small = u64::from(pages); + let per_core = add(add(small, small.div_ceil(256))?, u64::from(auxiliary_pages.max(8)))?; + let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; + request.minimum_bytes = add(request.minimum_bytes, bytes)?; + request.ideal_bytes = add(request.ideal_bytes, bytes)?; + request.has_spill = true; + } + Ok(request) + } + + /// Render's allocator publishes the already rounded auxiliary page count + /// in the descriptor, including its eight-page tail. This is separate + /// from the layout-derived small/large SCS inventories. The USC + /// spill-buffer allocation multiplies their sum by cores. + pub(crate) fn render_scs_requirements(&self, mut request: Requirements, + pages: u16, auxiliary_pages: u32) -> Result + { + if pages != 0 { + let small = u64::from(pages); + let per_core = add(add(small, small.div_ceil(256))?, + u64::from(auxiliary_pages))?; + let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; + request.minimum_bytes = add(request.minimum_bytes, bytes)?; + request.ideal_bytes = add(request.ideal_bytes, bytes)?; + request.has_spill = true; + } + Ok(request) + } + + pub(crate) fn sizes(&self, request: Requirements, history: History) + -> Result + { + let reserve = u64::from(request.self_reserve_pages) << 12; + let rt = u64::from(request.render_target_pages) << 12; + let reserved = add(reserve, rt)?; + if request.minimum_bytes < reserved || request.ideal_bytes < reserved { + return Err(PoolError::Geometry); + } + // Every enabled USC has execution-state storage. Programs with spill + // state add twelve pages; shared queues reserve another twenty-four. + let execution = mul(u64::from(self.active_cores), + if request.has_spill { 76 << 12 } else { 64 << 12 })?; + let constant = mul(u64::from(self.constant_pages), + u64::from(self.active_kicks - 1) << 12)?; + let per_kick = mul(add(reserve.max(history.self_reserve), + rt.max(history.render_target))?, u64::from(self.active_kicks))?; + let size = |base: u64, high_water: u64| { + let base = add(base, execution)?.checked_sub(reserved) + .ok_or(PoolError::Geometry)?.max(high_water); + add(add(base, constant)?, per_kick) + }; + let ideal = aligned(size(request.ideal_bytes, history.ideal_base)?)?; + let minimum = aligned(size(request.minimum_bytes, history.minimum_base)? + .min(ideal.max(history.minimum_pool)))?; + if minimum > ideal { return Err(PoolError::Geometry); } + Ok(Sizes { minimum, ideal }) + } +} + +impl Sizes { + /// Reserve the full address inventory but allocate only the desired + /// physical blocks. The hardware directory limit is checked by geometry. + pub(crate) fn geometry(self) -> Result { + if self.minimum == 0 || self.minimum > self.ideal { return Err(PoolError::Geometry); } + let blocks = usize::try_from(self.ideal / render_block_bytes() as u64) + .map_err(|_| PoolError::Geometry)?; + if self.ideal % render_block_bytes() as u64 != 0 { return Err(PoolError::Geometry); } + PoolGeometry::g17s(blocks) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/power.rs b/drivers/gpu/drm/asahi/g17/fw/m5/power.rs new file mode 100644 index 0000000000000..9d0ab94f392e8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/power.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Live M5 Region C inputs. These offsets do not follow the Neo layout. + +pub(crate) const EFFORT: usize = 0xe8; +pub(crate) const EFFORT_OVERRIDE: usize = 0xec; +pub(crate) const POWER_ASSERT_TALLY: usize = 0x1c99; + +/// Control records which assert power among the opcodes this driver emits. +pub(crate) const fn asserts_gpu_power(opcode: u32) -> bool { + matches!(opcode, 0x08 | 0x0a | 0x29) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs b/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs new file mode 100644 index 0000000000000..0a0b64b1de05d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 meanings of fields in the shared channel configuration record. +use kernel::prelude::*; +use super::super::queue::{Policy,QueueConfig,QueueConfigArgs,QueueConfigTarget}; + +/// The firmware's channel priority profiles, including the independent QoS +/// weight selected at queue registration. A single owner has a full share. +pub(crate) fn scheduling_policy(priority: u32) -> Result { + match priority { + 0 => Ok(Policy::new(0, 1, 15, 8192)), + 1 => Ok(Policy::new(1, 0, 20, 8192)), + 2 => Ok(Policy::new(2, 2, 4, 8192)), + 3 => Ok(Policy::new(3, 0, 20, 8192)), + _ => Err(EINVAL), + } +} + +/// M5's channel-state record. The scheduler pointer sits earlier than in +/// the Neo record; the policy and owner fields also moved. +#[repr(C, packed)] +#[derive(Clone, Copy)] +pub(crate) struct QueueRecord { + pointers_va: u64, + items_va: u64, + job_list_va: u64, + reserved_18: [u8; 0xc], + sentinel_24: u32, + priority: [u32; 2], + reserved_30: [u8; 6], + sentinel_36: u16, + reserved_38: u32, + policy: u32, + sentinel_40: u32, + owner_pid: u32, + reserved_48: [u8; 0x28], + scheduler_va: u64, + reserved_78: [u8; 8], +} +impl QueueRecord { + fn scheduling_words(policy: Policy) -> Result<([u8; 6], u16, u32)> { + match (policy.priority(), policy.policy()) { + (0, 1) => Ok(([0, 0, 0xff, 0xff, 0xff, 0xff], u16::MAX, 1)), + (1, 0) => Ok(([0, 0, 0, 0, 0xff, 0xff], u16::MAX, 0)), + (2, 2) => Ok(([0; 6], u16::MAX, 0)), + (3, 0) => Ok(([0; 6], 0, 0)), + _ => Err(EINVAL), + } + } + /// The scheduling words follow the queue's priority class. + pub(crate) fn new(args: &super::super::queue::QueueRecordArgs) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(args.policy)?; + Ok(Self { + pointers_va: args.pointers_va, items_va: args.items_va, + job_list_va: args.job_list_va, reserved_18: [0; 0xc], + sentinel_24: u32::MAX, priority: [u32::from(args.policy.priority()); 2], reserved_30, + sentinel_36: sentinel, reserved_38, policy: u32::from(args.policy.policy()), + sentinel_40: u32::MAX, owner_pid: args.owner_pid, + reserved_48: [0; 0x28], scheduler_va: args.scheduler_va, + reserved_78: [0; 8], + }) + } + /// Every host-owned identity word already holds what set_compute_owner + /// would store (so that update would not change the record). + pub(crate) fn names_owner(&self, work_head: u64, scheduler: u64, + owner_pid: u32, policy: super::super::queue::Policy) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(policy)?; + let (job_list, scheduler_va, pid, priority) = + (self.job_list_va, self.scheduler_va, self.owner_pid, self.priority); + let (r30, s36, r38, current) = (self.reserved_30, self.sentinel_36, self.reserved_38, self.policy); + Ok(job_list == work_head && scheduler_va == scheduler && pid == owner_pid + && priority == [u32::from(policy.priority()); 2] && r30 == reserved_30 + && s36 == sentinel && r38 == reserved_38 && current == u32::from(policy.policy())) + } + /// Replace only host-owned identity fields after the old owner retires. + pub(crate) fn set_compute_owner(&mut self, work_head: u64, scheduler: u64, + owner_pid: u32, policy: super::super::queue::Policy) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(policy)?; + // Changing a channel's priority invalidates this word when its hardware + // class changes. Retained physical queues can change class on reuse. + if self.priority[0] != u32::from(policy.priority()) { + self.sentinel_40 = u32::MAX; + } + self.job_list_va = work_head; + self.scheduler_va = scheduler; + self.owner_pid = owner_pid; + self.priority = [u32::from(policy.priority()); 2]; + self.reserved_30 = reserved_30; + self.sentinel_36 = sentinel; + self.reserved_38 = reserved_38; + self.policy = u32::from(policy.policy()); + Ok(()) + } +} +const _: () = { + assert!(core::mem::size_of::() == 0x80); + assert!(core::mem::offset_of!(QueueRecord, policy) == 0x3c); + assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x70); +}; + +pub(crate) struct ConfigProfile { + pub(crate) soft_fault_enabled:bool, + pub(crate) extended:bool, + pub(crate) parameter_manager:Option, +} +pub(crate) fn config(args:&QueueConfigArgs,p:ConfigProfile)->Result { + if p.parameter_manager.is_some() && !matches!(args.target,QueueConfigTarget::Tiling(Some(_))) { + return Err(EINVAL); + } + let mut out=QueueConfig::new(args)?; + out.unk_66=u16::from(p.soft_fault_enabled); + out.unk_68[2..6].copy_from_slice(&u32::from(p.extended).to_le_bytes()); + if let Some(manager)=p.parameter_manager {out.fragment_qid=manager;} + Ok(out) +} + +/// M5 publishes the current kick with the common tag-16 wire shape. All +/// reserved bytes are zero; the descriptor is reached through the shared ring's entry. +#[repr(C, packed)] +#[derive(Clone, Copy)] +pub(crate) struct Notification { + record: super::super::queue::KickPredecessor, + reserved: [u8; 6], +} +pub(crate) fn notification(qid:u8, current:super::super::kick::KickTimestamp)->Result { + Ok(Notification{record:super::super::queue::KickPredecessor::new(qid,current)?,reserved:[0;6]}) +} +const _: () = assert!(core::mem::size_of::() == 24); + +/// Host-port transaction for an M5 render pair. The FIFO is a register pair +/// with a 16-byte stride, selecting slot 9. Queue validity +/// is asserted high then low, after its configuration payload. An existing +/// configuration is invalidated in the same order before replacement. +#[derive(Clone, Copy)] +pub(crate) struct HardwareSubmission { + writes: [(usize,u64);16], + count: usize, +} +impl HardwareSubmission { + pub(crate) fn advance_pair(fragment:u8,tiling:u8)->Result { + if fragment==tiling {return Err(EINVAL);} + let mut writes=[(0,0);16]; + for (n,qid) in [fragment,tiling].into_iter().enumerate() { + writes[n*2..n*2+2].copy_from_slice(&[ + (0x4090,Self::configuration_tag(qid)?+0x10),(0x4098,1)]); + } + Ok(Self{writes,count:4}) + } + /// Configure the compute queue at first use, then publish only new kicks. + pub(crate) fn compute(qid:u8,va:u64,priority:u8,configure:bool)->Result { + use super::super::queue::{Policy,DataMaster}; + const ADDRESS_MASK:u64=0x0000_07ff_ffff_ffe0; + let tag=Self::configuration_tag(qid)?; + if priority>=Policy::PRIORITIES || va & !ADDRESS_MASK != 0 {return Err(EINVAL);} + let mut writes=[(0,0);16]; + let mut count=0; + if configure { + let (low,high)=if qid<64 {(1u64<Result { + use super::super::{kick::QID_MAX,queue::{Policy,DataMaster}}; + const ADDRESS_MASK:u64=0x0000_07ff_ffff_ffe0; + if fragment.0==tiling.0 || priority>=Policy::PRIORITIES {return Err(EINVAL);} + let mut writes=[(0,0);16]; + let mut count=0; + for ((qid,va),engine) in [(fragment,DataMaster::Fragment),(tiling,DataMaster::Tiling)] { + if qid>QID_MAX || va & !ADDRESS_MASK != 0 {return Err(EINVAL);} + let completion=(3*(4-u64::from(priority))+0x3fe).wrapping_shl(54); + let tag=0x0020_0800_0002_1008 | (u64::from(qid)<<44); + let payload=((engine as u64+1)<<60)|va|completion; + let (low,high)=if qid<64 {(1u64<Result { + if qid>super::super::kick::QID_MAX {return Err(EINVAL);} + Ok(0x0020_0800_0002_1008 | (u64::from(qid)<<44)) + } + pub(crate) fn writes(&self)->&[(usize,u64)] {&self.writes[..self.count]} +} + +/// M5's scheduler release uses the shared identity and pointer layout. +pub(crate) const SCHEDULER_RELEASE_OPCODE: u32 = 0x13; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs new file mode 100644 index 0000000000000..f53956102b610 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 tiling and fragment command encoders, with shared kick bindings. + +#![allow(dead_code)] + +use super::super::kick::{KickTimestamp, RegisterWrite, RegisterArrayBinding}; + +use super::render_tail::{self as tail, RenderStage, RenderTail}; + +pub(crate) const M5_TA_COMMAND_BYTES: usize = 0x990; +pub(crate) const M5_3D_COMMAND_BYTES: usize = 0x2240; +pub(crate) const M5_TA_SELECTOR: u32 = 0; +pub(crate) const M5_3D_SELECTOR: u32 = 1; + +pub(crate) const M5_TA_PROGRAM_OFFSET: u32 = 0x60; + +pub(crate) const M5_3D_PROGRAM_OFFSETS: [u32; 4] = [0xa0, 0x7c0, 0xee0, 0x1600]; + +const PROGRAM_AREA: usize = 0x700; +const REGISTER_BYTES: usize = 12; +pub(crate) const M5_PROGRAM_CAPACITY: usize = PROGRAM_AREA / REGISTER_BYTES; + +const TA_DISPATCH_REGISTER: u32 = 0x1c898; + +const FRAGMENT_TILE_REGISTER: u32 = 0x10038; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum M5RenderCommandError { + ProgramTooLong, + ProgramOffset, + RceCount, + MissingDispatchMirror, + AddressOverflow, + InvalidCompletionState, + InvalidUsageHint, + Kick, + SharedRing, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5ProgramRef<'a> { + pub registers: &'a [RegisterWrite], + + pub rce_count: u16, + pub command_offset: u32, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5FragmentProgramRefs<'a> { + + pub main: M5ProgramRef<'a>, + + pub main_mid_count: u16, + pub store: M5ProgramRef<'a>, + pub resume: M5ProgramRef<'a>, + pub load: M5ProgramRef<'a>, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct M5StageCompletion { + + pub stamp_addresses: [u64; 2], + + pub stamp: u32, + + pub queue_index: u32, + + pub ordinal: u32, + + pub has_wait: bool, + + pub completion: u64, + + pub key: u64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct M5HardwareStamp { + pub stamp: u32, + + pub entries: u32, + + pub generation: u64, + + pub base: u64, +} + +pub(crate) fn m5_hardware_stamp(p: &M5HardwareStamp) -> Result { + if p.entries == 0 { + return Err(M5RenderCommandError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +pub(crate) fn m5_render_completion( + p: &M5HardwareStamp, + pass: u8, +) -> Result { + Ok(m5_hardware_stamp(p)? | u64::from(pass) << 40) +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaLinked3d { + + pub queue_index: u32, + + pub stamp: u32, + + pub flag: u8, + + pub completion: u64, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaPolicy { + pub policy_a: u32, + pub policy_b: u8, + pub policy_c: u32, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5ExtraAddresses { + + pub count: u32, + pub addresses: [u64; 4], + + pub policy: u8, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaCommandFields { + + pub sequence: u64, + pub completion: M5StageCompletion, + + pub pb_slot: u32, + + pub ta_word_a: u32, + + pub pb_state: u64, + + pub pm_scene: u64, + + pub scene_end_va: u64, + pub ta_word_b: u32, + pub ta_word_e: u32, + pub context_token: u64, + pub tail_descriptor: u64, + pub resolved_tail_value: u64, + pub tpc_va: u64, + pub tpc_size: u64, + pub linked_3d: M5TaLinked3d, + pub ta_word_c: u64, + pub ta_word_d: u64, + pub deflake_state_va: u64, + pub stage_flag_a: u8, + pub predecessor_id: u32, + + pub stage_word_a: u32, + pub stage_word_b: u32, + pub timestamp_va: u64, + + pub diagnostic_ta_timestamp_upper: u64, + pub ta_pair_a: [u64; 2], + + pub timestamp_counts: [u32; 2], + + pub ta_word_f: u64, + /// The shifted usage hint: the seven bytes of the tail's descriptor value after + /// its first byte. The byte after them starts the resolved usage pointer. + pub shifted_usage_hint: u64, + + pub uma_state: u64, + + pub tail_flag: u8, + pub pb_growth_request: u8, + pub shader_write_class: u8, + pub policy: M5TaPolicy, + pub extra: M5ExtraAddresses, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M53dCommandFields { + pub sequence: u64, + pub completion: M5StageCompletion, + + pub pass: u8, + pub context_token: u64, + pub tail_descriptor: u64, + pub resolved_tail_value: Option, + pub fragment_word_l: u32, + + pub fragment_flag_a: bool, + + pub short_partial_programs: u8, + + pub tail_flag: u8, + + pub pm_state: u64, + + pub pm_scene: u64, + + pub scene_end_va: u64, + pub tilemap_va: u64, + pub msaa_control: u64, + pub fragment_word_a: u32, + pub sample_count: u32, + pub utile_grid: u32, + pub merge_bounds: [u64; 2], + pub fragment_word_b: u64, + pub tile_count: u32, + pub depth_bias_va: u64, + pub scissor_va: u64, + pub occlusion_query_va: u64, + pub fragment_word_c: u64, + + pub binding_pairs: [[u64; 2]; 23], + pub depth_dimensions: u64, + pub depth_va: u64, + pub depth_va_pair: [u64; 2], + pub stencil_va: u64, + pub stencil_va_pair: [u64; 2], + pub fragment_word_d: u64, + pub fragment_word_e: u64, + pub depth_stride: u64, + pub stencil_stride: u64, + pub depth_compression_va: u64, + pub depth_compression_stride: u64, + pub stencil_compression_va: u64, + pub stencil_compression_stride: u64, + pub sample_aux_flags: [u64; 2], + pub size_aux_control: [u64; 2], + pub eot_program: [u64; 2], + pub partial_eot_program: [u64; 2], + pub fragment_word_f: u64, + pub fragment_word_g: u32, + pub fragment_word_h: u64, + pub fragment_word_i: u32, + pub zs_clear_values: u64, + pub fragment_word_j: u64, + pub fragment_word_k: u32, + pub sample_size: u32, + pub fragment_pass_flag: u8, + pub fragment_word_n: u32, + pub fragment_flag_c: u8, + pub stage_flag_a: u8, + pub predecessor_id: u32, + + pub stage_word_a: u32, + pub stage_word_b: u32, + pub fragment_pair_a: [u64; 2], + + pub timestamp_counts: [u32; 2], + + pub fragment_word_m: u64, + + pub uma_state: u64, + + pub timestamp_va: [u64; 2], + + pub empty_tile_flags: [u8; 4], + pub fragment_flag_b: u8, + pub extra: M5ExtraAddresses, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5RenderCommandInputs { + + pub context_id: u32, + + pub scheduler_record: u64, + pub ta: M5TaCommandFields, + pub fragment: M53dCommandFields, +} + +impl Default for M5RenderCommandInputs { + fn default() -> Self { + + Self { + context_id: 0, + scheduler_record: 0, + ta: M5TaCommandFields { stage_word_b: u32::MAX, ..Default::default() }, + fragment: M53dCommandFields { stage_word_b: u32::MAX, ..Default::default() }, + } + } +} + +fn put8(out: &mut [u8], offset: usize, value: u8) { + out[offset] = value; +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn put128(out: &mut [u8], offset: usize, value: [u64; 2]) { + put64(out, offset, value[0]); + put64(out, offset + 8, value[1]); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn completion_tail(out: &mut [u8], base: usize, c: &M5StageCompletion) { + put64(out, base, c.stamp_addresses[0]); + put64(out, base + 8, c.stamp_addresses[1]); + put32(out, base + 0x10, c.stamp); + put32(out, base + 0x14, c.queue_index); + put32(out, base + 0x18, c.ordinal); + put32(out, base + 0x1c, u32::from(c.has_wait)); +} + +/// Fill publication-time completion values without replacing the retained +/// resource, lifecycle or register-program fields of either command. +pub(crate) fn link_completions( + ta: &mut [u8; M5_TA_COMMAND_BYTES], + fragment: &mut [u8; M5_3D_COMMAND_BYTES], + kicks: [KickTimestamp; 2], + pass: u8, + fragment_busy: bool, +) { + put32(ta, 0x7a4, u32::from(fragment_busy)); + put64(ta, 0x7a8, kicks[1].completion_value(pass) & 0xff_ffff_ffff); + put8(ta, 0x7b0, pass); + put8(ta, 0x7b1, pass); + put64(ta, 0x8c6, kicks[0].completion_value(pass)); + put8(fragment, 0x90, pass); + put64(fragment, 0x2160, kicks[1].completion_value(pass)); +} + +/// Bind the command-state slot in each engine's completion tail. +pub(crate) fn bind_work_slot( + ta: &mut [u8; M5_TA_COMMAND_BYTES], + fragment: &mut [u8; M5_3D_COMMAND_BYTES], + slot: u32, +) -> Result<(), M5RenderCommandError> { + if slot >= 4 { return Err(M5RenderCommandError::InvalidCompletionState); } + put32(ta, 0x8be, slot); + put32(fragment, 0x2158, slot); + Ok(()) +} + +fn extra_tail(out: &mut [u8], stage: RenderStage, e: &M5ExtraAddresses) { + tail::encode_extra(out, stage, &tail::ExtraAddresses { + count: e.count, policy: e.policy, addresses: e.addresses, + }).expect("fixed render command extent"); +} + +fn check_program(p: &M5ProgramRef<'_>, offset: u32) -> Result<(), M5RenderCommandError> { + if p.command_offset != offset { + return Err(M5RenderCommandError::ProgramOffset); + } + if p.registers.len() > M5_PROGRAM_CAPACITY { + return Err(M5RenderCommandError::ProgramTooLong); + } + if usize::from(p.rce_count) > p.registers.len() || p.rce_count > 0x7f { + return Err(M5RenderCommandError::RceCount); + } + Ok(()) +} + +fn place_program(out: &mut [u8], low_alias: u64, p: &M5ProgramRef<'_>) -> Result { + let offset = p.command_offset as usize; + let address = low_alias + .checked_add(u64::from(p.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + put16(out, offset + PROGRAM_AREA + 8, 0); + put64(out, offset + PROGRAM_AREA, address); + for (index, register) in p.registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + let header = (read32(out, entry) & 0xfffc0006) | (register.reg & 0x3fff9); + put32(out, entry, header); + put64(out, entry + 4, register.value); + } + put16(out, offset + PROGRAM_AREA + 8, p.registers.len() as u16); + Ok(address) +} + +/// The TA command's external-write class: whether the render's graphics +/// programs declare writes to memory, as the compiler reports it. +pub(crate) const fn shader_write_class(declared: bool) -> u8 { + // Forwarded as declared; the TA policy word is not inferred from it. + declared as u8 +} + +/// The shifted usage hint of a TA command, by programmed mesh class. The hint +/// covers only the seven bytes before the resolved usage pointer: what looks +/// like a high "pass rotation" bit in a wider read is that pointer's first +/// byte, not part of the hint. Other ordinary vertex programs have different +/// hints. +pub(crate) fn mesh_class_usage_hint(mesh_class: u8) -> u64 { + match mesh_class { + 0 => 0xc0c0, + 1 => 0x10ac0, + _ => 0x12b40, + } +} + +pub(crate) fn encode_m5_ta_command( + out: &mut [u8; M5_TA_COMMAND_BYTES], + low_alias: u64, + i: &M5RenderCommandInputs, + ta: &M5ProgramRef<'_>, +) -> Result<[Option; 4], M5RenderCommandError> { + check_program(ta, M5_TA_PROGRAM_OFFSET)?; + let dispatch = ta + .registers + .iter() + .find(|r| r.reg & !1 == TA_DISPATCH_REGISTER) + .ok_or(M5RenderCommandError::MissingDispatchMirror)? + .value; + low_alias.checked_add(u64::from(ta.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + let t = &i.ta; + let descriptor_value = tail::tiling_descriptor_value(t.tail_descriptor, t.shifted_usage_hint) + .ok_or(M5RenderCommandError::InvalidUsageHint)?; + put32(out, 0, M5_TA_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + tail::encode(out, RenderStage::Tiling, &RenderTail { + descriptor_value, resolved_value: Some(t.resolved_tail_value), + trailing_value: t.context_token, + }).expect("fixed TA command extent"); + put32(out, 0x8d6, t.ta_word_e); + put32(out, 0x48, t.ta_word_b); + put32(out, 0x18, t.pb_slot); + put32(out, 0x1c, t.ta_word_a); + put64(out, 0x20, t.pb_state); + put64(out, 0x28, t.pm_scene); + put64(out, 0x30, t.scene_end_va); + put64(out, 0x10, i.scheduler_record); + put64(out, 0x780, t.tpc_va); + put64(out, 0x788, t.tpc_size); + put32(out, 0x79c, t.linked_3d.queue_index); + put32(out, 0x7a0, t.linked_3d.stamp); + put32(out, 0x7a4, u32::from(t.linked_3d.flag & 1)); + put64(out, 0x7a8, t.linked_3d.completion & 0xff_ffff_ffff); + let pass = (t.linked_3d.completion >> 40) as u8; + put8(out, 0x7b1, pass); + put8(out, 0x7b0, pass); + put64(out, 0x7b6, t.ta_word_c); + put64(out, 0x7be, t.ta_word_d); + put64(out, 0x7d6, t.deflake_state_va); + put128(out, 0x7de, [0, 0]); + put64(out, 0x7ee, 0); + put32(out, 0x7b2, 0); + put32(out, 0x86a, u32::from(t.stage_flag_a & 1)); + put32(out, 0x86e, t.predecessor_id); + put32(out, 0x872, t.stage_word_a); + put32(out, 0x876, t.stage_word_b); + put128(out, 0x87a, [0, 0]); + put128(out, 0x90e, t.ta_pair_a); + put32(out, 0x91e, t.timestamp_counts[0]); + put32(out, 0x922, t.timestamp_counts[1]); + put64(out, 0x926, t.ta_word_f); + put64(out, 0x92e, t.uma_state); + put8(out, 0x936, t.tail_flag & 1); + put64(out, 0x8fe, t.timestamp_va); + put64(out, 0x906, t.diagnostic_ta_timestamp_upper); + put32(out, 0x892, u32::from(t.pb_growth_request & 1)); + put32(out, 0x89a, u32::from(t.shader_write_class & 1)); + put32(out, 0x89e, u32::from(m5_ta_policy_bit(&t.policy, t.shader_write_class))); + put32(out, 0x8a2, 0); + put64(out, 0x8f6, 0); + completion_tail(out, 0x8a6, &t.completion); + put64(out, 0x8c6, t.completion.completion); + put64(out, 0x8ce, t.completion.key); + + put32(out, 0x38, 0); + put16(out, 0x3c, 0); + put32(out, 0x40, 0); + let address = place_program(out, low_alias, ta)?; + put16(out, 0x38, ta.rce_count); + put16(out, 0x3a, ta.registers.len() as u16); + put16(out, 0x3c, ta.registers.len() as u16); + put64(out, 0x85e, dispatch); + extra_tail(out, RenderStage::Tiling, &t.extra); + let empty = None; + Ok([Some(RegisterArrayBinding::new(address, ta.rce_count as u8).map_err(|_| M5RenderCommandError::ProgramOffset)?), empty, empty, empty]) +} + +pub(crate) fn encode_m5_3d_command( + out: &mut [u8; M5_3D_COMMAND_BYTES], + low_alias: u64, + i: &M5RenderCommandInputs, + f: &M5FragmentProgramRefs<'_>, +) -> Result<[Option; 4], M5RenderCommandError> { + let programs = [f.main, f.store, f.resume, f.load]; + for (program, offset) in programs.iter().zip(M5_3D_PROGRAM_OFFSETS) { + check_program(program, offset)?; + low_alias.checked_add(u64::from(program.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + } + for partial in &programs[1..] { + if usize::from(partial.rce_count) != partial.registers.len() { + return Err(M5RenderCommandError::RceCount); + } + } + if f.main_mid_count < f.main.rce_count || usize::from(f.main_mid_count) > f.main.registers.len() { + return Err(M5RenderCommandError::RceCount); + } + let tile = f + .main + .registers + .iter() + .find(|r| r.reg & !1 == FRAGMENT_TILE_REGISTER) + .ok_or(M5RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.fragment; + put32(out, 0, M5_3D_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + tail::encode(out, RenderStage::Fragment, &RenderTail { + descriptor_value: t.tail_descriptor, resolved_value: t.resolved_tail_value, + trailing_value: t.context_token, + }).expect("fixed fragment command extent"); + put32(out, 0x2170, t.fragment_word_l); + put32(out, 0x2174, u32::from(t.fragment_flag_a)); + put32(out, 0x2100, u32::from(t.short_partial_programs & 1)); + put32(out, 0x20f8, 0); + + put128(out, 0x2178, [0, 0]); + put64(out, 0x2188, 0); + put8(out, 0x21d0, t.tail_flag & 1); + put64(out, 0x20, i.scheduler_record); + put64(out, 0x28, t.pm_state); + put64(out, 0x30, t.pm_scene); + put64(out, 0x38, t.scene_end_va); + put32(out, 0x58, t.fragment_word_a); + put64(out, 0x40, t.tilemap_va); + put64(out, 0x48, t.msaa_control); + put32(out, 0x50, t.sample_count); + put32(out, 0x54, t.utile_grid); + put128(out, 0x60, t.merge_bounds); + put64(out, 0x70, t.fragment_word_b); + put32(out, 0x78, t.tile_count); + put64(out, 0x1d30, t.scissor_va); + put64(out, 0x1d20, t.depth_bias_va); + put64(out, 0x1d40, t.occlusion_query_va); + for (index, pair) in t.binding_pairs.iter().enumerate() { + put128(out, 0x1d58 + index * 16, *pair); + } + put64(out, 0x1ec8, t.depth_va); + put64(out, 0x1ed0, t.depth_stride); + put64(out, 0x1ed8, t.depth_compression_stride); + put64(out, 0x1f28, t.fragment_word_d); + put128(out, 0x1ee0, t.depth_va_pair); + put64(out, 0x1ef0, t.depth_compression_va); + put64(out, 0x1ef8, t.stencil_va); + put64(out, 0x1f00, t.stencil_stride); + put64(out, 0x1f08, t.stencil_compression_stride); + put64(out, 0x1f30, t.fragment_word_e); + put128(out, 0x1f10, t.stencil_va_pair); + put64(out, 0x1f20, t.stencil_compression_va); + put64(out, 0x1f68, t.fragment_word_f); + put32(out, 0x1f70, t.fragment_word_g); + put128(out, 0x1f48, t.size_aux_control); + put128(out, 0x1f74, t.eot_program); + put64(out, 0x1f88, t.fragment_word_h); + put32(out, 0x1f90, t.fragment_word_i); + put128(out, 0x1f94, t.partial_eot_program); + put64(out, 0x1fa8, t.zs_clear_values); + put32(out, 0x1fb0, t.sample_size); + put128(out, 0x1f38, t.sample_aux_flags); + put64(out, 0x1d50, t.fragment_word_c); + put64(out, 0x1fc0, t.fragment_word_j); + put32(out, 0x1fc8, t.fragment_word_k); + put64(out, 0x1fd0, t.depth_dimensions); + out[0x1fdc..0x20dc].fill(0); + put32(out, 0x20dc, 0); + put128(out, 0x20e0, [0, 0]); + put32(out, 0x1fd8, 0); + put32(out, 0x20f0, 0); + put8(out, 0x90, t.pass); + put32(out, 0x20fc, u32::from(t.fragment_pass_flag & 1)); + put32(out, 0x2209, t.fragment_word_n); + put8(out, 0x220d, t.fragment_flag_c); + put32(out, 0x2104, u32::from(t.stage_flag_a & 1)); + put32(out, 0x2108, t.predecessor_id); + put32(out, 0x210c, t.stage_word_a); + put32(out, 0x2110, t.stage_word_b); + put128(out, 0x2114, [0, 0]); + put128(out, 0x21a8, t.fragment_pair_a); + put32(out, 0x21b8, t.timestamp_counts[0]); + put32(out, 0x21bc, t.timestamp_counts[1]); + put64(out, 0x21c0, t.fragment_word_m); + put64(out, 0x21c8, t.uma_state); + put128(out, 0x2198, t.timestamp_va); + completion_tail(out, 0x2140, &t.completion); + put64(out, 0x2160, t.completion.completion); + put64(out, 0x2190, 0); + for (index, flag) in t.empty_tile_flags.iter().enumerate() { + put32(out, 0x2124 + index * 4, u32::from(flag & 1)); + } + put32(out, 0x2138, u32::from(t.fragment_flag_b & 1)); + put64(out, 0x2168, t.completion.key); + extra_tail(out, RenderStage::Fragment, &t.extra); + + put32(out, 0x80, 0); + put16(out, 0x84, 0); + put32(out, 0x88, 0); + let mut rce = [None; 4]; + for (slot, program) in rce.iter_mut().zip(programs.iter()) { + let address = place_program(out, low_alias, program)?; + *slot = Some(RegisterArrayBinding::new(address, program.rce_count as u8).map_err(|_| M5RenderCommandError::ProgramOffset)?); + } + put64(out, 0x1f58, tile); + put64(out, 0x1f60, 0); + put16(out, 0x80, f.main.rce_count); + put16(out, 0x82, f.main_mid_count); + put16(out, 0x84, f.main_mid_count); + put16(out, 0x88, f.main.registers.len() as u16); + Ok(rce) +} + +fn m5_ta_policy_bit(p: &M5TaPolicy, shader_write_class: u8) -> bool { + if p.policy_a == 0 || !(1..=2).contains(&p.policy_b) || p.policy_c == 3 { + return false; + } + match p.policy_a { + 1 | 4 => shader_write_class & 1 == 0, + 3 => true, + _ => false, + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs new file mode 100644 index 0000000000000..1966ab22f4878 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Assemble an M5 render pair from shared render resources. +use kernel::prelude::*; +use super::{render_registers as regs, render as cmd}; +use super::render_registers::{Workspace, M5RenderControls}; +use super::super::{render::RenderArgs, kick::{RegisterArrayBinding, RegisterWrite}}; + +pub(crate) const FRAGMENT_OFFSET: usize = 0x4000; +pub(crate) const COMMAND_STORAGE_BYTES: usize = 0x8000; + +pub(crate) struct EncodedRender { + pub(crate) rce: [[Option; 4]; 2], + pub(crate) register_counts: [u16; 5], + pub(crate) kick_flags: super::super::kick::StageKickFlags, +} + +fn commands(storage: &mut [u8]) -> Result<( + &mut [u8; cmd::M5_TA_COMMAND_BYTES], + &mut [u8; cmd::M5_3D_COMMAND_BYTES], +)> { + if storage.len() < COMMAND_STORAGE_BYTES { return Err(EINVAL); } + let (ta, fragment) = storage.split_at_mut(FRAGMENT_OFFSET); + Ok(((&mut ta[..cmd::M5_TA_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + (&mut fragment[..cmd::M5_3D_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?)) +} + +pub(crate) fn link_completions(storage: &mut [u8], + kicks: [super::super::kick::KickTimestamp; 2], pass: u8, fragment_busy: bool) -> Result { + let (ta, fragment) = commands(storage)?; + cmd::link_completions(ta, fragment, kicks, pass, fragment_busy); + Ok(()) +} + +pub(crate) fn bind_work_slot(storage: &mut [u8], slot: u32) -> Result { + let (ta, fragment) = commands(storage)?; + cmd::bind_work_slot(ta, fragment, slot).map_err(|_| EINVAL) +} + +#[inline(never)] +pub(crate) fn encode_into(storage: &mut [u8], low: u64, + args: &RenderArgs<'_>, controls: &M5RenderControls, + extra: regs::TaExtra, dispatch: regs::FragmentDispatch, + inputs: &cmd::M5RenderCommandInputs, workspace: &mut Workspace) + -> Result { + if storage.len() < COMMAND_STORAGE_BYTES { return Err(EINVAL); } + let fragment_low = low.checked_add(FRAGMENT_OFFSET as u64).ok_or(EINVAL)?; + fragment_low.checked_add(cmd::M5_3D_COMMAND_BYTES as u64).ok_or(EINVAL)?; + if inputs.context_id != u32::from(args.resources.context_id) { return Err(EINVAL); } + workspace.write(args, controls, extra, dispatch, inputs)?; + let short_partial = [RegisterWrite::new(0x160a8, 0x20), RegisterWrite::new(0x16068, 0x4_0000)]; + let (store, resume, load): (&[_], &[_], &[_]) = if inputs.fragment.short_partial_programs & 1 != 0 { + (&short_partial, &short_partial, &short_partial) + } else { (&workspace.store, &workspace.resume, &workspace.load) }; + let partial_counts = [store.len() as u16, resume.len() as u16, load.len() as u16]; + let count = workspace.ta.registers().len() as u16; + let ta_program = cmd::M5ProgramRef { registers: workspace.ta.registers(), + rce_count: count - 3, command_offset: cmd::M5_TA_PROGRAM_OFFSET }; + let view = |registers, rce_count, command_offset| cmd::M5ProgramRef { + registers, rce_count, command_offset }; + let fragment_programs = cmd::M5FragmentProgramRefs { + main: view(&workspace.fragment[..workspace.fragment_count], + workspace.fragment_count as u16 - 4, 0xa0), + main_mid_count: workspace.fragment_count as u16 - 2, + store: view(store, partial_counts[0], 0x7c0), + resume: view(resume, partial_counts[1], 0xee0), + load: view(load, partial_counts[2], 0x1600), + }; + let ta = &mut workspace.encoded_ta; + let fragment = &mut workspace.encoded_fragment; + ta.fill(0); fragment.fill(0); + let fragment_rce = cmd::encode_m5_3d_command(fragment, fragment_low, inputs, &fragment_programs).map_err(|_| EINVAL)?; + let ta_rce = cmd::encode_m5_ta_command(ta, low, inputs, &ta_program).map_err(|_| EINVAL)?; + let (ta_storage, fragment_storage) = storage.split_at_mut(FRAGMENT_OFFSET); + ta_storage[..cmd::M5_TA_COMMAND_BYTES].copy_from_slice(ta); + fragment_storage[..cmd::M5_3D_COMMAND_BYTES].copy_from_slice(fragment); + // A kick sets header bit 61 for a stage whose register program has the + // SCS bracket and bit 63 for its matrix bracket, from the same flags that + // gate those brackets. The fragment matrix bracket is never emitted here. + let kick_flags = super::super::kick::StageKickFlags { + scratch: [extra.scs_bracket & 1 != 0 || controls.ta.scs.scratch.enabled(), + controls.fragment.scs.scratch.enabled()], + matrix: [extra.matrix_bracket & 1 != 0, false], + }; + Ok(EncodedRender { rce: [ta_rce, fragment_rce], + register_counts: [count, workspace.fragment_count as u16, partial_counts[0], partial_counts[1], partial_counts[2]], + kick_flags }) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs new file mode 100644 index 0000000000000..d99970e2f126e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Program and tile storage pressure for shared render pools. Program sizes +//! are byte counts after linking; they are independent of register allocation. + +use super::pool::PoolError; +use super::pool_budget::Requirements; + +type Result = core::result::Result; + +fn add(a: u64, b: u64) -> Result { a.checked_add(b).ok_or(PoolError::Geometry) } +fn mul(a: u64, b: u64) -> Result { a.checked_mul(b).ok_or(PoolError::Geometry) } +fn count(value: u64) -> Result { value.try_into().map_err(|_| PoolError::Geometry) } +fn pages(value: u64, size: u64) -> Result { Ok(add(value, size - 1)? / size) } + +#[derive(Clone, Copy, Default)] +pub(crate) struct FragmentStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, + pub(crate) auxiliary_count: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct FragmentParams { + pub(crate) pressure: u32, + pub(crate) page_units: u32, + pub(crate) large_units: u32, + pub(crate) auxiliary_units: u32, + pub(crate) primary_large: u32, + pub(crate) secondary_large: u32, +} + +impl FragmentStorage { + pub(crate) fn parameters(self) -> Result { + let primary = mul(add(u64::from(self.primary_bytes), u64::from(self.linked_bytes))? + .max(u64::from(self.minimum_primary_bytes)), 32)?; + let primary = add(primary, 0x4380)?; + let secondary = mul(u64::from(self.secondary_bytes), 32)?; + let primary_large = pages(primary, 0x10000)?; + let secondary_large = pages(secondary, 0x10000)?; + let small = add(pages(primary, 0x1000)?, pages(secondary, 0x1000)?)?; + let large = add(primary_large, secondary_large)?; + let auxiliary = mul(u64::from(self.auxiliary_count), 128)?; + let (auxiliary, reserved) = if auxiliary == 0 { (0, 0) } + else { (auxiliary | 64, auxiliary.max(0x4000) | 64) }; + let pressure = add(add(mul(small, 96)?, mul(large, 10)?)?, + add(mul(pages(auxiliary, 0x1000)?, 64)?, mul(pages(auxiliary, 0x10000)?, 5)?)?)?; + Ok(FragmentParams { + pressure: count(pressure)?, page_units: count(add(small, large)?)?, + large_units: count(large)?, + auxiliary_units: count(add(pages(reserved, 0x1000)?, pages(reserved, 0x10000)?)?)?, + primary_large: count(primary_large)?, secondary_large: count(secondary_large)?, + }) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct TileStorage { + pub(crate) threadgroup_bytes: u32, + pub(crate) imageblock_bytes: u32, + pub(crate) multisample_bytes: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TileParams { + pub(crate) pressure: u32, + pub(crate) page_units: u32, + pub(crate) large_units: u32, + pub(crate) threadgroup_large: u32, + pub(crate) imageblock_large: u32, + pub(crate) multisample_large: u32, +} + +impl TileStorage { + pub(crate) fn parameters(self, preempt_formula: bool) -> Result { + let imageblock = u64::from(self.imageblock_bytes); + let sizes = [u64::from(self.threadgroup_bytes), + if imageblock == 0 { 0 } else { add(imageblock, 64)? }, + u64::from(self.multisample_bytes)]; + let large = [pages(sizes[0], 0x10000)?, pages(sizes[1], 0x10000)?, + pages(sizes[2], 0x10000)?]; + let small = add(add(pages(sizes[0], 0x1000)?, pages(sizes[1], 0x1000)?)?, + pages(sizes[2], 0x1000)?)?; + let total_large = add(add(large[0], large[1])?, large[2])?; + let factor = if preempt_formula { 11 } else { 6 }; + Ok(TileParams { + pressure: count(add(mul(small, factor)?, mul(total_large, factor.min(8))?)?)?, + page_units: count(add(small, total_large)?)?, large_units: count(total_large)?, + threadgroup_large: count(large[0])?, imageblock_large: count(large[1])?, + multisample_large: count(large[2])?, + }) + } +} + +/// One submission's maximum pressure across all of its linked render +/// programs. Different programs may maximize different storage dimensions. +#[derive(Clone, Copy)] +pub(crate) struct RenderDemand { + counts: [u32; 5], + minimum_pages: u32, + ideal_pages: u32, + pressure: u32, + clusters: u64, + maximum: u64, + total: u64, + floor: u64, + limited_maximum: u64, + limited_total: u64, + included: bool, +} + +impl RenderDemand { + pub(crate) fn new(core_masks: &[u32], nominal_cores: u32, ptc_floor: u32) -> Result { + let clusters = core_masks.len() as u64; + let maximum = u64::from(core_masks.iter().map(|m| m.count_ones()).max().unwrap_or(0)); + let total = core_masks.iter().try_fold(0u64, |n, m| add(n, u64::from(m.count_ones())))?; + if total == 0 || ptc_floor == 0 || u64::from(nominal_cores) < total { + return Err(PoolError::Geometry); + } + let limit = mul(u64::from(nominal_cores), 2)? / clusters; + Ok(Self { + counts: [0; 5], minimum_pages: 0, ideal_pages: 0, pressure: 0, + clusters, maximum, total, floor: u64::from(ptc_floor), + limited_maximum: maximum.min(limit), limited_total: total.min(mul(limit, clusters)?), + included: false, + }) + } + + pub(crate) fn include_fragment(&mut self, fragment: FragmentParams, tile: TileParams) -> Result<()> { + let Self { clusters, maximum, total, floor, .. } = *self; + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let fragment_small = u64::from(fragment.page_units); + let tile_small = u64::from(tile.page_units); + let pressure = add(u64::from(fragment.pressure), u64::from(tile.pressure))?; + let large = add(add(u64::from(fragment.large_units), u64::from(fragment.auxiliary_units))?, + u64::from(tile.large_units))?; + let parts = [fragment.primary_large, fragment.secondary_large, + tile.threadgroup_large, tile.imageblock_large, tile.multisample_large]; + let tails = parts[..4].iter().try_fold(0u64, |n, &v| add(n, u64::from(v)))?; + let ideal = add(add(add(base, u64::from(tile.multisample_large))?, tails)?, + add(mul(pressure.saturating_sub(floor), maximum)?, 64)?)?; + let minimum = add(add(add(base, u64::from(tile.multisample_large))?, tails)?, + add(add(mul(add(add(fragment_small, tile_small)?.saturating_sub(floor), tile_small)?, + maximum)?, mul(large, total)?)?, 64)?)?; + self.merge(parts, minimum, ideal, pressure, clusters) + } + + /// Vertex work has no imageblock and uses the tiling execution limits. + pub(crate) fn include_vertex(&mut self, storage: VertexStorage) -> Result<()> { + let params = FragmentStorage { + primary_bytes: storage.primary_bytes, secondary_bytes: storage.secondary_bytes, + linked_bytes: storage.linked_bytes, minimum_primary_bytes: storage.minimum_primary_bytes, + auxiliary_count: 0, + }.parameters()?; + let Self { clusters, floor, limited_maximum: maximum, limited_total: total, .. } = *self; + let large = u64::from(params.large_units); + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let minimum = add(add(add(base, large)?, 64)?, + add(mul(u64::from(params.page_units).saturating_sub(floor), maximum)?, + mul(large, total)?)?)?; + let ideal = add(add(add(base, large)?, 64)?, + mul(u64::from(params.pressure).saturating_sub(floor), maximum)?)?; + self.merge([params.primary_large, params.secondary_large, 0, 0, 0], minimum, + ideal, u64::from(params.pressure), clusters) + } + + /// Compute storage is evaluated per dispatch before merging component maxima. + /// `resident_warps` and `shared_budget` are device policy, not API limits. + pub(crate) fn include_compute(&mut self, storage: ComputeStorage, + resident_warps: u32, shared_budget: u32) -> Result<()> { + if storage.threads == 0 || resident_warps == 0 { + return Err(PoolError::Geometry); + } + let frame = mul(add(u64::from(storage.primary_bytes), + u64::from(storage.linked_bytes))? + .max(u64::from(storage.minimum_primary_bytes)), 32)?; + let frame = add(frame, 0x4380)?; + let secondary = mul(u64::from(storage.secondary_bytes), 32)?; + let primary_large = pages(frame, 0x10000)?; + let secondary_large = pages(secondary, 0x10000)?; + let small = add(pages(frame, 0x1000)?, pages(secondary, 0x1000)?)?; + let large = add(primary_large, secondary_large)?; + let shared = u64::from(storage.threadgroup_bytes); + let shared_small = pages(shared, 0x1000)?; + let shared_large = pages(shared, 0x10000)?; + let warps = pages(u64::from(storage.threads), 32)?; + let mut groups = u64::from(resident_warps) / warps; + if shared != 0 { + let aligned_shared = mul(pages(shared, 64)?, 64)?; + groups = groups.min(u64::from(shared_budget) / aligned_shared); + } + if groups == 0 { return Err(PoolError::Geometry); } + let pressure = add(add(mul(mul(groups, warps)?.min(96), small)?, + mul(mul(groups, warps)?.min(10), large)?)?, + add(mul(groups, shared_small)?, mul(groups.min(8), shared_large)?)?)?; + let Self { clusters, maximum, total, floor, .. } = *self; + let group_large = mul(warps.min(10), large)?; + let tile = add(shared_small, shared_large)?; + let transient = add(add(mul(warps, small)?, group_large)?, tile)?; + let transient = add(transient.saturating_sub(floor), tile)?; + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let tail = add(add(large, shared_large)?, 64)?; + let minimum = add(add(add(base, tail)?, mul(transient, maximum)?)?, + mul(add(shared_large, group_large)?, total)?)?; + let ideal = add(add(base, tail)?, mul(pressure.saturating_sub(floor), maximum)?)?; + self.merge([count(primary_large)?, count(secondary_large)?, + count(shared_large)?, 0, 0], minimum, ideal, pressure, clusters) + } + + /// Mesh/object execute on the tiling engine. Reuse compute's occupancy + /// arithmetic with the tiling-limited core topology, never compute's full + /// enabled core count. Resident/shared limits are explicit device policy. + pub(crate) fn include_mesh(&mut self, storage: ComputeStorage, + resident_warps: u32, shared_budget: u32) -> Result<()> { + let maximum = self.maximum; + let total = self.total; + self.maximum = self.limited_maximum; + self.total = self.limited_total; + let result = self.include_compute(storage, resident_warps, shared_budget); + self.maximum = maximum; + self.total = total; + result + } + + fn merge(&mut self, parts: [u32; 5], minimum: u64, ideal: u64, pressure: u64, + clusters: u64) -> Result<()> { + // Stored page counts are signed before conversion to byte counts. + // Reject an unrepresentable request instead of wrapping into a small pool. + let round = |value| -> Result { + let value = add(mul(value, clusters)?, 7)? & !7; + if value > i32::MAX as u64 { return Err(PoolError::Geometry); } + count(value) + }; + let mut next = *self; + next.minimum_pages = next.minimum_pages.max(round(minimum)?); + next.ideal_pages = next.ideal_pages.max(round(ideal)?); + next.pressure = next.pressure.max(count(pressure)?); + for (out, value) in next.counts.iter_mut().zip(parts) { *out = (*out).max(value); } + next.resource_flags()?; + next.included = true; + *self = next; + Ok(()) + } + + pub(crate) fn pressure(&self) -> u32 { self.pressure } + + pub(crate) fn resource_flags(&self) -> Result { + let [primary, secondary, shared, imageblock, multisample] = self.counts; + if primary > 0xff || secondary > 0xf || shared > 3 || imageblock > 0xff + || multisample > 0xff { return Err(PoolError::Geometry); } + Ok(0x14b | (u64::from(primary) << 10) | (u64::from(secondary) << 18) + | (u64::from(shared) << 22) | (u64::from(imageblock) << 24) + | (u64::from(multisample) << 32)) + } + + pub(crate) fn requirements(&self) -> Result { + if !self.included { + return Err(PoolError::Geometry); + } + let Self { maximum, clusters, .. } = *self; + let counts = self.counts.iter().try_fold(0u64, |n, &v| add(n, u64::from(v)))?; + let render_target = mul(add(add(mul(maximum, 11)?, counts)?, 10)?, clusters)?; + Ok(Requirements { + minimum_bytes: u64::from(self.minimum_pages) << 12, + ideal_bytes: u64::from(self.ideal_pages) << 12, + self_reserve_pages: 0, render_target_pages: count(render_target)?, has_spill: false, + }) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct VertexStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, +} + +/// Storage and execution width of a linked compute dispatch. +#[derive(Clone, Copy, Default)] +pub(crate) struct ComputeStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, + pub(crate) threadgroup_bytes: u32, + pub(crate) threads: u32, +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs new file mode 100644 index 0000000000000..15e1887cee272 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 register adaptation over the shared render-pass/resource lowering. +#![allow(dead_code)] +use kernel::prelude::*; +use super::super::{kick::RegisterWrite, render::{RenderArgs, RenderPass, M5RegisterPrograms as RegisterPrograms}}; +use super::render::M5RenderCommandInputs; + +/// A pass's tiler context store for N clusters: 0x20 + 0x40 x N bytes of per-cluster control +/// words, the auxiliary sub-buffer at the next 64-byte boundary, the state stream +/// align64((N + 1) x 0x2e0) after it, and N x 0x680 + 0x800 bytes of the stream's storage +/// beyond it. Offsets from the store's base. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ContextStore { + pub(crate) auxiliary: usize, + pub(crate) stream: usize, + pub(crate) extent: usize, +} +const fn align64(bytes: usize) -> usize { + (bytes + 63) & !63 +} +impl ContextStore { + pub(crate) const fn of(clusters: usize) -> Self { + let auxiliary = align64(0x20 + 0x40 * clusters); + let stream = auxiliary + align64(0x2e0 * (clusters + 1)); + Self { auxiliary, stream, extent: stream + 0x680 * clusters + 0x800 } + } + + /// `board`'s store: laid out for the G17D's own cluster count with its forms; other boards + /// keep the two-cluster store. + pub(crate) const fn selected(board: &super::initdata::Board) -> Self { + Self::of(if board.g17d_forms() { board.clusters as usize } else { 2 }) + } +} +const _: () = { + let two = ContextStore::of(2); + assert!(two.auxiliary == 0xc0 && two.stream == 0x980 && two.extent == 0x1e80); + let eight = ContextStore::of(8); + assert!(eight.auxiliary == 0x240 && eight.stream == 0x1c40 && eight.extent == 0x5840); +}; + +fn utile_config(p: &RenderPass) -> u64 { + (u64::from(p.utile_width / 16) << 12) | (u64::from(p.utile_height / 16) << 14) + | u64::from(p.samples_log2) +} +#[derive(Copy, Clone)] +pub(crate) struct M5RenderStageControls { + pub(crate) scs: crate::g17::render_resources::StageScratch, + + pub(crate) resource_flags: u64, + + pub(crate) resource_control: u64, + + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Copy, Clone)] +pub(crate) struct M5RenderControls { + /// The board the render's encodings are for. + pub(crate) board: &'static super::initdata::Board, + /// Encoded event-command state for the initial fragment register list. + pub(crate) fragment_event_control: u64, + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + + pub(crate) fragment_tile_flags: u64, + + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_control: u64, +} + +struct RegisterWriter<'a> { + values: &'a mut [RegisterWrite], + count: usize, +} + +impl RegisterWriter<'_> { + fn push(&mut self, number: u32, value: u64) -> Result { + let slot = self.values.get_mut(self.count) + .ok_or(EINVAL)?; + *slot = RegisterWrite { reg: number, value }; + self.count += 1; + Ok(()) + } + fn finish(self) -> Result { + if self.count != self.values.len() { + return Err(EINVAL); + } + Ok(()) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.reg == number) + .map(|r| r.value) + .ok_or(EINVAL) +} + +/// The event-command state of the initial fragment register list: 0 when +/// disabled, 3 for kind 1, otherwise 1. +pub(crate) const fn fragment_event_control(enabled: bool, kind: u32) -> u64 { + if !enabled { 0 } else if kind == 1 { 3 } else { 1 } +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &RenderPass, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(EINVAL); + } + + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(EINVAL); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((utile_config(p) & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers_into( + p: &RenderPass, + controls: &M5RenderControls, + free_list_slot: u32, + out: &mut [RegisterWrite; 82], + common: &[RegisterWrite; 73], +) -> Result { + if controls.ta_dispatch_control & 1 != 0 { + return Err(EINVAL); + } + + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterWriter { values: out, count: 0 }; + for mut register in common.iter().copied() { + match register.reg { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + // A fixed word of the TA program. Sample locations are bound by + // their own register. + 0x10131 => register.value = 0x88, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + // Above four clusters the tiler's control word takes the topology term, as + // compute's dispatch control does; the fragment side's must agree with it. + register.value = + super::compute::dispatch_control(controls.board, controls.ta_tile_control); + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(EINVAL); + } + register.value |= controls.ta_dispatch_control; + } + 0x101c9 => register.value = controls.ta.scs.scratch.size_word(), + 0xd471 => register.value = u64::from(controls.ta.scs.auxiliary_allocation), + 0x1a0f1 => register.value = controls.ta.scs.scratch.stack_word(), + 0x1a099 => register.value = controls.ta.scs.intersection.words()[0], + 0x1a0a1 => register.value = controls.ta.scs.intersection.words()[1], + 0x1a069 => register.value = controls.ta.scs.intersection.words()[2], + 0x1a071 => register.value = controls.ta.scs.intersection.words()[3], + 0x1a0c9 => register.value = controls.ta.scs.intersection.words()[4], + 0x1a0d1 => register.value = controls.ta.scs.intersection.words()[5], + 0x10799 => { + register.value = controls.ta.scs.scratch.window_word() + | (register.value & (1 << 25)); + out.push(0x1a149, controls.ta.scs.scratch.driver_word())?; + }, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = free_list_slot as i32 as i64 as u64, + _ => {} + } + out.push(register.reg, register.value)?; + if register.reg == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers_into( + p: &RenderPass, + controls: &M5RenderControls, + free_list_slot: u32, + out: &mut [RegisterWrite; 90], + common: &[RegisterWrite; 89], +) -> Result { + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterWriter { values: out, count: 0 }; + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common.iter().copied() { + match register.reg { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.reg = 0x16469, + 0x100d9 => register.reg = 0x16471, + 0x100b8 => { + register.reg = 0x16138; + register.value = super::compute::dispatch_control(controls.board, register.value); + } + 0x10791 => { + let scratch = controls.fragment.scs.scratch; + register.value = if scratch.enabled() { + (controls.fragment_raster_control | (1 << 24)) & 0x200001000204 + | (scratch.window_word() & 0xff0000) + | (controls.fragment_raster_control & (1 << 25)) + } else { controls.fragment_raster_control }; + }, + 0x101c1 => register.value = controls.fragment.scs.scratch.size_word(), + 0xd469 => register.value = u64::from(controls.fragment.scs.auxiliary_allocation), + 0x1a0f9 => register.value = controls.fragment.scs.scratch.stack_word(), + 0x1a0a9 => register.value = controls.fragment.scs.intersection.words()[0], + 0x1a0b1 => register.value = controls.fragment.scs.intersection.words()[1], + 0x1a079 => register.value = controls.fragment.scs.intersection.words()[2], + 0x1a081 => register.value = controls.fragment.scs.intersection.words()[3], + 0x1a0d9 => register.value = controls.fragment.scs.intersection.words()[4], + 0x1a0e1 => register.value = controls.fragment.scs.intersection.words()[5], + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(EINVAL); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, controls.fragment.scs.scratch.driver_word())?, + 0x10211 => { + out.push(0x0a741, controls.fragment_event_control)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = free_list_slot as i32 as i64 as u64, + _ => {} + } + out.push(register.reg, register.value)?; + if register.reg == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.reg == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(EINVAL); + } + out.finish() +} + +#[derive(Copy, Clone)] +pub(crate) struct FragmentDispatch { + pub(crate) words: [u64; 2], +} + +impl FragmentDispatch { + + pub(crate) fn initial(self, descriptor_flag: u8) -> u64 { + if descriptor_flag & 1 != 0 { return 0x20; } + ((self.words[0] >> 27) & 0x1f_ffff_f860) + | (self.words[1].wrapping_shl(2) & 0x3_ffff_f87c) + | ((self.words[1] >> 32) << 7) + } +} + +#[derive(Copy, Clone)] +pub(crate) struct TaExtra { + pub(crate) gtp1: u64, + pub(crate) gtp2: u64, + pub(crate) gtp4: u64, + pub(crate) gtp5: u64, + pub(crate) gtp6: u64, + pub(crate) gtp7: u64, + pub(crate) segment_control: u64, + pub(crate) segment_index: u64, + pub(crate) min_free_tvb_pages: u64, + pub(crate) scs_bracket: u8, + pub(crate) matrix_bracket: u8, +} + +pub(crate) struct TaProgram { + registers: [RegisterWrite; 97], + length: usize, +} + +impl TaProgram { + pub(crate) const fn empty() -> Self { + Self { registers: [RegisterWrite {reg: 0, value: 0}; 97], length: 0 } + } + pub(crate) fn registers(&self) -> &[RegisterWrite] { &self.registers[..self.length] } + pub(crate) fn set_lifecycle(&mut self, value: u64) { + for r in &mut self.registers[..self.length] { + if matches!(r.reg, 0x1ca10 | 0x14a1 | 0xa349) { + r.value = value; + } + } + } + fn push(&mut self, number: u32, value: u64) -> Result { + let slot = self.registers.get_mut(self.length).ok_or(EINVAL)?; + *slot = RegisterWrite { reg: number, value }; + self.length += 1; + Ok(()) + } +} + +pub(crate) fn tiling_with_scratch_into(common: &[RegisterWrite], e: TaExtra, + mesh_storage: u64, out: &mut TaProgram) -> Result { + out.length = 0; + if e.scs_bracket & 1 != 0 { out.push(0x17e1, 1)?; } + if e.matrix_bracket & 1 != 0 { out.push(0x17f1, 1)?; } + let mut deflake = false; + for r in common.iter().copied() { + if r.reg == 0x1748 { continue; } + if r.reg == 0x1740 { + if e.scs_bracket & 1 != 0 { out.push(0x17d9, 1)?; } + if e.matrix_bracket & 1 != 0 { out.push(0x17e9, 1)?; } + continue; + } + out.push(r.reg, r.value)?; + let register = r.reg; + match register { + // In a mesh-object TA program the mesh rendering buffer register + // follows the scratch-window register. + 0x10799 if mesh_storage != 0 => out.push(0x1ca48, mesh_storage)?, + 0x1c9c8 => { out.push(0x1c041, e.gtp1)?; out.push(0x1c9d0, e.gtp1)?; } + 0x1c880 if !deflake => { + deflake = true; + for (n,v) in [(0x1c948,e.gtp4),(0x1c888,e.gtp5), + (0x1c890,e.segment_control),(0x1c918,e.segment_index)] { out.push(n,v)?; } + } + 0x1c9d8 => { + for (n,v) in [(0x1c089,e.gtp2),(0x1c9e0,e.gtp2), + (0x16c41,e.gtp6 & !0x7f),(0x1ca40,e.gtp6 & !0x7f), + (0x1c9a8,e.min_free_tvb_pages),(0x1c920,e.gtp7 & !0x3f)] { out.push(n,v)?; } + } + _ => {} + } + } + if out.length != 92 + usize::from(e.scs_bracket & 1) * 2 + usize::from(e.matrix_bracket & 1) * 2 + + usize::from(mesh_storage != 0) { + return Err(EINVAL); + } + Ok(()) +} + +pub(crate) fn fragment_main_with_scratch_into(p: &RenderPass, common: &[RegisterWrite], + dispatch: FragmentDispatch, descriptor_flag: u8, scs: bool, out: &mut [RegisterWrite]) -> Result { + + let mut n = 0; + let mut tile = 0; + for mut r in common.iter().copied() { + + if r.reg == 0x10051 { + r.value = isp_sample_control(p); + } + if r.reg == 0x16068 { + if tile == 2 { return Err(EINVAL); } + out[n] = RegisterWrite { reg: 0x160a8, + value: if tile == 0 { dispatch.initial(descriptor_flag) } else { 0x20 } }; + n += 1; + tile += 1; + } + let slot = out.get_mut(n).ok_or(EINVAL)?; + *slot = r; + n += 1; + // The SCS controls bracket the fragment main list after entry and exit. + if scs && matches!(r.reg, 0x1739 | 0x1731) { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite { + reg: if r.reg == 0x1739 { 0x17e1 } else { 0x17d9 }, value: 1 }; + n += 1; + } + } + if n != out.len() || tile != 2 { return Err(EINVAL); } + Ok(()) +} + +/// ISP sample allocation, in the M5 register's units. +pub(crate) fn isp_sample_control(p: &RenderPass) -> u64 { + let samples = 1u32 << p.samples_log2; + if samples <= 1 { return 0; } + let sample_bytes = (u32::from(p.sample_size).max(8) + 7) & !7; + let groups = sample_bytes.div_ceil(32); + let units = (u32::from(p.utile_width) >> 3) * (u32::from(p.utile_height) >> 2) * groups; + let value = match samples { 2 => units.max(8) >> 3, 4 => units >> 1, 8 => units, _ => 0 }; + let value = if samples == 2 && p.utile_width == 16 && p.utile_height == 16 { value } else { value >> 1 }; + u64::from(value & 0x1f) << 16 +} + +fn partial(common: &[RegisterWrite], insert_after: u32, + c: &M5RenderControls, load: bool, out: &mut [RegisterWrite; N]) -> Result { + let mut w = RegisterWriter { values: out, count: 0 }; + for r in common.iter().copied() { + if r.reg == 0x15359 { continue; } + w.push(r.reg, r.value | if r.reg == 0x10039 { c.partial_tile_flags } else { 0 })?; + if r.reg == insert_after { + w.push(0x153b1, if load { c.store_pipeline_2_bind } else { c.partial_store_pipeline_2_bind })?; + w.push(0x153b9, if load { c.store_pipeline_2 } else { c.partial_store_pipeline_2 })?; + } + } + w.finish() +} + +/// Caller-owned assembly scratch. All resource addresses come from RenderArgs. +pub(crate) struct Workspace { + common: RegisterPrograms, + base: [RegisterWrite; 90], + pub(crate) ta: TaProgram, + pub(crate) fragment: [RegisterWrite; 94], + pub(crate) fragment_count: usize, + pub(crate) store: [RegisterWrite; 17], + pub(crate) resume: [RegisterWrite; 24], + pub(crate) load: [RegisterWrite; 12], + // Assemble in normal cached memory. Firmware command mappings on M5 + // are write-combined and must not be used for header read/modify/write. + pub(crate) encoded_ta: [u8; super::render::M5_TA_COMMAND_BYTES], + pub(crate) encoded_fragment: [u8; super::render::M5_3D_COMMAND_BYTES], +} +// SAFETY: Every field consists only of integer storage. +unsafe impl Zeroable for Workspace {} +impl Workspace { + #[inline(never)] + pub(crate) fn write(&mut self, args: &RenderArgs<'_>, controls: &M5RenderControls, + extra: TaExtra, dispatch: FragmentDispatch, inputs: &M5RenderCommandInputs) -> Result { + self.common.write_m5(args)?; + let r = args.resources; + let p = args.pass; + // The two sub-buffers sit at the context store's offsets from its base. + let base = r.deflake_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?; + let store = ContextStore::selected(controls.board); + let (state_offset, aux_offset) = (store.stream as u64, store.auxiliary as u64); + let mut vdm_seen = false; + for reg in &mut self.common.ta { + let register = reg.reg; + reg.value = match register { + 0x1c880 if vdm_seen => base.checked_add(state_offset).ok_or(EINVAL)?, + 0x10111 | 0x1c9b0 => base.checked_add(state_offset).ok_or(EINVAL)?, + 0x10119 | 0x1c9b8 => base.checked_add(aux_offset).ok_or(EINVAL)?, + 0x1c079 | 0x1c9d8 if p.layers == 1 => r.heapmeta_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?, + _ => reg.value, + }; + if reg.reg == 0x1c880 { vdm_seen = true; } + } + for reg in &mut self.common.fragment { + let register = reg.reg; + reg.value = match register { + 0x16060 if p.layers == 1 => r.heapmeta_va, + 0x15021 => controls.aux_fb_flags, + 0x15049 => controls.aux_fb_control, + // Main tile-control bits 17..19 carry the empty-tile and + // depth/stencil attachment action flags. The ordinary M5 + // profile has one stored colour attachment. An unstored ZLS + // aspect or a pass that skips empty + // tiles needs its corresponding action bit. Partial phases + // clear these bits independently and retain their own policy. + 0x10039 => { + let skip_empty = !p.process_empty_tiles; + reg.value + | (u64::from(skip_empty && inputs.fragment.empty_tile_flags[3] & 1 == 0) << 17) + | (u64::from(p.depth.va != 0 + && (p.zls_control & (1 << 15) != 0 || p.zls_control & (1 << 19) == 0)) << 18) + | (u64::from(p.stencil.va != 0 + && (p.zls_control & (1 << 14) != 0 || p.zls_control & (1 << 18) == 0)) << 19) + } + _ => reg.value, + }; + } + let mut c = *controls; + c.ta.work_key = super::super::work_key(r.context_id, args.ta.state_word); + c.fragment.work_key = super::super::work_key(r.context_id, args.fragment.state_word); + let ta: &mut [RegisterWrite;82] = (&mut self.base[..82]).try_into().map_err(|_| EINVAL)?; + build_ta_registers_into(p, &c, r.free_list_slot, ta, &self.common.ta)?; + let mut extra = extra; + extra.scs_bracket |= u8::from(c.ta.scs.scratch.enabled()); + tiling_with_scratch_into(ta, extra, r.mesh_storage, &mut self.ta)?; + self.ta.set_lifecycle((u64::from(inputs.ta.predecessor_id) << 32) | u64::from(inputs.ta.completion.key as u32)); + c.fragment_tile_flags = (c.fragment_tile_flags & !0x7ffff) | 0x36011 + | (u64::from(inputs.fragment.empty_tile_flags[3] & 1) << 18); + build_fragment_registers_into(p, &c, r.free_list_slot, &mut self.base, &self.common.fragment)?; + self.fragment_count = 92 + usize::from(c.fragment.scs.scratch.enabled()) * 2; + fragment_main_with_scratch_into(p, &self.base, dispatch, inputs.fragment.empty_tile_flags[3], + c.fragment.scs.scratch.enabled(), &mut self.fragment[..self.fragment_count])?; + for reg in &mut self.fragment { + if matches!(reg.reg, 0x160e0 | 0x1499 | 0xa341) { + reg.value = (u64::from(inputs.fragment.predecessor_id) << 32) | u64::from(inputs.fragment.completion.key as u32); + } + } + partial(&self.common.store, 0x15381, &c, false, &mut self.store)?; + // The ordinary SPM store takes its own empty-tile flag, independent + // of main empty-tile processing and of the SPM load's flag. Stored + // ZLS surfaces still force empty-tile processing in the store. + for register in &mut self.store { + if register.reg == 0x10039 { + let empty = inputs.fragment.empty_tile_flags[1] & 1 != 0 + || p.depth.va != 0 || p.stencil.va != 0; + register.value = (register.value & !(1 << 16)) + | (u64::from(empty) << 16); + } + } + partial(&self.common.resume, 0x15381, &c, false, &mut self.resume)?; + // The load phase uses the primary normal store binding. + // Normal EOT values are in the main array, before any adaptation. + c.store_pipeline_2_bind = self.common.fragment[2].value << 32; + c.store_pipeline_2 = self.common.fragment[3].value; + partial(&self.common.load, 0x15309, &c, true, &mut self.load)?; + // The SPM load phase takes its empty-tile policy from its own flag, + // which the fragment command also carries. It differs from the main + // and partial-store policy for multisample resolve passes. + for register in &mut self.load { + if register.reg == 0x10039 { + register.value = (register.value & !(1 << 16)) + | (u64::from(inputs.fragment.empty_tile_flags[2] & 1) << 16); + } + } + Ok(()) + } +} + +/// Populate command-side copies from the same pass that produces the registers. +/// Queue identity, completion sequencing and resource budgets remain caller-owned. +pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, + i: &mut M5RenderCommandInputs) -> Result { + let p=args.pass;let r=args.resources; + if p.width==0 || p.width>32768 || p.height==0 || p.height>32768 + || p.layers==0 || p.layers>2048 || r.context_id==0 || r.context_id>63 || p.samples_log2>3 + || !matches!((p.utile_width,p.utile_height),(32,32)|(32,16)|(16,16)) { return Err(EINVAL); } + let usc=|v:u32|p.usc_base.checked_add(u64::from(v&!7)).ok_or(EINVAL); + let spec=|v:u64|if p.rsrc_spec_hi {v}else{v&u64::from(u32::MAX)}; + // Check every address before changing command inputs. + let bg=usc(p.bg.offset)?;let pbg=usc(p.partial_bg.offset)?; + let eot=usc(p.eot_offset)?;let peot=usc(p.partial_eot_offset)?; + let deflake=r.deflake_va.checked_sub(0x10_0000_0000) + .and_then(|offset|offset.checked_add(ContextStore::selected(c.board).stream as u64)).ok_or(EINVAL)?; + i.context_id=u32::from(r.context_id); + i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; + // With the G17D forms the tiler asks the host for parameter-buffer blocks when the buffer + // runs out (a growth request, served by the shared growth worker) instead of rendering + // partially. J714 keeps 0. + i.ta.pb_growth_request = u8::from(c.board.g17d_forms()); + let f=&mut i.fragment; + f.empty_tile_flags[0]=u8::from(p.process_empty_tiles); + // The fragment command's pass flag: 1 in every ordinary render pass with the G17D forms. It + // follows the render context, not the attachments, sample count, layers or tile size. J714 + // keeps 0. + f.fragment_pass_flag = u8::from(c.board.g17d_forms()); + // The admitted ordinary single-sample profile has one stored colour + // attachment. Its clear-intent flag follows empty-tile intent; a LOAD + // pass uses zero here, while partial-store ZLS addresses can + // independently force empty tiles in the register program. + if p.samples_log2 == 0 { + f.empty_tile_flags[1] = u8::from(p.process_empty_tiles); + f.empty_tile_flags[2] = 0; + } else { + // Multisample stored-colour CLEAR and LOAD passes both process empty + // tiles in the SPM load. Only CLEAR sets the store flag; the BG + // resource specification publishes that intent in bit 6. An empty + // seed pass has no BG or empty-tile intent. + f.empty_tile_flags[1] = u8::from(p.process_empty_tiles && p.bg.rsrc_spec & (1 << 6) != 0); + f.empty_tile_flags[2] = u8::from(p.process_empty_tiles); + } + f.depth_bias_va=p.depth_bias_va;f.scissor_va=p.scissor_va;f.occlusion_query_va=p.occlusion_query_va; + f.binding_pairs[18]=[spec(p.bg.rsrc_spec),bg]; + f.binding_pairs[21]=[spec(p.partial_bg.rsrc_spec),pbg];f.binding_pairs[22][0]=p.zls_control; + f.depth_dimensions=p.depth_dimensions;f.depth_va=p.depth.va;f.depth_va_pair=[p.depth.va;2]; + f.stencil_va=p.stencil.va;f.stencil_va_pair=[p.stencil.va;2]; + f.depth_stride=p.depth.stride;f.stencil_stride=p.stencil.stride; + f.depth_compression_va=p.depth.compression_va;f.depth_compression_stride=p.depth.compression_stride; + f.stencil_compression_va=p.stencil.compression_va;f.stencil_compression_stride=p.stencil.compression_stride; + f.sample_aux_flags=[isp_sample_control(p),c.aux_fb_flags]; + f.size_aux_control=[u64::from(p.width)|(u64::from(p.height)<<32),c.aux_fb_control]; + f.eot_program=[spec(p.eot_rsrc_spec)<<32,eot]; + f.partial_eot_program=[spec(p.partial_eot_rsrc_spec)<<32,peot]; + f.zs_clear_values=u64::from(p.depth_clear)|(u64::from(p.stencil_clear|0x300)<<32); + f.tilemap_va=r.tilemap_va;f.msaa_control=p.msaa_control;f.sample_count=1u32< Option { + if context >= 64 || generation == 0 || generation >= (1u64 << 58) { + return None; + } + Some((u64::from(context) << 58) | generation) +} + +/// Merge the shifted usage hint into the TA tail's descriptor value. +/// +/// The value is eight bytes. Its first byte belongs to the descriptor; the +/// shifted hint fits only the following seven bytes. The next byte starts the +/// independent resolved usage pointer, so accepting an eighth hint byte would +/// overwrite that pointer. +pub(crate) fn tiling_descriptor_value(descriptor: u64, shifted_hint: u64) -> Option { + if shifted_hint > (u64::MAX >> 8) { + return None; + } + Some((descriptor & 0xff) | (shifted_hint << 8)) +} + +pub(crate) struct RenderTail { + + pub(crate) descriptor_value: u64, + + pub(crate) resolved_value: Option, + + pub(crate) trailing_value: u64, +} + +pub(crate) fn encode(out: &mut [u8], stage: RenderStage, value: &RenderTail) -> Option<()> { + let base = match stage { RenderStage::Tiling => 0x937, RenderStage::Fragment => 0x21d1 }; + let tail = out.get_mut(base..base + 24)?; + tail[..8].copy_from_slice(&value.descriptor_value.to_le_bytes()); + if let Some(resolved) = value.resolved_value { + tail[8..16].copy_from_slice(&resolved.to_le_bytes()); + } + tail[16..24].copy_from_slice(&value.trailing_value.to_le_bytes()); + Some(()) +} + +pub(crate) struct ExtraAddresses { + pub(crate) count: u32, + pub(crate) policy: u8, + pub(crate) addresses: [u64; 4], +} + +pub(crate) fn encode_extra(out: &mut [u8], stage: RenderStage, value: &ExtraAddresses) -> Option<()> { + let (base, end) = match stage { + RenderStage::Tiling => (0x96f, 0x990), + RenderStage::Fragment => (0x220e, 0x2233), + }; + let tail = out.get_mut(base..end)?; + tail[32] &= 0xf2; + if value.count != 0 || value.addresses.iter().any(|&address| address != 0) { + tail[32] |= 1 | ((value.policy & 3) << 2); + for (slot, address) in tail[..32].chunks_exact_mut(8).zip(value.addresses) { + slot.copy_from_slice(&address.to_le_bytes()); + } + } + if let RenderStage::Fragment = stage { tail[33..37].fill(0); } + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs new file mode 100644 index 0000000000000..d03ecbfec77ec --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 render-target suballocation geometry. + + +/// The part's minimum parameter-buffer size in 32 KiB pages. +pub(crate) const fn min_parameter_buffer_pages(clusters: u32) -> u64 { + 32 + 16 * clusters as u64 +} + +pub(crate) fn cluster_control(clusters: u32, pb_pages: u64) -> Option<[u64; 3]> { + let difference = match clusters { + 1 => return Some([1, 0, pb_pages]), + 2 => 4u64, + 4 => 5u64, + 8 => 6u64, + _ => return None, + }; + Some([0x40 | (difference << 8) | (difference << 19), 4, pb_pages]) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TargetSlices { + pub(crate) sizes: [usize; 8], + pub(crate) alignments: [usize; 8], + pub(crate) offsets: [usize; 8], + pub(crate) bytes: usize, +} + +impl TargetSlices { + pub(crate) fn new(tilemap_bytes: usize, tpc_used_bytes: usize, layers: u16, + kind: u32, segments: u32, pb_bytes: usize) -> Option { + if tilemap_bytes == 0 || tpc_used_bytes == 0 || layers == 0 || kind == 0 { + return None; + } + let mut sizes = [0; 8]; + let mut alignments = [0; 8]; + sizes[0] = tpc_used_bytes; + alignments[0] = 0x40; + if kind >= 2 { + if segments == 0 || pb_bytes == 0 { return None; } + let segments = segments as usize; + sizes[1] = tilemap_bytes.checked_mul(segments)?; + // Each segment's layer metadata, single-layer passes included. When the parameter + // buffer runs out and the firmware renders partially, the tiler continues each later + // segment k at this slice's base (named in the TA command) plus 0x100 k, and the + // firmware names that root in the next partial program. A single-layer pass's first segment + // takes its metadata from the target records instead, so the slice is used only once + // a pass splits: left unsized, its base was the compact window's first byte, and the + // segments' metadata landed in whatever lived there. + sizes[2] = 0x100usize.checked_mul(segments)?; + sizes[4] = (kind as usize).checked_mul(0x300)?.checked_add(0x680)?; + sizes[5] = (kind as usize).checked_mul(0x5c0)?; + + let pages = pb_bytes.checked_add(0x7fff)? / 0x8000; + let groups = (pages.checked_add(0x17ff)? / 0x1800).checked_add(1)?; + sizes[6] = groups.checked_mul(8)?.checked_mul(segments)?; + sizes[7] = 0x80; + alignments = [0x40, 0x40, 0x100, 0, 0x40, 0x40, 0x80, 0x40]; + } + let mut bytes = 0usize; + let mut offsets = [0; 8]; + for i in 0..8 { + if alignments[i] != 0 { + bytes = bytes.checked_add(alignments[i] - 1)? & !(alignments[i] - 1); + } + offsets[i] = bytes; + bytes = bytes.checked_add(sizes[i])?; + } + Some(Self { sizes, alignments, offsets, bytes }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 1cb46fce2d602..df69458699b39 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -19,6 +19,9 @@ pub(crate) mod context; pub(crate) mod freelist; pub(crate) mod initdata; pub(crate) mod kick; +// Most M5 encodings are reached only once an M5 board has a page-table boot path. +#[allow(dead_code)] +pub(crate) mod m5; pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 66e7d07db48a5..3de7282c2c020 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -386,8 +386,8 @@ pub(crate) struct TilingBinding { /// Queue whose configuration a [`QueueConfig`] carries. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum QueueConfigTarget { - /// A tiling queue and its parameter-buffer binding. - Tiling(TilingBinding), + /// A tiling queue, optionally carrying a parameter-buffer binding. + Tiling(Option), /// A fragment queue. Fragment, /// A compute queue. @@ -513,7 +513,7 @@ impl QueueConfig { return Err(EINVAL); } let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { - QueueConfigTarget::Tiling(binding) => { + QueueConfigTarget::Tiling(Some(binding)) => { if binding.fragment_qid > QID_MAX { return Err(EINVAL); } @@ -525,6 +525,8 @@ impl QueueConfig { u32::from(binding.fragment_qid), ) } + QueueConfigTarget::Tiling(None) => + (DataMaster::Tiling, 0, u64::MAX, u32::MAX, u32::MAX), QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), }; diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 8fcc25500ab2d..b699c10923afe 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -243,6 +243,15 @@ pub(crate) struct RenderPass { pub(crate) few_primitives: bool, /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. pub(crate) fragment_barriers: bool, + /// Host-only: the tiling kick waits on its own render and compute prefixes + /// (`Order::tiling_wait_through`), the fragment kick on the whole ones. M5 only. + pub(crate) stage_prefixes: bool, + /// Host-only: the render writes whole attachments: its tile stores reach no pixel + /// outside its own render area (hazard ordering). M5 only. + pub(crate) whole_attachments: bool, + /// A graphics program of the render writes external memory (buffer or image stores, + /// atomics); render-target outputs do not count. M5 only. + pub(crate) shader_writes_memory: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. @@ -326,6 +335,10 @@ pub(crate) struct RenderResources { pub(crate) deflake_va: u64, /// GPU address of the tiling status word. pub(crate) ta_status_va: u64, + /// The TA mesh rendering buffer word for mesh/object kicks: the context VA + /// of the queue's mesh rendering buffer in the low 48 bits and its + /// 64 KiB chunk count above. Zero leaves the register out of the program. + pub(crate) mesh_storage: u64, /// GPU address of the fragment status word, as seen by the pass's VM. pub(crate) fragment_status_va: u64, /// Firmware alias of the fragment status word. @@ -469,7 +482,11 @@ struct PassWords { impl PassWords { fn new(pass: &RenderPass) -> Result { - if pass.samples_log2 > SAMPLES_LOG2_MAX { + Self::with_sample_limit(pass, SAMPLES_LOG2_MAX) + } + + fn with_sample_limit(pass: &RenderPass, max_samples_log2: u8) -> Result { + if pass.samples_log2 > max_samples_log2 { return Err(EINVAL); } let samples = 1u64 << pass.samples_log2; @@ -998,6 +1015,97 @@ fn partial_load_registers( ] } +/// The M5 render register storage, shared by its phases. Keep this in the caller's heap +/// workspace when constructing descriptors. +pub(crate) struct M5RegisterPrograms { + pub(crate) ta: [RegisterWrite; TA_REGISTERS], + pub(crate) fragment: [RegisterWrite; FRAGMENT_REGISTERS], + pub(crate) store: [RegisterWrite; PARTIAL_STORE_REGISTERS], + pub(crate) resume: [RegisterWrite; PARTIAL_RESUME_REGISTERS], + pub(crate) load: [RegisterWrite; PARTIAL_LOAD_REGISTERS], +} +// SAFETY: All members contain only integer register writes. +unsafe impl Zeroable for M5RegisterPrograms {} +impl M5RegisterPrograms { + /// M5 partial phases add only their own depth/stencil direction while + /// preserving the pass's existing operations and surface layout bits. + pub(crate) fn write_m5(&mut self, args: &RenderArgs<'_>) -> Result { + // The M5 forms of the shared lowering: the A18 Pro arrays without the SCS enable + // (the M5 adaptation brackets its own scratch and intersection words), the + // one-cluster deflake layout, and up to eight samples. + let none = super::render_rt::RayQueryStage { + intersection: super::compute::IntersectionTables::NONE, + scratch: super::compute::ScratchRequest::new(0)?, + }; + let args = &RenderArgs { + programs: RegisterPrograms::G17P, + deflake: Deflake::ONE_CLUSTER, + ray_query: [none; 2], + ..*args + }; + let g = Geometry::new(args.pass)?; + let w = PassWords::with_sample_limit(args.pass, 3)?; + without_scs_enable(&mut self.ta, &ta_registers(args, &g, &w)?, TA_SCS_ENABLE_INDEX)?; + without_scs_enable( + &mut self.fragment, + &fragment_registers(args, &g, &w), + FRAGMENT_SCS_ENABLE_INDEX, + )?; + self.store = partial_store_registers(args.pass, &w); + self.resume = partial_resume_registers(args.pass, &w); + self.load = partial_load_registers(args.pass, &w); + // The partial phases start from the pass's ZLS word with both directions of each + // present surface, as the M5 lowering always has. + let mut partial = args.pass.zls_control; + if args.pass.depth.va != 0 { + partial |= ZLS_LOAD_DEPTH | ZLS_STORE_DEPTH; + } + if args.pass.stencil.va != 0 { + partial |= ZLS_LOAD_STENCIL | ZLS_STORE_STENCIL; + } + for array in [&mut self.store[..], &mut self.resume[..], &mut self.load[..]] { + for register in array { + if register.reg == 0x15319 { + register.value = partial; + } + } + } + const LOAD: u64 = (1 << 14) | (1 << 15); + const STORE: u64 = (1 << 18) | (1 << 19); + for register in &mut self.store { + if register.reg == 0x15319 { + register.value = (register.value & !LOAD) | (args.pass.zls_control & LOAD); + } else if register.reg == 0x10039 + && (args.pass.depth.va != 0 || args.pass.stencil.va != 0) + { + // A partial store forces empty tiles when either stored ZLS + // address (depth or stencil) exists. + register.value |= TILE_CONFIG_PROCESS_EMPTY_TILES; + } + } + for register in &mut self.load { + if register.reg == 0x15319 { + register.value = (register.value & !STORE) | (args.pass.zls_control & STORE); + } + } + Ok(()) + } +} + +/// Copies a lowered array without its SCS enable entry. +fn without_scs_enable( + out: &mut [RegisterWrite; N], + lowered: &[RegisterWrite; M], + index: usize, +) -> Result { + if M != N + 1 || lowered.get(index).is_none_or(|write| write.reg != SCS_ENABLE) { + return Err(EINVAL); + } + out[..index].copy_from_slice(&lowered[..index]); + out[index..].copy_from_slice(&lowered[index + 1..]); + Ok(()) +} + /// Tiling (TA) work descriptor (tag 0). #[repr(C, packed)] #[derive(Debug)] diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs index e1e6a10949fb4..e8349d0fcd0e2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -113,7 +113,8 @@ impl Support { } } -/// The viewport words consumed from the deflake page at +0x900. +/// Viewport words shared by the tiling engines; their location and control +/// value are selected by the firmware profile. #[repr(C)] #[derive(Copy, Clone)] pub(crate) struct Viewport { @@ -129,6 +130,11 @@ pub(crate) struct Viewport { scale_z: u32, } impl Viewport { + pub(crate) fn with_control(mut self, control: u32) -> Self { + self.unk_0 = control; + self + } + /// Refresh only defined words; firmware-owned gaps retain their contents. pub(crate) fn apply(&self, destination: &mut Self) { destination.unk_0 = self.unk_0; diff --git a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs index 5ae23d84400eb..666209690efde 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs @@ -33,10 +33,12 @@ impl RayQueryStage { } } -pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2]) -> Result { +/// The render pool of an owner is shared by its `pairs` render pairs, each with +/// its own compute-pool extras. +pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2], pairs: u8) -> Result { let extra = compute_pool::extra_bytes(active_cores, pages[0])? .checked_add(compute_pool::extra_bytes(active_cores, pages[1])?) - .and_then(|bytes| bytes.checked_mul(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u64)) + .and_then(|bytes| bytes.checked_mul(u64::from(pairs))) .ok_or(EOVERFLOW)?; let bytes = extra.checked_add((RENDER_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) .ok_or(EOVERFLOW)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index c5287e66a641d..95a052fc5f5d9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -64,6 +64,9 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Renders may carry ray-query stages. The M5 command set has none: it encodes no render + /// ray-query state. + pub(crate) render_ray_query: bool, /// Tiling descriptors report whether the fragment queue still had earlier kicks /// outstanding (`fragment_busy`). pub(crate) fragment_busy: bool, @@ -77,6 +80,13 @@ pub(crate) struct Variant { pub(crate) compute: super::compute::ComputeProgram, pub(crate) opening_record: bool, pub(crate) split_firmware_area: bool, + /// Command validation applies the M5 command set: resource commands, the M5 render and + /// compute flags, 8x MSAA and the M5 tile-buffer limit. + pub(crate) m5_commands: bool, + /// The host holds the cores powered around a render pair's registration and its first + /// work. The bare-metal M5 firmware powers the cores for work by its own policy: under a + /// hold, the M5 Ultra's core-power witness reads unpowered until the hold is released. + pub(crate) registration_power_hold: bool, } /// A firmware build a SoC supports, identified by its loaded TEXT. @@ -114,6 +124,7 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + render_ray_query: true, fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, @@ -121,6 +132,8 @@ pub(crate) const V26A428: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, + m5_commands: false, + registration_power_hold: true, }; /// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). @@ -143,6 +156,7 @@ pub(crate) const V25G83: Variant = Variant { // A fragment stage with the few-primitives flag hangs. few_primitives: false, ray_traversal: true, + render_ray_query: true, // When a tiling pass runs out of memory with this set, the firmware resumes it without // adding memory and the tiler stops silently; macOS always writes 0. fragment_busy: false, @@ -152,6 +166,17 @@ pub(crate) const V25G83: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, + m5_commands: false, + registration_power_hold: true, +}; + +/// The 26A428 firmware of the M5 GPUs, which runs the M5 protocol (the M5 graph, queues and +/// command set). +pub(crate) const V26A428_M5: Variant = Variant { + m5_commands: true, + render_ray_query: false, + registration_power_hold: false, + ..V26A428 }; pub(crate) const V25F84: Variant = Variant { diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 4d9e99d9a9cc0..071b0a15e3111 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -4,20 +4,24 @@ //! Tile stores can overlap even when the application's pixel regions do not. //! Keep independent attachments concurrent; order only conflicting live spans. //! -//! Within one VM, a render of another logical owner is ordered at publication: -//! the device mutex totally orders publications, and the later render's -//! fragment kick waits in firmware for the newest overlapping fragment kick of -//! every other pair. Admission never waits for such an overlap. Only a +//! Within one VM, a render of another logical owner, or of the same owner on +//! its other pair, is ordered at publication: the device mutex totally orders +//! publications, and the later render's fragment kick waits in firmware for the +//! newest overlapping fragment kick of every other pair. Admission never waits +//! for such an overlap. Only a //! cross-VM overlap through a shared object waits on the host for retirement, //! so that one VM's fault can never hold another VM's kick in firmware. use super::job::Order; +use kernel::prelude::kunit_tests; const MAX_SPANS: usize = 24; pub(crate) const WAIT_MS: u32 = 1000; /// Firmware barriers one render publication may add for same-VM overlaps, /// one per other pair. pub(crate) const PUBLICATION_BARRIERS: usize = 7; +/// Render pairs one logical owner can hold. +pub(crate) const MAX_OWNER_PAIRS: usize = crate::hw::t8140::queues::RENDER_PAIRS_MAX; #[derive(Clone, Copy)] struct Span { @@ -32,10 +36,13 @@ pub(crate) struct Footprint { spans: [Span; MAX_SPANS], count: usize, shared: bool, + /// The render writes whole attachments: its tile stores reach no pixel + /// outside its own render area. + whole: bool, } impl Footprint { - pub(crate) fn new(vm: u64) -> Self { + pub(crate) fn new(vm: u64, whole: bool) -> Self { Self { vm, spans: [Span { @@ -45,6 +52,7 @@ impl Footprint { }; MAX_SPANS], count: 0, shared: false, + whole, } } @@ -75,6 +83,12 @@ impl Footprint { if self.vm != other.vm && !(self.shared && other.shared) { return false; } + // Tile-granular stores of two whole-attachment renders write only + // pixels both renders own; unordered, they race as the application + // asked. + if self.whole && other.whole { + return false; + } self.spans[..self.count].iter().any(|a| { other.spans[..other.count] .iter() @@ -86,7 +100,11 @@ impl Footprint { #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Plan { Any, + /// Only this pair of the owner's. Pair(u8), + /// This pair of the owner's, else another of its pairs: the publication + /// barrier orders the overlap from either. + Join(u8), Wait, } @@ -108,6 +126,20 @@ pub(crate) struct Live<'a> { /// than `PUBLICATION_BARRIERS + 1` other pairs of its VM hold admitted renders: /// of any set of renders overlapping one publication, the last admitted saw all /// the others live. +/// +/// An owner's own uncovered overlaps on one of its pairs prefer that pair: the +/// pair's queues run its commands in publication order. On any other pair of +/// the owner's (it holds at most [`MAX_OWNER_PAIRS`]) the render's fragment +/// kick waits in firmware for that pair's newest overlapping fragment kick, as +/// for another owner's render ([`Pool::publication_barriers`] names it); tile +/// loads and stores happen only in fragment work. So with overlaps on two of +/// its pairs the render prefers the pair of the newer overlap, and a full +/// preferred pair moves the render to the owner's other pair instead of waiting. +/// That barrier is one more than the foreign ones: without a slot for it, a +/// render with overlaps on one pair stays on that pair, and one with overlaps +/// on two waits as a crowded publication does. +/// +/// [`Pool::publication_barriers`]: super::queue::admission::Pool::publication_barriers pub(crate) fn plan<'a>( owner: u64, prefix: Option, @@ -115,7 +147,9 @@ pub(crate) fn plan<'a>( footprint: &Footprint, live: impl IntoIterator>, ) -> Plan { - let mut selected = None; + // The owner's uncovered overlaps: (pair, newest sequence) for each of at + // most two of its pairs. + let mut own: [Option<(u8, u64)>; MAX_OWNER_PAIRS] = [None; MAX_OWNER_PAIRS]; let mut ordered = false; let mut pairs = [0u8; PUBLICATION_BARRIERS]; let mut count = 0; @@ -151,14 +185,93 @@ pub(crate) fn plan<'a>( if Order::contains(prefix, entry.sequence) { continue; } - match selected { - None => selected = Some(slot), - Some(prior) if prior == slot => {} - Some(_) => return Plan::Wait, + match own.iter_mut().find(|known| known.is_none_or(|(pair, _)| pair == slot)) { + Some(known) => { + let newest = known.map_or(entry.sequence, |(_, sequence)| sequence.max(entry.sequence)); + *known = Some((slot, newest)); + } + // More pairs than an owner can hold: not reachable. + None => return Plan::Wait, } } - if ordered && crowded { + // Foreign barriers come only with a foreign overlap, at most one per + // counted pair: the own cross-pair barrier needs a slot beside them. + let room = !(ordered && count == PUBLICATION_BARRIERS); + if ordered && crowded || own[1].is_some() && !room { return Plan::Wait; } - selected.map_or(Plan::Any, Plan::Pair) + match own { + [None, _] => Plan::Any, + [Some((slot, _)), None] if room => Plan::Join(slot), + [Some((slot, _)), None] => Plan::Pair(slot), + [Some(a), Some(b)] => Plan::Join(if a.1 >= b.1 { a.0 } else { b.0 }), + } +} + +#[kunit_tests(asahi_g17_hazard_plan)] +mod plan_tests { + use super::*; + + fn footprint(vm: u64, start: u64) -> Footprint { + let mut footprint = Footprint::new(vm, false); + footprint.push_vm(start, start + 0x1000); + footprint + } + + fn live(owner: u64, sequence: u64, slot: u8, footprint: &Footprint) -> Live<'_> { + Live { owner, sequence, slot: Some(slot), ticket: 0, footprint, quarantined: false } + } + + #[test] + fn an_own_overlap_prefers_its_pair() { + let mine = footprint(1, 0); + let earlier = footprint(1, 0x800); + assert!(plan(7, None, None, &mine, [live(7, 3, 4, &earlier)]) == Plan::Join(4)); + // A covered predecessor needs nothing. + assert!(plan(7, Some(3), None, &mine, [live(7, 3, 4, &earlier)]) == Plan::Any); + // Disjoint attachments stay free. + let other = footprint(1, 0x10000); + assert!(plan(7, None, None, &mine, [live(7, 3, 4, &other)]) == Plan::Any); + } + + #[test] + fn overlaps_on_two_own_pairs_prefer_the_newer_and_never_wait() { + let mine = footprint(1, 0); + let a = footprint(1, 0x800); + let b = footprint(1, 0x400); + assert!(plan(7, None, None, &mine, [live(7, 5, 2, &a), live(7, 9, 3, &b)]) == Plan::Join(3)); + assert!(plan(7, None, None, &mine, [live(7, 9, 2, &a), live(7, 5, 3, &b)]) == Plan::Join(2)); + assert!(plan(7, Some(5), None, &mine, [live(7, 5, 2, &a), live(7, 9, 3, &b)]) == Plan::Join(3)); + // A foreign same-VM overlap beside them is a barrier too. + assert!( + plan(7, None, None, &mine, [live(7, 5, 2, &a), live(7, 9, 3, &b), live(8, 1, 6, &b)]) + == Plan::Join(3) + ); + } + + #[test] + fn the_cross_pair_barrier_keeps_a_slot() { + let mine = footprint(1, 0); + let near = footprint(1, 0x800); + let far = footprint(1, 0x10000); + // Another owner of the VM holds `foreign` pairs, one with an overlap, + // then the owner's own overlaps on pair 2 and, if `across`, pair 3. + fn entries<'a>( + near: &'a Footprint, + far: &'a Footprint, + foreign: usize, + across: bool, + ) -> impl Iterator> + 'a { + (0..foreign) + .map(move |index| live(9, 1, 20 + index as u8, if index == 0 { near } else { far })) + .chain(core::iter::once(live(7, 5, 2, near))) + .chain(across.then(|| live(7, 6, 3, near))) + } + let full = PUBLICATION_BARRIERS; + assert!(plan(7, None, None, &mine, entries(&near, &far, full - 1, false)) == Plan::Join(2)); + assert!(plan(7, None, None, &mine, entries(&near, &far, full - 1, true)) == Plan::Join(3)); + // No slot left: one pair's overlap stays on that pair; two pairs' wait. + assert!(plan(7, None, None, &mine, entries(&near, &far, full, false)) == Plan::Pair(2)); + assert!(plan(7, None, None, &mine, entries(&near, &far, full, true)) == Plan::Wait); + } } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 56e7335a23bd2..08d72402d4115 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -12,11 +12,15 @@ use super::{ }; use crate::{driver::AsahiDrmDevice, hw::t8140, mmu}; use core::ops::Range; -use core::sync::atomic::{fence, AtomicU32, Ordering}; -use kernel::{c_str, prelude::*}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::{c_str, prelude::*, sync::Arc}; use t8140::{kernel_window as high, low}; mod v25g83; +mod m5; +mod work_pair; +pub(crate) mod outer_credit; +pub(crate) use outer_credit::{OuterClaim, OuterScope}; const EVENT_RING_RECHECK: u32 = 256; @@ -135,6 +139,7 @@ impl ComputeScheduler { /// Checked views used only within one publication call. Every fallible operation completes /// before either member of a paired render publication is copied into shared memory. struct WorkPublication<'a> { + published: Option<&'a AtomicU64>, record: &'a super::fw::channels::WorkSlot, destination: *mut u8, producer: &'a AtomicU32, @@ -153,10 +158,21 @@ impl WorkPublication<'_> { ); } } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + if let Some(published) = self.published { published.fetch_add(1, Ordering::Release); } + fence(Ordering::SeqCst); + self.next + } } /// Session-long storage and mappings. Aliases are released before their backing objects. pub(super) struct InitData { + m5: bool, + outer_credits: Option>, // Live publication uses interior writes under the device mutex. This owner is Send but not // Sync, so safe host code cannot issue those writes concurrently through shared references. _single_publisher: core::marker::PhantomData>, @@ -169,9 +185,12 @@ pub(super) struct InitData { qos: usize, pb: usize, page_pool: usize, - compute_global: usize, - boot_control: usize, - boot_state: usize, + /// The G17P compute objects: the global state page and the boot free + /// list's control and state. M5 has none of them, and its build leaves + /// these `None`, so an unset handle can never name another object. + compute_global: Option, + boot_control: Option, + boot_state: Option, metrics_fw_va: u64, /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). pm_metrics_va: u64, @@ -186,10 +205,12 @@ pub(super) struct InitData { compute_scheduler: Option, compute_active: core::cell::Cell, event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], - completion_rings: [usize; 2], + /// Completion-ring objects; a family sets only the rings it has. + completion_rings: [Option; 4], stamps: [usize; 2], - render_pages: usize, - render_runs: usize, + /// The G17P render page and run lists (client aliases). M5 has none. + render_pages: Option, + render_runs: Option, render_blocks: core::ops::Range, } @@ -262,6 +283,8 @@ impl InitData { /// Empty construction state, retained even if graph construction fails. pub(super) fn new() -> Self { Self { + m5: false, + outer_credits: None, _single_publisher: core::marker::PhantomData, aliases: KVec::new(), objects: KVec::new(), @@ -272,9 +295,9 @@ impl InitData { qos: 0, pb: 0, page_pool: 0, - compute_global: 0, - boot_control: 0, - boot_state: 0, + compute_global: None, + boot_control: None, + boot_state: None, metrics_fw_va: 0, pm_metrics_va: 0, recovery_slot_keys: None, @@ -285,10 +308,10 @@ impl InitData { compute_scheduler: None, compute_active: core::cell::Cell::new(false), event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], - completion_rings: [0; 2], + completion_rings: [None; 4], stamps: [0; 2], - render_pages: 0, - render_runs: 0, + render_pages: None, + render_runs: None, render_blocks: 0..0, } } @@ -304,6 +327,18 @@ impl InitData { Ok(self.metrics_fw_va) } + /// Names the M5 renders' metrics object to the firmware through a kernel-window alias. + pub(super) fn bind_m5_metrics(&mut self, uat: &mmu::Uat, metrics: &KernelObject) -> Result { + if !self.m5 || self.metrics_fw_va != 0 { + return Err(EINVAL); + } + let alias = metrics.alias_in(uat.kernel_vm(), uat.kernel_dynamic_range(), + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + self.metrics_fw_va = alias.iova(); + self.aliases.push(alias, GFP_KERNEL)?; + Ok(()) + } + pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { self.object(self.pb) } @@ -335,16 +370,25 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks( - &self, - slot: u16, - page_list_va: u64, - blocks: usize, - ) -> Result { + /// Called for a fresh row or after its free-list release was consumed. The row has the + /// family's format: a G17P block row, or the M5 pool's descriptor. + pub(super) fn install_compute_pool(&self, descriptor: super::freelist::PoolDescriptor) -> Result { + use super::freelist::PoolDescriptor; + let slot = match descriptor { + PoolDescriptor::Neo { slot, .. } | PoolDescriptor::M5 { slot, .. } => slot, + }; if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } + match descriptor { + PoolDescriptor::Neo { slot, page_list, blocks } if !self.m5 => + self.initialize_compute_pool_blocks(slot, page_list, blocks), + PoolDescriptor::M5 { slot, words } if self.m5 => self.set_m5_uma(slot, words), + _ => Err(EINVAL), + } + } + + fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { let entry = abi::PagePoolDescriptor::compute_blocks(page_list_va, blocks); let size = core::mem::size_of::(); let offset = usize::from(slot) * size; @@ -367,6 +411,53 @@ impl InitData { Ok(()) } + /// One M5 page-pool row: the pool's eight descriptor words at its 64-byte slot. + fn set_m5_uma(&self, slot: u16, words: [u64; 8]) -> Result { + let object = self.object(self.page_pool)?; + let offset = usize::from(slot) * core::mem::size_of::<[u64; 8]>(); + // Check the full row before the first store. + object.pointer(offset, core::mem::size_of::<[u64; 8]>())?; + for (index, word) in words.into_iter().enumerate() { + object.dword(offset + index * 8)?.store(word, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Whether a stopped session can prove its GPU engines retired, so that its pages may be + /// freed. M5 has no engine-retirement witness yet. + pub(super) fn has_engine_retirement_witness(&self) -> bool { + !self.m5 + } + + /// The statistics record tags the family's firmware emits. + pub(super) fn statistics_types(&self) -> u64 { + if self.m5 { + super::fw::m5::STATISTICS_TYPES + } else { + u64::from(super::fw::channels::STATISTICS_TYPES) + } + } + + /// A scheduler-state release record in the family's opcode. + pub(super) fn scheduler_release(&self, page_va: u64, id: [u8; 4]) -> super::fw::channels::SchedulerStateRelease { + let mut record = super::fw::channels::SchedulerStateRelease::new(page_va, id); + if self.m5 { + record.opcode = super::fw::m5::queue::SCHEDULER_RELEASE_OPCODE; + } + record + } + + /// Whether a control record of `opcode` holds the GPU powered until the firmware drains + /// it, by the family's rule. + pub(super) fn control_asserts_gpu_power(&self, opcode: super::fw::channels::DeviceControlOp) -> bool { + if self.m5 { + super::fw::m5::power::asserts_gpu_power(opcode as u32) + } else { + opcode.asserts_gpu_power() + } + } + pub(super) fn qos(&self) -> Result<&KernelObject> { self.object(self.qos) } @@ -395,7 +486,9 @@ impl InitData { if self.compute_shared.is_some() { return Ok(()); } - let state_va = self.object(self.boot_state)?.gpu_va(); + let boot_control = self.boot_control.ok_or(ENODEV)?; + let boot_state = self.boot_state.ok_or(ENODEV)?; + let state_va = self.object(boot_state)?.gpu_va(); let mut control = abi::FreeListControl::new(&abi::FreeListArgs { buffer_slot: u32::MAX, class: abi::FreeListClass::Two, @@ -405,9 +498,9 @@ impl InitData { state_va, }); control.run_cursor = COMPUTE_CONTROL_CURSOR; - self.object_mut(self.boot_control)?.bytes_mut().fill(0); - self.object_mut(self.boot_control)?.write(0, control)?; - self.object_mut(self.boot_state)?.bytes_mut().fill(0); + self.object_mut(boot_control)?.bytes_mut().fill(0); + self.object_mut(boot_control)?.write(0, control)?; + self.object_mut(boot_state)?.bytes_mut().fill(0); shared .pages .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { @@ -427,10 +520,14 @@ impl InitData { Ok(()) } + /// Whether `vm` has the shared compute aperture mapped. M5 has no shared + /// aperture: its compute queues need only their own pools, so every VM is ready. pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { - self.compute_shared - .as_ref() - .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + self.m5 + || self + .compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) } pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { @@ -460,7 +557,7 @@ impl InitData { } for slot in 0..2 { mappings.push( - self.object(self.completion_rings[slot])?.map_alias( + self.object(self.completion_rings[slot].ok_or(EINVAL)?)?.map_alias( vm, low::COMPLETION_RINGS[slot], mmu::PROT_GPU_SHARED_RW, @@ -479,7 +576,7 @@ impl InitData { low::PAGE_POOL_DESCRIPTORS, self.object(self.page_pool)?.gpu_va(), ]; - let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + let base = self.main_config_offset(Role::Primary) + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); let bundle = self.object(self.bundle)?; let mut changed = false; @@ -500,7 +597,7 @@ impl InitData { return Ok(()); } let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); - let object = self.object(self.compute_global)?; + let object = self.object(self.compute_global.ok_or(ENODEV)?)?; let state = object .pointer(0, core::mem::size_of::())? .cast::(); @@ -526,15 +623,69 @@ impl InitData { Ok(()) } + /// The domain outer-ring waiters sleep on: unposted claims returned, or firmware + /// consumption seen by `poll_outer_progress`. Register only after dropping the device + /// mutex; a successful claim allocates no waiter fence. ENODEV off M5. + pub(crate) fn outer_available(&self) -> Result> { + Ok(self.outer_credits.as_ref().ok_or(ENODEV)?.available()) + } + + /// M5 TA, fragment and compute share the compute outer ring per priority. + /// The physical queue claim prevents reassignment while this credit lives. + pub(crate) fn reserve_outer(&self, priority: u8, count: u32) -> Result { + if !self.m5 { return Err(EINVAL); } + let cursors = self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?; + self.outer_credits.as_ref().ok_or(EIO)?.reserve(priority, count, cursors) + } + + /// Hold the device mutex for this scope, including its Drop. The result + /// borrows only the claim, allowing the caller to publish through Firmware. + pub(crate) fn outer_scope<'a>(&self, claim: &'a mut OuterClaim) -> Result> { + self.outer_credits.as_ref().ok_or(EINVAL)?.scope(claim) + } + + pub(crate) fn outer_waiting(&self) -> bool { + self.outer_credits.as_ref().is_some_and(|credits| credits.waiting()) + } + + /// Return a notification for off-lock dispatch only on consumer progress. + /// Call while actual outer waiters exist; firmware may consume a config + /// without generating a GPU completion IRQ, so those waiters need polling. + pub(crate) fn poll_outer_progress(&self) -> Result>> { + let Some(credits) = self.outer_credits.as_ref().filter(|credits| credits.waiting()) else { + return Ok(None); + }; + let mut progressed = false; + for priority in 0..4 { + progressed |= credits.progressed(priority, + self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?)?; + } + Ok(progressed.then(|| credits.available())) + } + + fn outer_capacity(&self, priority: u8, cursors: [u32; 3]) -> Result { + match &self.outer_credits { + Some(credits) => credits.capacity(priority, cursors), + None => outer_credit::free(cursors), + } + } + /// The sole host publisher holds the device mutex through the following /// inner and outer publication. Firmware consumers can only free capacity; /// neither this check nor waiting for room reserves or limits other queues. - /// `priority` selects the engine's ring of that priority class. + /// `priority` selects the engine's ring of that priority class. On M5, tiling, + /// fragment and compute share the compute ring, whose unclaimed credits this checks. pub(super) fn work_ready( &self, priority: u8, engine: super::fw::queue::DataMaster, ) -> Result { + use super::fw::queue::DataMaster; + if self.m5 { + let needed = if engine == DataMaster::Compute { 1 } else { 2 }; + return Ok(self.outer_capacity(priority, + self.work_cursors(priority, DataMaster::Compute)?)? >= needed); + } let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; Ok(next != consumer && next != consumer_b) @@ -549,17 +700,23 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - // Compute publications reserve their slots through the port. + // M5 compute is a shared-ring submission on the compute ring, published here + // under the device mutex; the classic compute path reserves through the port. + 2 if self.m5 => DataMaster::Compute, _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; + if self.m5 && engine == DataMaster::Compute + && self.outer_capacity(priority, [consumer, consumer_b, producer])? < 1 { + return Err(EBUSY); + } let next = (producer + 1) % channels::WORK_RING_SLOTS; if next == consumer || next == consumer_b { return Err(EINVAL); } let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), + self.work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( @@ -567,7 +724,11 @@ impl InitData { + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; + let published = if self.m5 && engine == DataMaster::Compute { + self.outer_credits.as_ref().map(|credits| credits.counter(priority)).transpose()? + } else { None }; Ok(WorkPublication { + published, record, destination, producer, @@ -602,6 +763,53 @@ impl InitData { Ok(values) } + /// Publishes an M5 compute record on the shared compute ring under the device mutex. + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) + } + + /// Reserve both common-ring slots before touching either host port. The + /// returned views retain the graph borrow until the pair is published. + pub(super) fn m5_work_pair( + &self, + priority: u8, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + if !self.m5 + || !matches!(fragment.data_master, 1 | 2) + || !matches!(tiling.data_master, 0 | 2) + { + return Err(EINVAL); + } + let cursors = self.work_cursors(priority, DataMaster::Compute)?; + if self.outer_capacity(priority, cursors)? < 2 { return Err(EBUSY); } + let window = work_pair::Window::new(cursors)?; + let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let bundle = self.object(self.bundle)?; + let destinations = [window.start, window.next[0]].map(|index| { + bundle.pointer(self.work_ring_offset(channel) + + index as usize * size_of::(), size_of::()) + }); + let [fragment_destination, tiling_destination] = destinations; + let destinations = [fragment_destination?, tiling_destination?]; + let producer = self.object(self.cluster)?.word(self.state_offset(Role::Primary) + + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER)?; + let copy = |slot: &channels::WorkSlot| channels::WorkSlot::new( + DataMaster::Compute, slot.queue_va, slot.qid, slot.items_end, slot.first != 0, + ); + let records = [copy(fragment)?, copy(tiling)?]; + // SAFETY: Complete, distinct slots were checked in the retained bundle; + // the device's single publisher holds its mutex across this borrow. + let published = self.outer_credits.as_ref().map(|credits| credits.counter(priority)).transpose()?; + Ok(unsafe { work_pair::Publication::new(records, destinations, producer, window, published) }) + } + /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -620,6 +828,13 @@ impl InitData { { return Err(EINVAL); } + if self.m5 { + let pair = self.m5_work_pair(priority, fragment, tiling)?; + pair.prepare(); + commit_accounting()?; + publish_late_tiling(); + return Ok(if deferred { pair.next() } else { pair.publish() }); + } let fragment = self.work_publication(priority, fragment)?; let tiling = self.work_publication(priority, tiling)?; fragment.copy_record(); @@ -662,10 +877,11 @@ impl InitData { return Err(EBUSY); } let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), + self.work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), core::mem::size_of_val(record), )?; Ok(WorkPublication { + published: None, record, destination, producer, @@ -683,6 +899,12 @@ impl InitData { next: [u32; 2], ) -> Result { use super::fw::queue::DataMaster; + if self.m5 { + let pair = self.m5_work_pair(priority, fragment, tiling)?; + if pair.next() != next { return Err(EBUSY); } + pair.publish(); + return Ok(()); + } if fragment.data_master != DataMaster::Fragment as u32 || tiling.data_master != DataMaster::Tiling as u32 { @@ -716,7 +938,7 @@ impl InitData { |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; Ok(super::port::RingView { slots: self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel), + self.work_ring_offset(channel), channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), )?, consumers: [ @@ -742,9 +964,8 @@ impl InitData { qos: self .qos()? .pointer(0, core::mem::size_of::())?, - recovery: self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + // The word recovery_state() reads: the status block moves with the firmware build. + recovery: self.object(self.cluster)?.word(self.recovery_state_offset())?, mailbox, }) } @@ -763,7 +984,7 @@ impl InitData { seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { - let object = self.object(self.completion_rings[0])?; + let object = self.object(self.completion_rings[0].ok_or(EINVAL)?)?; if seen.len() != COMPUTE_COMPLETION_SLOTS { return Err(EINVAL); } @@ -789,7 +1010,7 @@ impl InitData { seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { - let object = self.object(self.completion_rings[0])?; + let object = self.object(self.completion_rings[0].ok_or(EINVAL)?)?; if seen.len() != COMPUTE_COMPLETION_SLOTS { return Err(EINVAL); } @@ -849,7 +1070,7 @@ impl InitData { active_cores: u32, variant: &super::fw::Variant, ) -> Result { - self.pm_metrics_va = cfg.pm_metrics.va; + self.pm_metrics_va = cfg.pm_metrics()?.va; self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { @@ -908,7 +1129,7 @@ impl InitData { }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; - self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + self.compute_global = Some(self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?); let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? .initialize::(0, |table| { @@ -984,8 +1205,8 @@ impl InitData { let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; self.object_mut(state)?.write(0, initial)?; if class == abi::FreeListClass::Two { - self.boot_control = control; - self.boot_state = state; + self.boot_control = Some(control); + self.boot_state = Some(state); } } self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; @@ -1025,8 +1246,8 @@ impl InitData { WB, )?)?; } - self.render_pages = pages; - self.render_runs = runs; + self.render_pages = Some(pages); + self.render_runs = Some(runs); self.render_blocks = first_block..self.objects.len(); self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; @@ -1126,7 +1347,7 @@ impl InitData { for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; - self.completion_rings[slot] = ring; + self.completion_rings[slot] = Some(ring); self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], @@ -1137,7 +1358,7 @@ impl InitData { self.qos = qos; fence(Ordering::SeqCst); let metrics_alias = - metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics()?.size as u64, GFS)?; self.metrics_fw_va = metrics_alias.iova(); self.aliases.push(metrics_alias, GFP_KERNEL)?; fence(Ordering::SeqCst); @@ -1178,7 +1399,7 @@ impl InitData { windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: cfg.perf, + perf: cfg.perf()?, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1186,7 +1407,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(cfg.region_c()?); Ok(()) })?; self.object_mut(self.cluster)? @@ -1308,7 +1529,7 @@ impl InitData { ) -> Result> { let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; mappings.push( - self.object_mut(self.render_pages)?.map_alias( + self.object_mut(self.render_pages.ok_or(EINVAL)?)?.map_alias( vm, low::RENDER_PAGE_LIST, mmu::PROT_GPU_SHARED_RW, @@ -1316,7 +1537,7 @@ impl InitData { GFP_KERNEL, )?; mappings.push( - self.object_mut(self.render_runs)?.map_alias( + self.object_mut(self.render_runs.ok_or(EINVAL)?)?.map_alias( vm, low::RENDER_RUN_LIST, mmu::PROT_GPU_SHARED_RW, @@ -1340,12 +1561,28 @@ impl InitData { Ok(mappings) } + pub(super) fn is_m5(&self) -> bool { + self.m5 + } + + /// The effort value the firmware is given for a requested `effort`. M5 publishes none while + /// work streams, except while a host thread waits for it (`waited`): then the full effort, + /// which the firmware takes as a floor for its performance state, so a submit-and-wait loop + /// runs at the top state. + pub(super) fn submission_effort(&self, effort: u32, waited: bool) -> u32 { + if self.m5 && !waited { 0 } + else { effort } + } + /// Writes the effort input in firmware order, then cleans its cache line to coherency. pub(super) fn publish_effort(&self, effort: u32) -> Result { let region = self.object(self.region_c)?; - let override_word = region.word(abi::RegionC::UNK_E8)?; - let effort_word = region.word(abi::RegionC::EFFORT)?; - let line = region.pointer(abi::RegionC::EFFORT, 4)?; + let (effort_offset, override_offset) = if self.m5 { + (super::fw::m5::power::EFFORT, super::fw::m5::power::EFFORT_OVERRIDE) + } else { (abi::RegionC::EFFORT, abi::RegionC::UNK_E8) }; + let override_word = region.word(override_offset)?; + let effort_word = region.word(effort_offset)?; + let line = region.pointer(effort_offset, 4)?; override_word.store(0, Ordering::Relaxed); effort_word.store(effort, Ordering::Relaxed); // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache @@ -1360,7 +1597,7 @@ impl InitData { pub(super) fn effort(&self) -> Result { Ok(self .object(self.region_c)? - .word(abi::RegionC::EFFORT)? + .word(if self.m5 { super::fw::m5::power::EFFORT } else { abi::RegionC::EFFORT })? .load(Ordering::Relaxed)) } @@ -1417,7 +1654,18 @@ impl InitData { Ok(opps) } + fn main_config_offset(&self, role: Role) -> usize { + if self.m5 { super::fw::m5::initdata::M5_MAIN_BUNDLE_OFFSETS[role as usize] } + else { abi::bundle::MAIN_CONFIG[role as usize] } + } + + fn work_ring_offset(&self, channel: usize) -> usize { + if self.m5 { super::fw::m5::initdata::M5_WORK_RING_BUNDLE_OFFSETS[channel] } + else { abi::work_ring_offset(channel) } + } + fn state_offset(&self, role: Role) -> usize { + if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATE_GRIDS[role as usize]; } match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, @@ -1425,6 +1673,7 @@ impl InitData { } fn status_offset(&self, role: Role) -> usize { + if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATUS_A[role as usize]; } match role { Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, @@ -1445,7 +1694,7 @@ impl InitData { cache.set((checked.0, checked.1, checked.2 - 1)); return Ok(()); } - let offset = abi::bundle::MAIN_CONFIG[role as usize] + let offset = self.main_config_offset(role) + core::mem::offset_of!(abi::MainConfig, channels) + abi::EVENT_CHANNEL * size_of::(); let bundle = self.object(self.bundle)?; @@ -1495,7 +1744,7 @@ impl InitData { abi::ChannelState::PRODUCER, ]) { *counter = state - .word(abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::Relaxed); } Ok(counters) @@ -1503,12 +1752,12 @@ impl InitData { pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) } pub(super) fn control_producer(&self) -> Result<&AtomicU32> { self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) } /// Publishes a control record under the device lock. All checked views are acquired before @@ -1520,13 +1769,15 @@ impl InitData { asserts_power: bool, next: u32, ) -> Result { - let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + abi::MainConfig::CONTROL_RING - + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; + let ring_base = if self.m5 { + super::fw::m5::initdata::M5_CONTROL_RING_BUNDLE_OFFSETS[0] + } else { self.main_config_offset(Role::Primary) + abi::MainConfig::CONTROL_RING }; + let offset = ring_base + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; let ring = self.object(self.bundle)?.pointer(offset, record.len())?; let power = self .object(self.region_c)? - .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; + .pointer(if self.m5 { super::fw::m5::power::POWER_ASSERT_TALLY } + else { abi::RegionC::POWER_ASSERT_TALLY }, 4)?; let producer = self.control_producer()?; // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. // Firmware only reads records after the producer store and the device mutex serializes @@ -1566,13 +1817,42 @@ impl InitData { } impl InitData { + /// The M5 firmware's progress word ([`super::fw::m5::initdata::M5_PROGRESS`]); ENODEV on + /// other firmware. + pub(super) fn m5_progress(&self) -> Result { + use super::fw::m5::initdata as m; + if !self.m5 { + return Err(ENODEV); + } + fence(Ordering::Acquire); + Ok(self + .object(self.cluster)? + .word(m::M5_PRIVATE_LAYOUT.status_b + m::M5_PROGRESS)? + .load(Ordering::Relaxed)) + } + + fn recovery_state_offset(&self) -> usize { + if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_STATE + } else { self.status_b + abi::PrimaryStatusB::RECOVERY_STATE } + } + /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + let extent = if self.m5 { + *super::fw::m5::initdata::M5_BUNDLE_VIEW_EXTENTS.get(view).ok_or(EINVAL)? + } else { + *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)? + }; // Separately allocated views live at offset 0 of their own object. let (object, base) = match self.views { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None if self.m5 => ( + bundle, + *super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS.get(view).ok_or(EINVAL)?, + ), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; let end = offset @@ -1581,9 +1861,8 @@ impl InitData { if end > extent { return Err(EINVAL); } - let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) - + view * size_of::(); + let pointer_at = self.main_config_offset(Role::Primary) + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1595,11 +1874,26 @@ impl InitData { } impl super::recovery::Memory for InitData { + fn is_m5(&self) -> bool { + self.m5 + } fn fault_sources(&self) -> Result { + let mask_offset = if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_SLOT_MASK + } else { + self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK + }; let mask = self .object(self.cluster)? - .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(mask_offset)? .load(Ordering::Relaxed); + // M5's slot-identity storage is not available through this mapping. + // Keep firmware's explicit blame when attribution evidence is missing, + // while still allowing the recovery handshake to complete. + if self.m5 { + return Ok(super::recovery::Sources::sample(mask as u32, |_| None)); + } let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { // Progress is independently available even when this firmware's key // address is unknown. Zero keeps ownership unknown and cannot override @@ -1625,21 +1919,26 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.recovery_state_offset())? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_HOST_RECOVERY + } else { + self.status_b + abi::PrimaryStatusB::HOST_RECOVERY + })? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.recovery_state_offset())?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1655,6 +1954,10 @@ impl super::recovery::Memory for InitData { // their own object. let (table, offset) = match self.views { Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None if self.m5 => ( + self.object(self.bundle)?, + super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS[4], + ), None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), }; // Validate the complete table before publishing its first live word. @@ -1678,9 +1981,13 @@ impl super::recovery::Memory for InitData { return Err(EINVAL); } let cluster = self.object(self.cluster)?; - let offset = self.status_b - + abi::PrimaryStatusB::RECOVERY_INFO - + index * core::mem::size_of::(); + let base = if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_INFO + } else { + self.status_b + abi::PrimaryStatusB::RECOVERY_INFO + }; + let offset = base + index * core::mem::size_of::(); fence(Ordering::Acquire); let flags = cluster .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? @@ -1695,3 +2002,49 @@ impl super::recovery::Memory for InitData { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); } } + +/// M5 thermal governor inputs and the host power budget it publishes. +impl InitData { + /// M5 only: the live host GPU power budget word of Region C. + pub(super) fn m5_power_budget(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + Ok(self.object(self.region_c)?.word(super::thermal::REGION_C_POWER_BUDGET)?.load(Ordering::Relaxed)) + } + + /// M5 only: the GPU's current power draw in mW, as the firmware's power controller reports it + /// (integer word of the shared power-policy block). + pub(super) fn m5_gpu_power_mw(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + Ok(self.object(self.bundle)? + .word(super::fw::m5::initdata::M5_SHARED_LAYOUT.policy + super::thermal::POLICY_GPU_POWER)? + .load(Ordering::Relaxed)) + } + + /// M5 only: whether the firmware's emergency clamp is active, from its primary performance + /// limiter in the shared power-policy block. + pub(super) fn m5_firmware_clamped(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + let block = self.object(self.bundle)?; + let base = super::fw::m5::initdata::M5_SHARED_LAYOUT.policy; + let mask = block.word(base + super::thermal::POLICY_LIMITER_MASK)?.load(Ordering::Relaxed); + let limit = block.word(base + super::thermal::POLICY_LIMITER_LIMIT)?.load(Ordering::Relaxed); + Ok(super::thermal::is_emergency_clamp(mask, limit)) + } + + /// M5 only: publishes the host GPU power budget like the effort input. + pub(super) fn publish_m5_power_budget(&self, budget_mw: u32) -> Result { + if !self.m5 { return Err(EINVAL); } + let region = self.object(self.region_c)?; + let offset = super::thermal::REGION_C_POWER_BUDGET; + let word = region.word(offset)?; + let line = region.pointer(offset, 4)?; + word.store(budget_mw, Ordering::Relaxed); + // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache + // line and the following system barrier publish the naturally aligned store. + unsafe { + core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, + options(nostack, preserves_flags)) + }; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs new file mode 100644 index 0000000000000..ec150b54f1bee --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware graph backed by the shared G17 object and GPUVM owners. + +use super::*; +use crate::g17::fw::m5::initdata as m; + +const BASE: u64 = 0xffff_fc20_0000_0000; +const PRIVATE: u64 = BASE + 0x2_0000; +const REGION_C: u64 = BASE + 0x1a_0000; +const ROOTS: u64 = BASE + 0x1a_8000; +const REGION_A: u64 = BASE + 0x1b_8000; +const FWCTL: u64 = BASE + 0x1c_0000; +const QOS: u64 = BASE + 0x1c_8000; +const AON: u64 = BASE + 0x1d_0000; +const KICK_SHARED: u64 = BASE + 0x1d_8000; +const BUNDLE: u64 = BASE + 0xc078_8000; +const COMPLETION_HIGH: u64 = BASE + 0xc080_0000; +const COMPLETION_LOW: u64 = 0x70_0184_8000; +const COMPLETION_STRIDE: u64 = 0x2_8000; +const PB: u64 = BASE + 0x15e_0000; +const UMA: u64 = BASE + 0x15e_8000; + +/// Lower-root entry 1 of the M5 layout (Mesa's driver window), linked before the firmware starts +/// as the bare-metal J775D boot has always run. +const BOOT_LINK: u64 = 0x10_0008_0000; + +impl InitData { + /// The kernel lower-root ranges a bare-metal M5 firmware reaches, whose top-level links must + /// exist before it starts: entry 1, the descriptor tables and completion rings (entry 7) and + /// the first page of the canonical arena (`canonical`, entry 8), where queues go later. + pub(in crate::g17) fn m5_lower_links(canonical: core::ops::Range) + -> [core::ops::Range; 8] + { + let page = |address: u64| address..address + mmu::UAT_PGSZ as u64; + let ring = |index: u64| page(COMPLETION_LOW + index * COMPLETION_STRIDE); + [page(BOOT_LINK), page(m::REGION_VIEW_LOW_ADDRS[0]), page(m::REGION_VIEW_LOW_ADDRS[1]), + ring(0), ring(1), ring(2), ring(3), page(canonical.start)] + } + + /// Diagnostic-only read of an aligned word within the existing policy allocation. + /// Callers hold the device state mutex, retaining this mapping for the entire sample. + pub(crate) fn m5_policy_telemetry_word(&self, offset: usize) -> Result { + if !self.m5 || offset & 3 != 0 || offset.checked_add(4).is_none_or(|end| end > m::M5_POLICY_SIZE) { + return Err(EINVAL); + } + let at = m::M5_SHARED_LAYOUT.policy.checked_add(offset).ok_or(EINVAL)?; + Ok(self.object(self.bundle)?.word(at)?.load(Ordering::Relaxed)) + } + + pub(crate) fn note_m5_submission_history(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + let private = self.object(self.cluster)?; + let base = m::M5_PRIVATE_LAYOUT.status_config; + for _ in 0..64 { + let generation = private.word(base + 0x74c)?.load(Ordering::Acquire); + if generation & 1 != 0 { core::hint::spin_loop(); continue; } + let config = private.read_words::<4>(base + 0x73c)?; + fence(Ordering::Acquire); + if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) { + core::hint::spin_loop(); continue; + } + let period = u64::from(config[0]) | u64::from(config[1]) << 32; + let count = u64::from(config[2]) | u64::from(config[3]) << 32; + if period == 0 || count == 0 || count > 256 { return Err(EINVAL); } + let (counter, frequency) = crate::g17::qos::clock(); + let ticks = crate::g17::qos::time_from_host(counter, frequency)?; + // One slot per 25000 ticks of the period. + let index = (ticks % period) / 25_000; + if index >= count { return Err(ERANGE); } + let byte = private.pointer(base + 0x750 + index as usize, 1)?; + // SAFETY: retained, bounds-checked WC byte, written as a single byte + // after reading the even, stable generation, as the firmware expects. + unsafe { + let value = core::ptr::read_volatile(byte).wrapping_add(1); + core::ptr::write_volatile(byte, value); + }; + fence(Ordering::SeqCst); + return Ok(()); + } + Err(EBUSY) + } + + pub(crate) fn build_m5(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + id: &crate::regs::G17Id, board: &m::Board) -> Result { + use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, + PROT_GPU_SHARED_RW as GPU}; + self.m5 = true; + // TA, fragment and compute share the compute work ring per priority; every + // preparation's claim on it is counted from the first publication on. + self.outer_credits = Some(super::outer_credit::Credits::new()?); + let fixed = |address, size, protection| KernelObject::new(dev, + uat.kernel_vm(), Placement::At(address), size, protection, CpuMap::WriteCombined); + let fixed_controller = |address, size| { + KernelObject::new(dev, uat.kernel_vm(), Placement::At(address), size, + mmu::PROT_FW_PRIV_RW, CpuMap::WriteBack) + }; + let private = m::M5_PRIVATE_LAYOUT; + let shared = m::M5_SHARED_LAYOUT; + self.cluster = self.add(fixed(PRIVATE, private.total, RW)?)?; + self.region_c = self.add(fixed(REGION_C, mmu::UAT_PGSZ, RO)?)?; + self.roots = self.add(fixed(ROOTS, 0xc000, RO)?)?; + let region_a = self.add(fixed(REGION_A, mmu::UAT_PGSZ, RO)?)?; + self.add(fixed(FWCTL, mmu::UAT_PGSZ, RW)?)?; + self.qos = self.add(fixed(QOS, m::G17P_QOS_RESOURCE_SIZE, RW)?)?; + self.add(fixed(KICK_SHARED, m::M5_KICK_SHARED_SIZE, mmu::PROT_GPU_FW_SHARED_RW)?)?; + let aon = self.add(fixed_controller(AON, m::M5_AON_PTD_SIZE)?)?; + self.bundle = self.add(fixed_controller(BUNDLE, shared.total)?)?; + self.pb = self.add(fixed(PB, mmu::UAT_PGSZ, RW)?)?; + self.page_pool = self.add(fixed(UMA, mmu::UAT_PGSZ, RW)?)?; + for stage in 0..2 { + self.stamps[stage] = self.add(KernelObject::new(dev, uat.kernel_vm(), + Placement::In(uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64), + mmu::UAT_PGSZ, mmu::PROT_FW_SHARED_RW, CpuMap::WriteCombined)?)?; + } + self.alias(self.pb, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[0], GPU)?; + self.alias(self.page_pool, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[1], GPU)?; + let mut completions = [m::G17PKsmCompletionAliases { low: 0, high: 0 }; m::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (index, pair) in completions.iter_mut().enumerate() { + pair.low = COMPLETION_LOW + index as u64 * COMPLETION_STRIDE; + pair.high = COMPLETION_HIGH + index as u64 * COMPLETION_STRIDE; + // After initialization the host only reads completion records. + // Match the standard G17 completion ring's CPU write-back mapping; + // keep firmware/GPU aliases, size and lifetime unchanged. + let backing = if index == 0 + { + KernelObject::new(dev, uat.kernel_vm(), Placement::At(pair.high), + m::M5_25G76_COMPLETION_BACKING_SIZE, RW, CpuMap::WriteBack)? + } else { + fixed(pair.high, m::M5_25G76_COMPLETION_BACKING_SIZE, RW)? + }; + let object = self.add(backing)?; + *self.completion_rings.get_mut(index).ok_or(EINVAL)? = Some(object); + self.alias(object, uat.kernel_lower_vm(), pair.low, GPU)?; + } + m::encode_common26_aon_ptd(board, &mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) + .map_err(|_| EINVAL)?; + m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; + m::encode_m5_private_cluster(PRIVATE, FWCTL, + &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; + let windows = board.register_windows(); + let windows = &windows[..board.register_window_count()]; + for (index, window) in windows.iter().enumerate() { + let spec = board.register_window_spec(index).ok_or(EINVAL)?; + let stride = (u64::from(window.size) + mmu::UAT_PGMSK as u64) & !(mmu::UAT_PGMSK as u64); + for (instance, physical) in spec.instances.iter().copied().enumerate() { + let offset = physical & mmu::UAT_PGMSK as u64; + let address = window.device_va.checked_sub(offset).ok_or(EINVAL)? + + instance as u64 * stride; + let size = (offset + u64::from(window.size) + mmu::UAT_PGMSK as u64) + & !(mmu::UAT_PGMSK as u64); + self.aliases.push(uat.kernel_vm().map_io(dev.as_ref().as_ref(), address, + (physical & !(mmu::UAT_PGMSK as u64)).try_into()?, size.try_into()?, + mmu::PROT_FW_MMIO_RW)?, GFP_KERNEL)?; + } + } + let addresses = m::SharedAddresses { + region_a: self.object(region_a)?.gpu_va(), region_c: REGION_C, + hw_data_bundle: BUNDLE, hw_data_bundle_alloc: shared.total, + }; + let views = m::PrimaryRegionViews { + sentinel_high: BASE + 0xc07e_0000, alias_high: [PB, UMA], + }; + m::encode_m5_hw_data(board, completions, QOS, windows, + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + m::set_m5_kick_shared( + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], KICK_SHARED) + .map_err(|_| EINVAL)?; + let masked_cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let enabled_cores = if masked_cores != 0 { masked_cores } + else { id.num_clusters.checked_mul(id.num_cores).ok_or(EOVERFLOW)? }; + m::set_m5_enabled_cores( + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], enabled_cores) + .map_err(|_| EINVAL)?; + m::encode_m5_bundle_policy(m::M5PolicyBindings { + state: PRIVATE + private.policy_state as u64, aon_ptd: AON, boot_timestamp: 0, + }, self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; + // The controller configuration applies where the part's leakage calibration matches the + // one it was made for, or on a board whose configuration holds for every part. + let calibrated = match board.calibration { + None => true, + Some(expected) => { + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let calibration: Result> = + node.get_property(c_str!("apple,leakage-calibration-words")); + calibration.as_ref().is_ok_and(|words| words.as_slice() == expected) + } + }; + if let Some(defaults) = board.controller_defaults.filter(|_| calibrated) { + defaults(self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; + let maximum = self.object(self.bundle)?.word(shared.policy + m::M5_POLICY_MAXIMUM_POWER)? + .load(Ordering::Relaxed); + m::set_m5_region_maximum_power(self.object_mut(self.region_c)?.bytes_mut(), maximum) + .map_err(|_| EINVAL)?; + } + if let Some(microseconds) = board.idle_standby_us { + m::set_m5_region_idle_standby(self.object_mut(self.region_c)?.bytes_mut(), microseconds) + .map_err(|_| EINVAL)?; + } + for (index, role) in [m::InstanceRole::Primary, m::InstanceRole::Secondary].into_iter().enumerate() { + let instance = m::InstanceAddresses { + root: ROOTS + index as u64 * 0x8000, + main_config: BUNDLE + shared.main_config[index] as u64, + status_a: PRIVATE + private.roles[index].status_a as u64, + status_b: if index == 0 { PRIVATE + private.status_b as u64 } else { 0 }, + secondary_extras: if index == 0 { [0; 2] } else { + [PRIVATE + private.status_config as u64, PRIVATE + private.secondary_extra_1 as u64] + }, + }; + let profile = m::RootProfile::J714G17S26A428; + let channels = m::derive_channel_table_for_profile(profile, role, + PRIVATE + m::M5_PRIVATE_STATE_GRIDS[index] as u64, + instance.status_a, instance.main_config, BUNDLE); + let offset = shared.main_config[index]; + m::encode_main_config_for_profile(profile, role, &addresses, &instance, + &channels, if index == 0 { Some(&views) } else { None }, + &mut self.object_mut(self.bundle)?.bytes_mut()[offset..offset + m::MAIN_CONFIG_SIZE]) + .map_err(|_| EINVAL)?; + let offset = index * 0x8000; + m::encode_root_for_profile(profile, role, &addresses, &instance, + &mut self.object_mut(self.roots)?.bytes_mut()[offset..offset + m::M5_ROOT_SIZE]) + .map_err(|_| EINVAL)?; + } + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "M5 firmware graph ready: private {:#x}+{:#x}, shared {:#x}+{:#x}\n", + PRIVATE, private.total, BUNDLE, shared.total); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs new file mode 100644 index 0000000000000..08804ff1fc498 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pending capacity claims for M5's per-priority shared outer ring. Firmware +//! consumption frees posted slots; RAII cancellation frees unposted claims. + +use crate::g17::{availability::Availability, fw::channels::WORK_RING_SLOTS}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +const PRIORITIES: usize = 4; + +pub(super) struct Credits { + reserved: [AtomicU32; PRIORITIES], + published: [AtomicU64; PRIORITIES], + consumers: [AtomicU64; PRIORITIES], + scoped: AtomicBool, + available: Arc, +} + +pub(crate) struct OuterClaim { + domain: Arc, + priority: usize, + remaining: u32, +} + +/// Borrows the claim only. The caller must hold the device mutex from entry +/// through Drop: its temporarily lent credits are available to that publisher. +pub(crate) struct OuterScope<'a> { + claim: &'a mut OuterClaim, + before: u64, +} + +pub(super) fn free([a, b, producer]: [u32; 3]) -> Result { + if [a, b, producer].iter().any(|&v| v >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(((a + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS) + .min((b + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS)) +} + +impl Credits { + pub(super) fn new() -> Result> { + Ok(Arc::new(Self { + reserved: [const { AtomicU32::new(0) }; PRIORITIES], + published: [const { AtomicU64::new(0) }; PRIORITIES], + consumers: [const { AtomicU64::new(u64::MAX) }; PRIORITIES], + scoped: AtomicBool::new(false), + available: Availability::new()?, + }, GFP_KERNEL)?) + } + + pub(super) fn available(&self) -> Arc { self.available.clone() } + + pub(super) fn waiting(&self) -> bool { self.available.has_waiters() } + + pub(super) fn capacity(&self, priority: u8, cursors: [u32; 3]) -> Result { + let pending = self.reserved.get(usize::from(priority)).ok_or(EINVAL)? + .load(Ordering::Acquire); + Ok(free(cursors)?.saturating_sub(pending)) + } + + /// Reserve under the sole publisher mutex. Off-lock cancellation may only + /// reduce reserved counts, so the sampled capacity is conservative. + pub(super) fn reserve(self: &Arc, priority: u8, count: u32, + cursors: [u32; 3]) -> Result + { + if count == 0 || count >= WORK_RING_SLOTS { return Err(EINVAL); } + if self.scoped.load(Ordering::Acquire) { return Err(EBUSY); } + let index = usize::from(priority); + let counter = self.reserved.get(index).ok_or(EINVAL)?; + let seen = (u64::from(cursors[0]) << 32) | u64::from(cursors[1]); + self.consumers[index].compare_exchange(u64::MAX, seen, Ordering::AcqRel, Ordering::Acquire).ok(); + if self.capacity(priority, cursors)? < count { return Err(EBUSY); } + counter.fetch_add(count, Ordering::AcqRel); + Ok(OuterClaim { domain: self.clone(), priority: index, remaining: count }) + } + + pub(super) fn scope<'a>(self: &Arc, claim: &'a mut OuterClaim) -> Result> { + if !Arc::ptr_eq(self, &claim.domain) { return Err(EINVAL); } + if self.scoped.swap(true, Ordering::AcqRel) { return Err(EBUSY); } + self.reserved[claim.priority].fetch_sub(claim.remaining, Ordering::AcqRel); + let before = self.published[claim.priority].load(Ordering::Acquire); + Ok(OuterScope { claim, before }) + } + + pub(super) fn counter(&self, priority: u8) -> Result<&AtomicU64> { + self.published.get(usize::from(priority)).ok_or(EINVAL) + } + + pub(super) fn progressed(&self, priority: u8, cursors: [u32; 3]) -> Result { + free(cursors)?; + let counter = self.consumers.get(usize::from(priority)).ok_or(EINVAL)?; + let current = (u64::from(cursors[0]) << 32) | u64::from(cursors[1]); + let previous = counter.swap(current, Ordering::AcqRel); + Ok(previous != u64::MAX && previous != current) + } +} + +impl Drop for OuterScope<'_> { + fn drop(&mut self) { + let domain = &self.claim.domain; + let published = domain.published[self.claim.priority].load(Ordering::Acquire) + .wrapping_sub(self.before); + // Every publication also checks unreserved physical capacity. If an + // enclosing transaction posts extra records, only this claim's own + // credits are consumed here; another claim can never be debited. + self.claim.remaining -= published.min(u64::from(self.claim.remaining)) as u32; + domain.reserved[self.claim.priority].fetch_add(self.claim.remaining, Ordering::AcqRel); + domain.scoped.store(false, Ordering::Release); + } +} + +impl OuterClaim { + /// Credits not yet consumed by a publication. A claim with none left can be dropped + /// anywhere: only returned credits notify waiters. + pub(crate) fn remaining(&self) -> u32 { + self.remaining + } +} + +impl Drop for OuterClaim { + /// As for physical queue claims, drop outside the device mutex because + /// availability notification can run DMA-fence callbacks. + fn drop(&mut self) { + if self.remaining != 0 { + self.domain.reserved[self.priority].fetch_sub(self.remaining, Ordering::AcqRel); + self.domain.available.notify(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index f7b1952685761..c0485c4932e48 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -130,7 +130,7 @@ impl InitData { calibration.max_power_mw, &calibration.leakage, ), - (None, Some(chip)) => (cfg.perf, chip.max_power_mw, &chip.leakage), + (None, Some(chip)) => (cfg.perf()?, chip.max_power_mw, &chip.leakage), _ => return Err(EINVAL), }; let hw_args = abi::HwDataArgs { @@ -165,7 +165,7 @@ impl InitData { })?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(cfg.region_c()?); region.max_power_mw = max_power_mw; Ok(()) })?; diff --git a/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs b/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs new file mode 100644 index 0000000000000..eafd6daac4a85 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Two render queues sharing one outer ring. Capacity and both destinations +//! are checked before the host-port transaction can make either queue runnable. + +use crate::g17::fw::channels::{WorkSlot, WORK_RING_SLOTS}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Window { + pub(super) start: u32, + pub(super) next: [u32; 2], +} + +impl Window { + pub(super) fn new([consumer, consumer_b, start]: [u32; 3]) -> Result { + if [consumer, consumer_b, start].iter().any(|&p| p >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + let next = [(start + 1) % WORK_RING_SLOTS, (start + 2) % WORK_RING_SLOTS]; + if next.iter().any(|&p| p == consumer || p == consumer_b) { + return Err(EBUSY); + } + Ok(Self { start, next }) + } +} + +pub(in crate::g17) struct Publication<'a> { + published: Option<&'a AtomicU64>, + records: [WorkSlot; 2], + destinations: [*mut u8; 2], + producer: &'a AtomicU32, + window: Window, +} + +impl<'a> Publication<'a> { + /// SAFETY: Both destinations must identify distinct, writable WorkSlot + /// storage in the retained ring at window.start and window.next[0]. The + /// caller holds the sole host publication lock until this value is dropped. + pub(super) unsafe fn new( + records: [WorkSlot; 2], + destinations: [*mut u8; 2], + producer: &'a AtomicU32, + window: Window, + published: Option<&'a AtomicU64>, + ) -> Self { + Self { records, destinations, producer, window, published } + } + + pub(in crate::g17) fn next(&self) -> [u32; 2] { self.window.next } + + pub(in crate::g17) fn prepare(&self) { + for (record, destination) in self.records.iter().zip(self.destinations) { + // SAFETY: Constructor checked both live destinations. Records are + // initialized, padding-free integer structures owned by this value. + unsafe { + core::ptr::copy_nonoverlapping( + (record as *const WorkSlot).cast::(), destination, size_of::(), + ); + } + } + } + + /// Publish both records with one cursor update, after both inner queues and + /// their accounting are visible. No fallible operation follows the port write. + pub(in crate::g17) fn publish(self) -> [u32; 2] { + self.prepare(); + fence(Ordering::SeqCst); + self.producer.store(self.window.next[1], Ordering::Relaxed); + if let Some(published) = self.published { published.fetch_add(2, Ordering::Release); } + fence(Ordering::SeqCst); + self.window.next + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index fcd3a1a7a74f3..c724a2266bb15 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -7,6 +7,9 @@ use crate::driver::AsahiDrmDevice; use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ bindings, c_str, + cpu::CpuId, + cpumask::Cpumask, + workqueue, dma_fence::{Fence, RawDmaFence}, drm::sched, prelude::*, @@ -14,8 +17,9 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -/// One logical render queue owns one physical pair, whose active ring holds -/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// Each physical pair of a logical render queue holds `RENDER_DEPTH` commands +/// in its active ring, so a queue gets that many credits per pair it may hold +/// ([`Backend::render_pairs`]). A deeper scheduler would only park run_job in /// admission until a retirement; the scheduler waits for the credit instead. const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = crate::hw::t8140::queues::COMPUTE_DEPTH as u32; @@ -47,6 +51,9 @@ impl Engine { pub(crate) struct Order { pub(crate) sequence: u64, pub(crate) wait_through: [Option; 2], + /// What a render's tiling half waits on: `wait_through` unless the command + /// carries its own, never later, tiling prefixes. + pub(crate) tiling_wait_through: [Option; 2], } impl Order { @@ -87,9 +94,22 @@ impl Order { Ok(Self { sequence: sequences[engine.index()], wait_through, + tiling_wait_through: wait_through, }) } + /// Give the tiling half the narrower prefixes of `tiling`, which resolves the + /// command's tail against the same ioctl position. + pub(crate) fn narrow_tiling(&mut self, tiling: &Self) -> Result { + for index in 0..2 { + if tiling.wait_through[index].unwrap_or(0) > self.wait_through[index].unwrap_or(0) { + return Err(EINVAL); + } + } + self.tiling_wait_through = tiling.wait_through; + Ok(()) + } + pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { sequence != 0 && prefix.is_some_and(|last| sequence <= last) } @@ -121,8 +141,8 @@ impl Packet { firmware: KVec, ) -> Result> { let footprint = match &command { - Validated::Render { attachments, .. } => { - context.vm().render_footprint(&attachments[1])? + Validated::Render { attachments, pass, .. } => { + context.vm().render_footprint(&attachments[1], pass.whole_attachments)? } Validated::Compute { .. } => None, }; @@ -191,10 +211,19 @@ impl Packet { } check_status(fence)?; } + self.check_firmware_producers()?; + self.completion.check_source_inputs()?; + Ok(()) + } + + /// Producers this packet waits for in firmware (another engine's commands that + /// were published, not complete, when it was scheduled). Checked before + /// publication, and again at retirement: a producer that fails after this + /// packet was published fails it too, although the firmware ran it. + pub(crate) fn check_firmware_producers(&self) -> Result { for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; Ok(()) } @@ -281,9 +310,25 @@ pub(crate) trait Backend: Send + Sync { None } + /// The device accepts command submission (an M5 GPU before its queues take submission + /// does not). + fn supports_submission(&self) -> bool { true } + /// Whether this backend's scheduler workers run in their queue creator's cluster (M5). + fn creator_cluster_scheduling(&self) -> bool { false } + /// The reclaim-safe queue for the scheduler's timeout work, whose handler signals the + /// fences of timed-out jobs. + fn fence_queue(&self) -> Arc; + /// Render pairs this logical queue may hold, fixed when it is created: its + /// render scheduler's credits are fixed then too. + fn render_pairs(&self) -> u32 { 1 } + fn supports_engine(&self, _engine: Engine) -> bool { true } + /// Device policy applied to a validated command before it is ordered (the M5 resource + /// plan). + fn prepare_command(&self, _command: &mut super::command::Validated) -> Result { Ok(()) } + /// Validate compute sizing without reserving a physical channel. Queued /// commands may wait on a host signal submitted after this ioctl returns. - fn ensure_compute(&self, scs_pages: u16) -> Result; + fn ensure_compute(&self, command: &super::command::Validated) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } Ok(super::fw::freelist::RENDER_BLOCKS) @@ -302,6 +347,9 @@ pub(crate) trait Backend: Send + Sync { fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; /// A long job of another queue is running beside `packet`. fn long_job_beside(&self, _packet: &Arc) -> bool { false } + /// A published job's deadline may be renewed while other work retires + /// ([`super::timeout::renew_shared`]). + fn renews_shared(&self) -> bool { true } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -357,8 +405,9 @@ impl sched::JobImpl for Job { } // Other work retiring means the GPU is shared, not hung. So does another long job // running beside a long job, though neither retires until it ends. - let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) - || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); + let shared = job.backend.renews_shared() + && (super::timeout::retiring(super::recovery::since_retired(super::now_ns())) + || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet))); if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { return true; } @@ -432,10 +481,26 @@ impl Fences { impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), + Engine::Render => ( + RENDER_CREDITS.checked_mul(backend.render_pairs()).ok_or(EOVERFLOW)?, + c_str!("aurora_g17_render"), + ), Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), }; - let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; + let device = dev.as_ref().as_ref(); + let submit = if backend.creator_cluster_scheduling() { + // The run and free work stays in the creating CPU's cluster, as the event work + // follows the submitter, instead of wherever an unbound worker last ran. + let queue = workqueue::OwnedQueue::new_ordered_reclaim(name)?; + queue.restrict(Cpumask::cluster_of(CpuId::current()))?; + Some(queue) + } else { + // drm_sched allocates its own ordered reclaim-safe queue. + None + }; + let scheduler = sched::Scheduler::new_with_queues( + device, 1, credits, 0, TIMEOUT_MS, name, submit, Some(backend.fence_queue()), + )?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; Ok(Self { entity, diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 6499cae106309..0d0bf6a1277c4 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -20,7 +20,7 @@ use super::{ fw::{kick::*, queue::DataMaster}, object::{Allocator, CpuMap, KernelObject}, }; -use crate::{hw::t8140, mem, mmu}; +use crate::{mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; @@ -56,7 +56,9 @@ const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); // Compute keeps this many channels even with every execution context open. assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); - assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); + // The reserve counts one pair per owner. A second pair of an owner comes + // only from surplus (g17::pairs), never from this reserve. + assert!(FULL_RENDER_RESERVE == 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2); }; /// Stable physical-channel identity. The owner must not be reused within a @@ -398,11 +400,27 @@ enum Backing { Render(RenderRing), } +/// A client VM's view of a kick ring. The host writes the entries through the kernel's mapping +/// and the firmware reads them through its own; a client's GPU work only reads them, so its +/// view is read-only: a client cannot rewrite an entry after the kernel built and checked it. +const CLIENT_KICK_PROT: mmu::Prot = mmu::PROT_GPU_SHARED_RO; + /// A render ring's pages and the shared lower reservation clients alias. struct RenderStorage { // The lower alias drops before its backing. low: mmu::KernelMapping, high: KernelObject, + /// The client VM the storage was created for, whose alias of it lasts as long as that VM. + vm: u64, + /// That VM's status, which tells when its client closed it. + status: Option>, +} + +impl RenderStorage { + /// The client closed the VM this storage was created for. + fn closed(&self) -> bool { + self.status.as_ref().is_some_and(|status| status.client_closed()) + } } /// Render ring storage outlives its rings for the life of the device. @@ -442,12 +460,27 @@ impl RenderRings { ) } - fn take(&self) -> Option { + /// Reusable storage created for `client`, else storage whose VM its client has closed, + /// which then belongs to `client`. Storage of another open VM is never handed out: that VM + /// keeps its read-write alias of the pages, so it could read and write the new ring. + fn take(&self, client: &mmu::Vm) -> Option { + let vm = client.id(); let mut stock = self.stock.lock(); - let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let reusable = |own: bool| { + stock.retired.iter().rposition(|(reusable, storage)| { + *reusable && if own { storage.vm == vm } else { storage.closed() } + }) + }; + let index = reusable(true).or_else(|| reusable(false))?; let last = stock.retired.len() - 1; stock.retired.swap(index, last); - stock.retired.pop().map(|(_, storage)| storage) + let (_, mut storage) = stock.retired.pop()?; + drop(stock); + if storage.vm != vm { + storage.vm = vm; + storage.status = client.status().cloned(); + } + Some(storage) } /// Reserves the return of one more, just created storage. @@ -538,11 +571,11 @@ impl Queue { mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; - let mut range = alloc.uat.geometry().kernel_range(); - range.start += t8140::dynamic::KERNEL_OFFSET; + // The UAT's dynamic range: on M5 it lies past the boot graph, its unmapped sentinel and the + // firmware register aperture. let high = low.alias_in( alloc.uat.kernel_vm(), - range, + alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, )?; @@ -567,7 +600,7 @@ impl Queue { if id.engine == DataMaster::Compute { return Err(EINVAL); } - if let Some(storage) = rings.take() { + if let Some(storage) = rings.take(client) { let mut ring = RenderRing { storage: ManuallyDrop::new(storage), rings: rings.clone(), @@ -580,7 +613,7 @@ impl Queue { let alias = ring.storage .high - .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + .prepare_alias(client, address, CLIENT_KICK_PROT, false); // On failure the storage returns to the stock and a fresh ring is built. if let Ok(alias) = alias { return Ok(( @@ -594,13 +627,22 @@ impl Queue { )); } } + let (vm, status) = (client.id(), client.status().cloned()); + // The firmware's mapping sits in the upper half, which every execution context includes. + // On M5 it is firmware-only, as a compute ring's is; the GPU reads the ring at its lower + // address. + let firmware_prot = if alloc.uat.is_m5() { + mmu::PROT_FW_SHARED_RW + } else { + mmu::PROT_GPU_FW_SHARED_RW + }; let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + firmware_prot, CpuMap::WriteCombined, )?; - let mut range = t8140::dynamic::LOWER; + let mut range = alloc.uat.kernel_lower_dynamic_range(); let (low, client) = loop { let low = high.alias_in( alloc.uat.kernel_lower_vm(), @@ -612,7 +654,7 @@ impl Queue { .iova() .checked_add(KICK_RING_SIZE as u64) .ok_or(EOVERFLOW)?; - match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { + match high.map_alias(client, low.iova(), CLIENT_KICK_PROT) { Ok(client) => break (low, client), Err(error) if error == ENOSPC && end < range.end => { drop(low); @@ -621,7 +663,7 @@ impl Queue { Err(error) => return Err(error), } }; - let storage = RenderStorage { low, high }; + let storage = RenderStorage { low, high, vm, status }; // Count the storage only once it exists. Without room to return it, // nothing has aliased it but `client`, which drops first. if let Err(error) = rings.admit() { @@ -668,7 +710,31 @@ impl Queue { pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let address = self.low_va(); self.object() - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + .map_alias(vm, address, CLIENT_KICK_PROT) + } + + /// Clears an idle compute ring before another VM is given a view of it, so the new owner + /// cannot read the entries the previous owner's commands left. The firmware has consumed + /// every entry; the host's position in the ring is kept. + pub(crate) fn clear_ring(&mut self) -> Result { + let Backing::Compute { low, .. } = &mut self.backing else { + return Err(EINVAL); + }; + low.bytes_mut().fill(0); + fence(Ordering::SeqCst); + Ok(()) + } + + /// Clears an idle render ring before a context of another VM is given a view of it, so the + /// new owner cannot read the entries earlier owners' commands left. The firmware has + /// consumed every entry; the host's position in the ring is kept. + pub(crate) fn clear_render_ring(&mut self) -> Result { + let Backing::Render(ring) = &mut self.backing else { + return Err(EINVAL); + }; + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + Ok(()) } /// Reuse a permanent client view only when every page names this ring. @@ -677,14 +743,25 @@ impl Queue { Backing::Compute { low, .. } => low, Backing::Render(ring) => &ring.storage.high, }; - object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) + object.prepare_alias(vm, self.low_va(), CLIENT_KICK_PROT, false) } /// Both aliases are owned mappings of the same full ring. Publish their /// page-table updates to each live translation regime before installation. + /// + /// With published roots each alias reaches every context whose lower or + /// upper half names its VM's root: the low alias lives in the kernel lower + /// VM, the firmware alias in the kernel upper VM, whose root sits in the + /// upper half of every execution context. Under the handoff the first four + /// contexts are invalidated. pub(crate) fn prepare_compute_install(&self) -> Result { - if self.id.engine != DataMaster::Compute { + let Backing::Compute { high, low } = &self.backing else { return Err(EINVAL); + }; + if high.tracks_contexts() { + low.publish_to_contexts()?; + high.publish_to_contexts(); + return Ok(()); } mem::sync(); for asid in 0..4 { @@ -710,10 +787,17 @@ impl Queue { _ => self.timestamp.prev(), }) } + /// The parent's predecessor, while this queue remembers a previous kick: a G17D fragment + /// kick names it after its pipelined self-parent. Never on the first kick's synthetic + /// parent; recovery clears the history, and with it this. + pub(crate) fn ancestor(&self) -> Option { + self.previous.map(KickTimestamp::prev) + } pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { if args.qid != self.id.qid || args.timestamp != self.timestamp || args.parent != self.parent() + || args.ancestor.is_some_and(|ancestor| Some(ancestor) != self.ancestor()) { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ffb0c20d0267f..81ffe37c872a4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -21,12 +21,15 @@ //! nests inside nothing. Capacity waits block on an availability domain signalled by the //! release that provides the capacity, never on a sleep. -mod availability; +pub(crate) mod availability; mod buffer; mod calibration; mod channel; mod command; pub(crate) use command::Attachments; +// M5 render encodings, reached only once the M5 render runtime is wired. +#[allow(dead_code)] +mod render_resources; mod completion; pub(crate) mod context; mod data_segment; @@ -43,6 +46,7 @@ mod fence; mod freelist; pub(crate) mod fw; pub(crate) mod hazard; +mod pairs; mod ids; mod initdata; mod job; @@ -61,6 +65,7 @@ pub(crate) mod stats; pub(crate) mod status; mod teardown; pub(crate) mod telemetry; +mod thermal; mod timeout; mod validation; @@ -157,6 +162,43 @@ pub(crate) struct FixedBuffer { pub(crate) size: usize, } +/// Firmware protocol of a G17 GPU. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum FirmwareProfile { + /// The firmware publishes its kernel roots and the driver owns the page tables (A18 Pro, + /// M4 Pro). + PublishedRoots, + /// An M5 board without bare-metal GPU tables yet: probe refuses it. + M5NoBareMetalTables, + /// The M5 firmware on bare metal: both roles publish their kernel roots + /// ([`mmu::UatFirmware::PublishedRoots`]) and the driver owns the page tables. + M5PublishedRoots, +} + +impl Config { + /// The fixed parameter-buffer metrics buffer of a 26A428 or 25G83 graph. + pub(crate) fn pm_metrics(&self) -> Result<&FixedBuffer> { + self.pm_metrics.as_ref().ok_or(EINVAL) + } + + /// The performance tables of a 26A428 or 25G83 graph. + pub(crate) fn perf(&self) -> Result<&'static fw::initdata::PerfConfig> { + self.perf.ok_or(EINVAL) + } + + /// Region C of a 26A428 or 25G83 graph. + pub(crate) fn region_c(&self) -> Result<&'static fw::initdata::RegionCConfig> { + self.region_c.ok_or(EINVAL) + } +} + +impl FirmwareProfile { + /// The M5 firmware's protocol (26A428 InitData graph, queues and submission). + pub(crate) const fn is_m5(self) -> bool { + matches!(self, Self::M5NoBareMetalTables | Self::M5PublishedRoots) + } +} + /// Static configuration of a SoC with a G17 GPU. pub(crate) struct Config { /// SoC identifier. @@ -172,14 +214,18 @@ pub(crate) struct Config { pub(crate) secondary_root_offset: Option, /// Where the coprocessors' crash log buffers live. pub(crate) crash_buffers: CrashBuffers, - /// Parameter buffer page metrics shared with the firmware. - pub(crate) pm_metrics: FixedBuffer, + /// Firmware protocol. + pub(crate) profile: FirmwareProfile, + /// Parameter buffer page metrics shared with the firmware; `None` where the firmware graph + /// places its own (M5). + pub(crate) pm_metrics: Option, /// Register windows the firmware maps through the hardware data. pub(crate) windows: &'static [fw::initdata::RegisterWindow], /// Register-map slots that are reserved but have no window. pub(crate) reserved_slots: &'static [usize], - /// Per-SoC parts of the performance tables. - pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Per-SoC parts of the performance tables; `None` where the firmware graph has its own + /// (M5, [`fw::m5::initdata`]). + pub(crate) perf: Option<&'static fw::initdata::PerfConfig>, /// Number of operating points the device tree lists. pub(crate) opps: usize, /// UAT policy where it differs from the A18 Pro's. @@ -191,9 +237,11 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, - /// The SoC's values of region C. - pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// The SoC's values of region C; `None` where the firmware graph has its own (M5). + pub(crate) region_c: Option<&'static fw::initdata::RegionCConfig>, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, + /// The M5 board whose HW data and power policy the M5 graph is built with; `None` off M5. + pub(crate) m5: Option<&'static fw::m5::initdata::Board>, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -645,6 +693,8 @@ struct Firmware { recovery: recovery::State, effort: power::Effort, hold: power::Hold, + /// M5: host GPU power budget governor (Region C's budget word). + governor: thermal::Governor, registration_power_release_pending: bool, variant: &'static fw::Variant, queues: runtime::Registry, @@ -658,6 +708,18 @@ struct Firmware { uat: Arc, } +/// Set once this module is pinned until reboot for a session that is never freed: an M5 +/// session once a role has run, which no engine retirement witness releases +/// ([`initdata::InitData::has_engine_retirement_witness`]). Its RTKit and mailbox callbacks +/// run this module's code. +static SESSION_RETAINED: AtomicBool = AtomicBool::new(false); + +/// Pins this module until reboot for a session that is never freed, once +/// ([`crate::retain::retain_module`]). ENODEV once the module's removal has begun. +fn retain_session() -> Result { + crate::retain::retain_module(&SESSION_RETAINED) +} + impl Drop for Firmware { fn drop(&mut self) { if let Some(secondary) = self.secondary.as_mut() { @@ -697,10 +759,16 @@ impl Firmware { /// Holds idle descent while a fresh physical render pair is registered. fn acquire_registration_power(&mut self) -> Result { + if !self.variant.registration_power_hold { + return Ok(()); + } power::acquire(self) } fn release_registration_power(&mut self) -> Result { + if !self.variant.registration_power_hold { + return Ok(()); + } // An idle hold keeps the cores powered past registration; it releases them itself. if self.hold.active() { self.registration_power_release_pending = false; @@ -709,13 +777,102 @@ impl Firmware { power::release(self) } + /// Retry a temporarily unavailable transaction once under a registration + /// power hold. Other access errors propagate to the retained owner. + fn submit_m5_work(&mut self, submission: &fw::m5::queue::HardwareSubmission) -> Result { + match self.bridge.regs.submit_m5_work(submission) { + Err(error) if error == EAGAIN => { + self.acquire_registration_power()?; + self.bridge.regs.submit_m5_work(submission) + } + result => result, + } + } + + /// Both outer slots and their capacity are checked before any host-port + /// write. An EAGAIN retry drops the old views and revalidates them under a + /// power hold. After a successful transaction, publication is infallible. + fn submit_m5_render_pair( + &mut self, + priority: u8, + records: &[fw::channels::WorkSlot; 2], + submission: &fw::m5::queue::HardwareSubmission, + commit_accounting: impl FnOnce() -> Result, + publish_inner: impl FnOnce(), + deferred: bool, + ) -> Result<[u32; 2]> { + let mut commit_accounting = Some(commit_accounting); + let mut publish_inner = Some(publish_inner); + for attempt in 0..2 { + let pair = self.init.m5_work_pair(priority, &records[0], &records[1])?; + pair.prepare(); + if let Some(commit) = commit_accounting.take() { commit()?; } + match self.bridge.regs.submit_m5_work(submission) { + Err(error) if error == EAGAIN && attempt == 0 => { + drop(pair); + self.acquire_registration_power()?; + } + Err(error) => return Err(error), + Ok(()) => { + if let Some(publish) = publish_inner.take() { publish(); } + return Ok(if deferred { pair.next() } else { pair.publish() }); + } + } + } + Err(EAGAIN) + } + /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.primary.state.shared.note_submitter(); self.port.note_gpu_activity(); + self.primary.state.shared.note_gpu_activity(); self.effort.submit(); + self.effort.note_waiters(now_ns(), fence::waiters()); self.apply_effort(power::FULL_EFFORT) } + /// One governor step with the hotspot temperature reading; returns whether to run again and + /// what the CPU ceiling should do (applied by the caller without the device lock). + fn governor_tick(&mut self, temp_mc: Option, fans: bool) + -> (bool, thermal::CpuAdvice, bool) + { + let now = now_ns(); + // Work published since the last step, with or without the device mutex. + let published = self.primary.state.shared.governor_activity_ns.swap(0, Ordering::AcqRel); + if published != 0 { + self.governor.note_activity(published); + } + // Effort remains asserted until submitted work retires. A long job + // must not look idle merely because it has no subsequent submission. + if self.effort.applied() != 0 && thermal::requested() { + self.governor.note_activity(now); + } + let Ok(current) = self.init.m5_power_budget() else { + return (false, thermal::CpuAdvice::Release, false); + }; + self.governor.set_maximum(current); + let power = self.init.m5_gpu_power_mw().unwrap_or(0); + let clamped = self.init.m5_firmware_clamped().unwrap_or(false); + let (publish, again) = self.governor.tick(temp_mc, power, clamped, now); + if let Some(budget) = publish { + if self.init.publish_m5_power_budget(budget).is_err() { + return (false, thermal::CpuAdvice::Release, false); + } + } + let advice = self.governor.cpu_advice(power, now); + let want_fans = self.governor.fans_wanted(now); + // Keep ticking while the fans are still boosted, so the hold can expire. + (again || fans, advice, want_fans) + } + + /// Restore the calibrated maximum budget (shutdown). + fn governor_restore(&mut self) { + if let Some(budget) = self.governor.restore() { + let _ = self.init.publish_m5_power_budget(budget); + } + } + fn apply_effort(&mut self, effort: u32) -> Result { Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } @@ -726,13 +883,16 @@ impl Firmware { port: &port::Port, effort: u32, ) -> Result { - if state.applied() == effort && init.effort()? == effort { + // The performance-controller demand follows the requested effort; the firmware is + // given, and the state records, the M5 value (InitData::submission_effort). + let published = init.submission_effort(effort, state.waited()); + port.set_effort_full(effort == power::FULL_EFFORT); + if state.applied() == published && init.effort()? == published { return Ok(()); } - init.publish_effort(effort)?; - trace::message(trace::EFFORT, u64::from(effort)); - state.did_apply(effort); - port.set_effort_full(effort == power::FULL_EFFORT); + init.publish_effort(published)?; + trace::message(trace::EFFORT, u64::from(published)); + state.did_apply(published); Ok(()) } @@ -758,6 +918,10 @@ impl Firmware { if submitted { self.effort.submit(); } + // A waiter that began or ended since the last note moves the effort while work streams. + if self.effort.streaming() && self.effort.note_waiters(now_ns, fence::waiters()) { + self.apply_effort(power::FULL_EFFORT)?; + } let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { @@ -784,6 +948,12 @@ impl Firmware { /// refused control (recovery in progress) is retried on a later check. fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { const RETRY_NS: u64 = 1_000_000; + // The M5 firmware powers the cores for work by its own policy, and under a host + // hold J775D's core-power witness reads unpowered until the hold is released. M5 + // takes no host hold, whatever delay the tree or sysfs asks for. + if self.init.is_m5() { + return None; + } let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { match self @@ -846,6 +1016,27 @@ impl power::Control for Firmware { } } +/// Every feature the M5 command set reports. GET_PARAMS leaves out the MSAA compression +/// and mesh storage bits while those paths are off. +const M5_FEATURES: u64 = (uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_RESOURCES + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RESOURCES + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2) as u64 + | crate::file::DRM_ASAHI_FEATURE_SPARSE_TILES + | crate::file::DRM_ASAHI_FEATURE_COMPUTE_MATRIX + | crate::file::DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES + | crate::file::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 + | crate::file::DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS + | M5_MSAA_COMPRESSION_FEATURES + | crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; +const M5_MSAA_COMPRESSION_FEATURES: u64 = crate::file::DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION + | crate::file::DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION; +/// Features 24 and 25, which no kernel may report again (see the UAPI header): userspace +/// that knows them sends staged renders whose prefix fields the render command gives to +/// its ray-query stages. +const RETIRED_FEATURES: u64 = (uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES) as u64; +const _: () = assert!(M5_FEATURES & RETIRED_FEATURES == 0); + /// Event work item ID. const EVENT_WORK: u64 = 0; /// Short control/retirement polls do not wait behind the longer idle deadline. @@ -854,6 +1045,7 @@ const IDLE_WORK: u64 = 2; const GROW_WORK: u64 = 3; const RECLAIM_WORK: u64 = 4; const FEED_WORK: u64 = 5; +const GOVERNOR_WORK: u64 = 6; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] @@ -869,10 +1061,18 @@ struct Shared { variant: &'static fw::Variant, /// Layout of compute queues' operand-state buffers for this GPU's cores. operand_state: &'static fw::compute::OperandState, + /// The M5 board ([`Config::m5`]) of an M5 profile. + m5_board: Option<&'static fw::m5::initdata::Board>, /// Layout of renders' deflake buffers for this GPU's cores. deflake: fw::render::Deflake, compute_blocks: usize, descriptor_flags: [u32; 2], + /// Firmware events and work run: off for an M5 profile without the bare-metal boot. + submission_enabled: bool, + /// Such an M5 profile admits no render command. + firmware_test: bool, + /// The M5 shared-queue resource policy (frozen device topology). + render_policy: Option, crashed: AtomicBool, feed: Arc, #[pin] @@ -904,6 +1104,25 @@ struct Shared { poll: DelayedWork, #[pin] idle: DelayedWork, + /// M5: the thermal governor runs (the firmware has the host power budget word). + governs: bool, + #[pin] + governor: DelayedWork, + governor_armed: AtomicBool, + /// When work was last published while the governor is requested (0: none since its last + /// step). Publishers without the device mutex set it too. + governor_activity_ns: AtomicU64, + /// The queue of all work that signals job fences: the event work, and every scheduler's + /// timeout work. Memory reclaim may wait on those fences, so it is reclaim-safe. On M5 it is + /// unbound, so the event work runs in the affinity pod (the L2) of the CPU that last + /// published work, not on whichever CPU the completion IRQ thread first ran on; on a two-die + /// part that keeps completion processing on the submitter's die. Elsewhere it is per-CPU and + /// high priority, as `system_highpri_wq`. + events: Arc, + /// M5: the event work and the scheduler workers follow the submitting CPU. + follows_submitter: bool, + /// M5: the CPU of the last publication, `u32::MAX` before the first. + submitter: AtomicU32, #[pin] grow: Work, #[pin] @@ -918,6 +1137,10 @@ struct Shared { /// The device lock. `None` before boot completes and after detachment. #[pin] state: Mutex>>, + /// M5: the CPU frequency ceiling the thermal governor shares the SoC's heat with, and the + /// fan boost. Never taken with `state` held. + #[pin] + actuators: Mutex, #[pin] changed: CondVar, /// The port of the session in `state`; null before boot and once detachment begins. @@ -934,6 +1157,30 @@ impl_has_delayed_work! { impl_has_delayed_work! { impl HasDelayedWork for Shared { self.idle } } +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.governor } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.governor_armed.store(false, Ordering::Release); + // Sensor reads may sleep: take them before the device lock. + let temp = thermal::hotspot_temperature_mc(); + let fans = this.actuators.lock().fans.boosted(); + let (again, advice, want_fans) = { + let mut state = this.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => firmware.governor_tick(temp, fans), + None => (false, thermal::CpuAdvice::Release, false), + } + }; + // Frequency QoS updates and SMC accesses may sleep: outside the device lock. + this.actuators.lock().apply(advice, want_fans, temp, now_ns()); + if again { + this.queue_governor(); + } + } +} impl_has_work! { impl HasWork for Shared { self.grow } } @@ -1042,8 +1289,35 @@ impl Shared { /// during a pass. The pass skips the housekeeping that nothing needs (see /// `runtime_event_turn`). Callable from any context. fn queue_event_records(self: &Arc) { + if !self.submission_enabled { + return; + } // Already queued if this fails, which is just as good. - let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); + let _ = match self.submitter() { + Some(cpu) => self.events.enqueue_on::, EVENT_WORK>(cpu, self.clone()), + None => self.events.enqueue::, EVENT_WORK>(self.clone()), + }; + } + + /// The queue for the schedulers' timeout work, which signals the fences of timed-out jobs. + pub(super) fn fence_queue(&self) -> Arc { + self.events.clone() + } + + /// Whether the event work and the scheduler workers follow the submitting CPU (M5). + pub(super) fn follows_submitter(&self) -> bool { + self.follows_submitter + } + + /// Records the publishing CPU for the event work's placement (M5). + fn note_submitter(&self) { + if self.follows_submitter { + self.submitter.store(kernel::cpu::CpuId::current().as_u32(), Ordering::Relaxed); + } + } + + fn submitter(&self) -> Option { + kernel::cpu::CpuId::from_u32(self.submitter.load(Ordering::Relaxed)) } fn queue_grow(self: &Arc) { @@ -1089,6 +1363,23 @@ impl Shared { .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); } + fn queue_governor(self: &Arc) { + if !self.governor_armed.swap(true, Ordering::AcqRel) { + let _ = workqueue::system_unbound().enqueue_delayed::, GOVERNOR_WORK>( + self.clone(), msecs_to_jiffies(thermal::PERIOD_MS)); + } + } + + /// M5: work was published. Arms the thermal governor while it is requested. Called at + /// actual publication, by publishers with or without the device mutex; callable from any + /// context. + pub(super) fn note_gpu_activity(self: &Arc) { + if self.governs && thermal::requested() { + self.governor_activity_ns.store(now_ns().max(1), Ordering::Release); + self.queue_governor(); + } + } + fn queue_idle(self: &Arc, delay_ns: u64) { // Effort::idle bounds this by the 16 ms idle interval. Round up so a // timer never changes the firmware effort word before that deadline. @@ -1192,12 +1483,37 @@ impl Gpu { ); return Err(ENODEV); } + let m5 = cfg.profile.is_m5(); + if m5 { + let board = cfg.m5.ok_or(EINVAL)?; + // The M5 layouts are sized for every core slot of the board, fused-off cores + // included, so the GPU is matched on clusters and slots, not on enabled cores. + if id.num_clusters != board.clusters || id.num_clusters * id.num_cores != board.gpu_cores + { + dev_err!( + dev.as_ref(), + "M5 board expects {} clusters of {} core slots, the GPU reports {} x {}\n", + board.clusters, + board.gpu_cores / board.clusters, + id.num_clusters, + id.num_cores + ); + return Err(ENODEV); + } + fw::m5::pool::select_render_pool(board); + thermal::select_board(board); + pairs::select_board(board); + } let geometry = mmu::UatGeometry { ias: cfg.uat_ias, oas: cfg.uat_oas, }; - let max_frequency_khz = Self::max_frequency_khz(dev)?; + let max_frequency_khz = if m5 { + cfg.m5.ok_or(EINVAL)?.perf_states[13].0 * 1000 + } else { + Self::max_frequency_khz(dev)? + }; let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; let idle_delay = power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; @@ -1213,7 +1529,7 @@ impl Gpu { let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; - let feed = feed::Feed::new()?; + let feed = feed::Feed::new(m5)?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { clusters == id.num_clusters @@ -1244,6 +1560,17 @@ impl Gpu { "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" ); } + let render_policy = if m5 { Some(render_resources::M5Policy::new(&id)?) } else { None }; + // The bare-metal M5 boot runs both engines; other M5 profiles take no submission. + let submits = !m5 || cfg.profile == FirmwareProfile::M5PublishedRoots; + let events = Arc::new( + if m5 { + workqueue::OwnedQueue::new_unbound_reclaim(c"aurora_g17_events", true)? + } else { + workqueue::OwnedQueue::new_percpu_reclaim(c"aurora_g17_events", true)? + }, + GFP_KERNEL, + )?; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1252,9 +1579,13 @@ impl Gpu { active_cores, variant, operand_state: &cfg.core_layout.operand_state, + m5_board: cfg.m5.filter(|_| m5), deflake: cfg.core_layout.deflake, compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], + submission_enabled: submits, + firmware_test: !submits, + render_policy, crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), @@ -1269,11 +1600,19 @@ impl Gpu { housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), + governs: m5, + governor <- new_delayed_work!("g17::Shared::governor"), + governor_armed: AtomicBool::new(false), + governor_activity_ns: AtomicU64::new(0), + events, + follows_submitter: m5, + submitter: AtomicU32::new(u32::MAX), grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), + actuators <- new_mutex!(thermal::Actuators::new(), "g17::Shared::actuators"), changed <- new_condvar!("g17::Shared::changed"), port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), @@ -1315,7 +1654,7 @@ impl Gpu { }, GFP_KERNEL, )?; - let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.id, &gpu.shared)?; gpu.shared .port .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); @@ -1396,6 +1735,175 @@ impl Gpu { Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) } + /// Starts the M5 firmware on bare metal: both roles boot and publish their kernel roots, + /// the M5 graph is built in the kernel window they share, then the M5 preparation, + /// InitData to the primary and then the secondary, POWER_ON_DONE to both, and both opening + /// records retired. + fn boot_m5( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + cfg: &'static Config, + regs: regs::G17Resources, + geometry: mmu::UatGeometry, + id: ®s::G17Id, + shared: &Arc, + ) -> Result> { + if cfg.profile != FirmwareProfile::M5PublishedRoots { + return Err(ENODEV); + } + let board = cfg.m5.ok_or(EINVAL)?; + // Names the step a failed bootstrap stopped at. + let at = |step: &'static str| { + move |e: &Error| { + dev_err!(dev.as_ref(), "M5 bare-metal bootstrap failed at {}: {:?}\n", step, e) + } + }; + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + geometry, + true, + mmu::UatFirmware::PublishedRoots { + secondary_root_offset: cfg.secondary_root_offset, + }, + cfg.uat_quirks, + ) + .inspect_err(at("UAT"))? + .with_m5(), + GFP_KERNEL, + )?; + // The firmware samples the kernel lower root's top-level entries only when it starts: + // link the tables of every lower range the M5 graph and its queues use first. + uat.prepare_lower_tables(&initdata::InitData::m5_lower_links( + uat.kernel_lower_dynamic_range(), + )) + .inspect_err(at("lower links"))?; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?; + let render_global = uat.new_vm(0, geometry.user_range()).inspect_err(at("render VM"))?; + let metrics_ids = Arc::new( + buffer::MetricsIds::with_stride(64).inspect_err(at("metrics IDs"))?, + GFP_KERNEL, + )?; + let metrics = Arc::new( + KernelObject::new( + dev, + &render_global, + Placement::In(buffer::BlockGeometry::M5_COMPACT, mmu::UAT_PGSZ as u64), + metrics_ids.storage_bytes()?, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + ) + .inspect_err(at("metrics"))?, + GFP_KERNEL, + )?; + let session_metrics = metrics.clone(); + let firmware = KBox::try_init( + kernel::try_init!(Firmware { + secondary: Some(secondary), + primary, + bridge: Bridge::unused(regs), + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, + seen_submissions: 0, + rings: channel::Rings::new(shared.variant.graph)?, + recovery: recovery::State::new(), + effort: power::Effort::new(), + hold: power::Hold::new(), + governor: thermal::Governor::new(), + registration_power_release_pending: false, + variant: shared.variant, + queues <- runtime::Registry::with_waiters( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), + ), + pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, + render_ids: freelist::RenderIds::new()?, + init: initdata::InitData::new(), + _render_global: render_global, + _pm_metrics: session_metrics, + uat, + }), + GFP_KERNEL, + ) + .inspect_err(at("session"))?; + // Once a role runs, a stopped CPU is not proof that no engine still reads what it was + // given: a failed bootstrap keeps the session, nothing is freed or unmapped. A kept + // session's callbacks run this module's code, so the module is pinned first; once its + // removal has begun nothing is started and the session is freed unstarted. + retain_session().inspect_err(at("module pin"))?; + let mut retained = core::mem::ManuallyDrop::new(firmware); + let session = &mut **retained; + // M5 renders take 64-byte metrics records. + session.queues.metrics = metrics_ids; + session.primary.boot().inspect_err(at("primary boot"))?; + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.boot().inspect_err(at("secondary boot"))?; + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = session + .uat + .adopt_firmware_root_entries() + .inspect_err(at("root adoption"))?; + session + .init + .build_m5(dev, &session.uat, id, board) + .inspect_err(at("M5 graph"))?; + session + .init + .bind_m5_metrics(&session.uat, &metrics) + .inspect_err(at("metrics binding"))?; + // Publishers reach the mailbox, QoS table and recovery word through the port, as on + // G17P; M5's rings are the shared ones its InitData views name. + let mailbox: *const Mailbox = &*session.primary.rtkit; + session + .port + .attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + shared.variant.registration, + !shared.variant.m5_commands, + ) + .inspect_err(at("port"))?; + session + .uat + .confirm_firmware_root_entries(root_entries) + .inspect_err(at("root confirmation"))?; + session.uat.mirror_secondary_root().inspect_err(at("secondary root"))?; + session + .bridge + .regs + .prepare_m5_firmware() + .inspect_err(at("M5 preparation"))?; + let ack = shared.variant.init_ack; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack) + .inspect_err(at("primary InitData"))?; + dev_info!(dev.as_ref(), "M5 primary initdata acknowledged\n"); + let root_va = session.init.root_va(Role::Secondary)?; + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.init(root_va, ack).inspect_err(at("secondary InitData"))?; + dev_info!(dev.as_ref(), "M5 secondary initdata acknowledged\n"); + session.primary.notify(MSG_POWER_ON_DONE)?; + secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref()) + .inspect_err(at("opening records"))?; + dev_info!(dev.as_ref(), "M5 firmware opening records retired\n"); + Ok(core::mem::ManuallyDrop::into_inner(retained)) + } + /// Starts both coprocessors and initializes their firmware. fn boot( pdev: &platform::Device>, @@ -1403,8 +1911,12 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, geometry: mmu::UatGeometry, + id: ®s::G17Id, shared: &Arc, ) -> Result> { + if cfg.profile.is_m5() { + return Self::boot_m5(pdev, dev, cfg, regs, geometry, id, shared); + } let uat = Arc::new( mmu::Uat::new( dev.into(), @@ -1423,8 +1935,8 @@ impl Gpu { let pm_metrics = KernelObject::new( dev, uat.kernel_lower_vm(), - Placement::At(cfg.pm_metrics.va), - cfg.pm_metrics.size, + Placement::At(cfg.pm_metrics()?.va), + cfg.pm_metrics()?.size, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; @@ -1469,6 +1981,7 @@ impl Gpu { recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), + governor: thermal::Governor::new(), registration_power_release_pending: false, variant: shared.variant, queues <- runtime::Registry::with_waiters( @@ -1481,7 +1994,7 @@ impl Gpu { ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, - render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, + render_ids: freelist::RenderIds::new()?, init: initdata::InitData::new(), _render_global: render_global, _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, @@ -1518,6 +2031,7 @@ impl Gpu { session.init.port_views(mailbox)?, session.bridge.regs.clone(), shared.variant.registration, + !shared.variant.m5_commands, )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { @@ -1628,6 +2142,11 @@ impl Gpu { self.shared.stop_queues(); self.shared.disable_timer(&self.shared.poll); self.shared.disable_timer(&self.shared.idle); + self.shared.disable_timer(&self.shared.governor); + if let Some(firmware) = (*self.shared.state.lock()).as_deref_mut() { + firmware.governor_restore(); + } + self.shared.actuators.lock().release(); self.shared.disable_work(&self.shared.grow); self.shared.disable_work(&self.shared.reclaim_work); self.shared.disable_work(&self.shared.work); @@ -1676,16 +2195,29 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; + // The M5 command set has no AS-build flag: its validation refuses one. + if !variant.m5_commands { + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + } if variant.few_primitives { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } - if variant.ray_traversal { + if variant.render_ray_query { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); } + if variant.m5_commands { + let mut features = M5_FEATURES; + if !queue::render::m5_render_submission::depth_msaa_compression_enabled() { + features &= !M5_MSAA_COMPRESSION_FEATURES; + } + if crate::module_parameters::m5_mesh_storage_chunks.value() == 0 { + features &= !crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; + } + params.features |= features; + } params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; @@ -1727,7 +2259,7 @@ impl gpu::Gpu for Gpu { fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { max_queues: kick::queue_limit(self.shared.variant.qid_floor), - max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, + max_in_flight_per_queue: pairs::per_owner().into(), ..Default::default() }) } @@ -1744,9 +2276,23 @@ impl gpu::Gpu for Gpu { .uat .new_vm(self.ids.vm.next(), kernel_range)? .with_status()?; + if self.cfg.profile.is_m5() { + // The canonical arena the client's queues alias, the client-private one and the + // compact window, where the kernel places the parameter buffers' page lists, render + // targets, scratch and metrics records, stay reserved against userspace binds from + // creation, so no bind replaces a kernel alias the firmware reads. The client's own + // kernel range need not cover the compact window. + let ranges = firmware.uat.m5_client_reserved_ranges(); + let mut reserved = KVec::with_capacity(ranges.len() + 1, GFP_KERNEL)?; + for range in Iterator::chain(ranges.into_iter(), [buffer::BlockGeometry::M5_COMPACT]) { + reserved.push(range, GFP_KERNEL)?; + } + vm.install_driver_mappings(KVec::new(), reserved)?; + return Ok(vm); + } let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; let ranges = - hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; + hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics()?.va).ok_or(EINVAL)?; let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; for range in ranges { reserved.push(range, GFP_KERNEL)?; @@ -1772,7 +2318,9 @@ impl gpu::Gpu for Gpu { self.geometry.user_usable_range(), self.shared.variant, )?; - let policy = scheduling_policy(priority)?; + let policy = if self.cfg.profile.is_m5() { + fw::m5::queue::scheduling_policy(priority)? + } else { scheduling_policy(priority)? }; let admission_started = now_ns(); let (backend, slot) = loop { let observed = self.shared.queues.observe(); @@ -1823,8 +2371,7 @@ impl gpu::Gpu for Gpu { let mapping = { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + let addresses = firmware.uat.kernel_dynamic_range(); bo.map_range_into_range( firmware.uat.kernel_vm(), range, diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index aa30da6fcb41e..0b9854e84de2c 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -75,9 +75,13 @@ impl KernelObject { prot: mmu::Prot, cpu: CpuMap, ) -> Result { - let mut this = Self::backing(dev, size, cpu)?; + Self::backing(dev, size, cpu)?.placed(vm, placement, prot) + } + + /// Maps fresh backing into `vm`. + fn placed(mut self, vm: &mmu::Vm, placement: Placement, prot: mmu::Prot) -> Result { let mapping = match placement { - Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, + Placement::At(addr) => self.obj.map_at(vm, addr, prot, false)?, Placement::In(range, align) => { let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start && range.end <= crate::hw::t8140::dynamic::LOWER.end { @@ -85,21 +89,24 @@ impl KernelObject { } else { super::capacity::Site::UpperVa }; - this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + self.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { if error == ENOSPC { super::capacity::refused(site) } else { error } })? } }; - this.mapping = Some(mapping); - Ok(this) + self.mapping = Some(mapping); + Ok(self) } /// Allocates CPU-accessible backing whose GPU mappings will be installed later. pub(crate) fn backing(dev: &AsahiDrmDevice, size: usize, cpu: CpuMap) -> Result { - let mut obj = match cpu { + Self::with_object(match cpu { CpuMap::WriteBack => gem::new_kernel_object(dev, size)?, CpuMap::WriteCombined => gem::new_kernel_object_wc(dev, size)?, - }; + }, size) + } + + fn with_object(mut obj: gem::ObjectRef, size: usize) -> Result { let vmap = obj.vmap()?; let ptr = NonNull::new(vmap.addr.cast::()).ok_or(ENOMEM)?; // SAFETY: The mapping covers the whole object. @@ -229,6 +236,13 @@ impl KernelObject { self.mapping.take().ok_or(EINVAL) } + /// Publishes the page-table updates of the object's own mapping to every context naming its + /// VM's root (see [`mmu::KernelMapping::publish_to_contexts`]). + pub(crate) fn publish_to_contexts(&self) -> Result { + self.mapping.as_ref().ok_or(EINVAL)?.publish_to_contexts(); + Ok(()) + } + /// Returns the GPU address of the object. pub(crate) fn gpu_va(&self) -> u64 { self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) @@ -338,8 +352,7 @@ impl Allocator<'_> { prot: mmu::Prot, cpu: CpuMap, ) -> Result { - let mut range = self.uat.geometry().kernel_range(); - range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + let range = self.uat.kernel_dynamic_range(); KernelObject::new( self.dev, self.uat.kernel_vm(), @@ -362,13 +375,29 @@ impl Allocator<'_> { KernelObject::new( self.dev, self.uat.kernel_lower_vm(), - Placement::In(crate::hw::t8140::dynamic::LOWER, align), + Placement::In(self.uat.kernel_lower_dynamic_range(), align), size, prot, cpu, ) } + /// [`Self::lower`] write-combined backing that is not counted as kernel memory: the + /// caller charges it to a client ([`crate::memacct::Charge::user`]). + pub(crate) fn lower_user_charged( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + ) -> Result { + KernelObject::with_object(gem::new_kernel_object_wc_user_charged(self.dev, size)?, size)? + .placed( + self.uat.kernel_lower_vm(), + Placement::In(self.uat.kernel_lower_dynamic_range(), align), + prot, + ) + } + /// Place fresh backing at one address in the canonical and both alias VMs. /// A client may retain mappings after their canonical reservation is freed. /// Retry only address collisions, keeping the same backing throughout. @@ -448,9 +477,11 @@ pub(crate) enum Family { WorkState = 1, QueueState = 2, QueueConfig = 3, + /// An M5 context's scheduler page, whose mapping the M5 firmware caches. + Scheduler = 4, } -const POOL_FAMILIES: usize = 4; +const POOL_FAMILIES: usize = 5; const POOL_SLOTS: usize = 64; const POOL_BYTES: usize = 128 * 1024 * 1024; diff --git a/drivers/gpu/drm/asahi/g17/pairs.rs b/drivers/gpu/drm/asahi/g17/pairs.rs new file mode 100644 index 0000000000000..c7515c1c76417 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/pairs.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render pairs one logical render queue may hold: two on the G17D (M5 Ultra), one elsewhere. +//! +//! A second pair lets one queue's depth run on two physical render rings. It comes only from +//! surplus: the QID budget ([`super::kick::render_reserve`]) keeps counting one pair per open +//! queue, admission claims a sibling only while a pair can still be built, and an unused +//! sibling is reassigned to an owner that needs a first pair. An owner's overlapping renders on +//! its two pairs are ordered in firmware ([`super::hazard::plan`]). +//! +//! A queue samples the count once, when it is created ([`per_owner`]): its scheduler credits, +//! render pool and pair limit all follow that one value. + +use core::sync::atomic::{AtomicU8, Ordering::Relaxed}; + +use crate::hw::t8140::queues::RENDER_PAIRS_MAX; + +/// G17D's count. +const G17D_PAIRS: u8 = RENDER_PAIRS_MAX as u8; + +static PAIRS: AtomicU8 = AtomicU8::new(1); + +/// The count for the probed board, set once at probe (one GPU per system). +pub(crate) fn select_board(board: &crate::g17::fw::m5::initdata::Board) { + PAIRS.store(if board.g17d_forms() { G17D_PAIRS } else { 1 }, Relaxed); +} + +/// Render pairs a logical render queue created now may hold. +pub(crate) fn per_owner() -> u8 { + PAIRS.load(Relaxed) +} + +/// Renders a busy pair of a two-pair owner takes before its next render goes to +/// an idle sibling: 1 keeps a queue at depth 1-2 on one pair. +const SIBLING_BACKLOG: u8 = 1; +static BACKLOG: AtomicU8 = AtomicU8::new(SIBLING_BACKLOG); + +/// A two-pair owner's next render stays on its least-loaded busy pair while +/// that pair holds at most this many renders. +pub(crate) fn sibling_backlog() -> u8 { + BACKLOG.load(Relaxed) +} + +#[cfg(CONFIG_KUNIT = "y")] +pub(crate) fn set_sibling_backlog(renders: u8) { + BACKLOG.store(renders, Relaxed); +} diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 425eb6f517b71..be27fedaca625 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -188,13 +188,16 @@ impl Port { views: Views, regs: crate::regs::G17Resources, registration: super::fw::variant::QueueRegistration, + clamp_rx: bool, ) -> Result { // SAFETY: The session owns the mailbox and outlives every port user. let rx = unsafe { (*views.mailbox).rx_util_min_handle() }; + // Without `clamp_rx` (M5) no handle is attached and the receive thread keeps its + // default; rx_update() then returns at once. if self.views.populate(views) && self.regs.populate(regs) && self.registration.populate(registration) - && self.rx.populate(rx) + && (!clamp_rx || self.rx.populate(rx)) { Ok(()) } else { @@ -453,6 +456,16 @@ impl Port { self.qos.lock().cancel(&view, publication) } + /// Clears the QoS table rows of released QoS IDs (`QosIds::forget_released`); returns + /// those left because work still runs on them. + pub(super) fn qos_forget(&self, ids: u128) -> Result { + if ids == 0 { + return Ok(0); + } + let view = self.qos_view()?; + Ok(self.qos.lock().forget(&view, ids)) + } + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { let view = self.qos_view()?; self.qos.lock().complete(&view, owners)?; @@ -460,10 +473,10 @@ impl Port { Ok(()) } - /// Requests the receive thread's minimum utilization for the feed's current view of the GPU. - /// Called at every publication and completion: the feed's state lingers for a few - /// milliseconds, so after a pause the publication that ends it restores the default before - /// its completion arrives. + /// Requests the receive thread's minimum utilization for the feed's current view of the GPU, + /// where the port clamps it (not on M5, see attach()). Called at every publication and + /// completion: the feed's state lingers for a few milliseconds, so after a pause the + /// publication that ends it restores the default before its completion arrives. fn rx_update(&self) { let Some(rx) = self.rx.as_ref() else { return; diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index a2e293df91510..5c57f6a289440 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -40,6 +40,13 @@ pub(super) struct Effort { idle_since: Option, applied: u32, submissions: u64, + /// A host thread waits, or waited within [`IDLE_HOLD_NS`], for submitted work + /// ([`Self::note_waiters`]). + waited: bool, + /// The waits begun when the waiters were last noted. + seen_waits: u64, + /// When a waiter was last seen. + last_wait_ns: u64, } impl Effort { @@ -49,9 +56,36 @@ impl Effort { idle_since: None, applied: 0, submissions: 0, + waited: false, + seen_waits: 0, + last_wait_ns: 0, } } + pub(super) fn streaming(&self) -> bool { + self.streaming + } + + /// Whether a host thread waits for the work ([`Self::note_waiters`]). + pub(super) fn waited(&self) -> bool { + self.waited + } + + /// Notes the waiters on submitted work at `now_ns` ([`super::fence::waiters`]): a thread + /// waits now or began a wait since the last note. The note holds for [`IDLE_HOLD_NS`] after + /// the last waiter, across a submit-and-wait loop's turns between its waits. Returns whether + /// that changed. + pub(super) fn note_waiters(&mut self, now_ns: u64, (waiters, waits): (u32, u64)) -> bool { + if waiters > 0 || waits != self.seen_waits { + self.last_wait_ns = now_ns.max(1); + } + self.seen_waits = waits; + let waited = self.last_wait_ns != 0 && now_ns.saturating_sub(self.last_wait_ns) < IDLE_HOLD_NS; + let changed = waited != self.waited; + self.waited = waited; + changed + } + pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; @@ -84,6 +118,8 @@ impl Effort { let idle = now_ns.saturating_sub(since); if idle >= IDLE_HOLD_NS { self.streaming = false; + self.waited = false; + self.last_wait_ns = 0; self.idle_since = None; IdleAction::Clear } else { diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index c7d1761699a24..85994ab8e6c73 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -30,6 +30,10 @@ pub(super) struct Admission<'a> { check: &'a dyn Fn() -> Result, } +/// Bound for a compute publication's waits: a physical compute queue, or room on the shared +/// work ring, frees only as other owners' work retires or goes idle. +pub(super) const QUEUE_WAIT_NS: u64 = 30_000_000_000; + impl<'a> Admission<'a> { pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { Self { deadline: Cell::new(deadline), started: super::now_ns(), check } diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index b36656b0174a7..a90fe51539cb1 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -237,6 +237,29 @@ impl Accounting { self.complete(view, &[ticket.owner]) } + /// Clears the scheduler and share rows of the QoS IDs in `ids`, which no context holds: a + /// released context's row otherwise keeps naming its scheduler page, which the pool may + /// already have given to another process's context. Rows of IDs with work in flight are + /// left alone and returned, for a later call once that work completed. + pub(crate) fn forget(&self, view: &View<'_>, ids: u128) -> u128 { + let mut left = 0; + for qos in 0..fw::QUEUES.min(128) { + if ids & (1u128 << qos) == 0 { + continue; + } + if self.in_flight.iter().zip(view.rows.iter()) + .any(|(&count, row)| count != 0 && usize::from(row[0].load(Ordering::Relaxed)) == qos) + { + left |= 1u128 << qos; + continue; + } + view.scheduler[qos].store(0, Ordering::Relaxed); + view.share[qos].store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + left + } + /// Preflights the entire set before clearing any final row. QID membership, /// scheduler mappings and monotonically increasing counts remain intact. pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 9337ba9de281f..6fd8f13730c5c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -11,7 +11,7 @@ use crate::g17::{ fw::kick::KickDependency, hazard::{self, Footprint, Plan}, - job::Packet, + job::{Order, Packet}, }; use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; use core::sync::atomic::{AtomicU32, Ordering}; @@ -107,12 +107,25 @@ impl State { fn exhausted(&self) -> bool { self.quarantined & Self::MASK == Self::MASK } + /// The owner's quarantined pairs, if any: `Some(true)` when every one of + /// them is spared (quarantined behind another VM's recovery). One + /// quarantined pair stops the owner on all of its pairs, so a healthy + /// sibling never bypasses it. + fn owner_quarantine(&self, owner: u64) -> Option { + let owned = (0..cfg::RENDER_SLOTS) + .filter(|slot| self.owners[*slot] == owner) + .fold(0u64, |mask, slot| mask | 1u64 << slot); + let quarantined = self.quarantined & owned; + (quarantined != 0).then_some(quarantined & !self.spared == 0) + } fn refusal(&self, owner: u64) -> Result { if self.exhausted() { return Err(EIO); } + if self.owner_quarantine(owner).is_some() { + return Err(EIO); + } match self.owners.iter().position(|id| *id == owner) { - Some(slot) if self.quarantined & (1u64 << slot) != 0 => Err(EIO), None if !self.owners[1..].contains(&0) && self.released & !self.active @@ -147,10 +160,28 @@ impl State { self.advance(slot); slot as u8 } - fn acquire(&mut self, owner: u64) -> Option { + fn acquire(&mut self, owner: u64, pairs: usize) -> Option { if owner == 0 { return None; } + // A shallow queue stays on its busy pair: a second pair pays off only + // once the first has a backlog (at depth 2 a switch on every render + // cost a 1080p fill about 45 us a submit on M5 Ultra). + if pairs > 1 { + let busy = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] != 0 + && self.active & (1u64 << slot) != 0 + && self.usable(owner, *slot) + }) + .min_by_key(|slot| (self.refs[*slot], u64::MAX - self.epochs[*slot])); + if let Some(slot) = busy.filter(|slot| self.refs[*slot] <= crate::g17::pairs::sibling_backlog()) { + self.refs[slot] += 1; + self.advance(slot); + return Some(slot as u8); + } + } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. let idle = (0..cfg::RENDER_SLOTS) @@ -164,7 +195,7 @@ impl State { if !(0..cfg::RENDER_SLOTS) .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - self.claim_sibling(owner); + self.claim_sibling(owner, pairs); } return Some(self.take_idle(slot, owner)); } @@ -203,11 +234,12 @@ impl State { }) .or_else(unowned)?; let first = self.take_idle(slot, owner); - self.claim_sibling(owner); + self.claim_sibling(owner, pairs); Some(first) } - fn claim_sibling(&mut self, owner: u64) { - if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + /// Claims another pair for an owner that may hold `pairs`. + fn claim_sibling(&mut self, owner: u64, pairs: usize) { + if self.owners.iter().filter(|id| **id == owner).count() >= pairs { return; } // A speculative sibling never takes a killed context's pair. @@ -238,6 +270,8 @@ impl State { /// The device lock separately proves the physical graph was never published. /// An open owner keeps another usable pair. Exclude every live borrower, /// including off-lock sibling construction; a closed owner needs no spare. + /// A spare goes only to an owner without a pair: a second pair comes only + /// from surplus, through claim_sibling. fn reassign_unused_spare( &mut self, slot: u8, @@ -249,8 +283,7 @@ impl State { || slot >= cfg::RENDER_SLOTS || owner.is_some_and(|owner| { owner == 0 - || self.owners.iter().filter(|value| **value == owner).count() - >= cfg::RENDER_PAIRS_PER_OWNER + || self.owners.contains(&owner) }) { return false; @@ -336,14 +369,11 @@ impl State { /// Whether `pick` would return a slot, without taking it. fn can_pick(&self, owner: u64, plan: Plan) -> bool { - let usable = |slot: usize| { - self.owners[slot] == owner - && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 - && self.refs[slot] < cfg::RENDER_DEPTH - }; + let usable = |slot: usize| self.usable(owner, slot); match plan { Plan::Wait => false, - Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Pair(slot) => usable(usize::from(slot)), + Plan::Join(_) => (0..cfg::RENDER_SLOTS).any(usable), Plan::Any if owner == 0 => false, Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), Plan::Any => { @@ -356,26 +386,36 @@ impl State { } } - fn pick(&mut self, owner: u64, plan: Plan) -> Option { + fn pick(&mut self, owner: u64, pairs: usize, plan: Plan) -> Option { match plan { - Plan::Any => self.acquire(owner), + Plan::Any => self.acquire(owner, pairs), Plan::Wait => None, - Plan::Pair(slot) => { - let index = usize::from(slot); - if self.owners.get(index) != Some(&owner) - || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0 - || self.refs[index] >= cfg::RENDER_DEPTH - { - return None; - } - if self.refs[index] == 0 { - Some(self.take_idle(index, owner)) - } else { - self.refs[index] += 1; - self.advance(index); - Some(slot) - } - } + Plan::Pair(slot) => self.join(owner, usize::from(slot)), + // The preferred pair, else the owner's least-loaded other pair: + // the publication barrier orders the overlap from there. + Plan::Join(slot) => self.join(owner, usize::from(slot)).or_else(|| { + let other = (0..cfg::RENDER_SLOTS) + .filter(|other| *other != usize::from(slot) && self.usable(owner, *other)) + .min_by_key(|other| (self.refs[*other], u64::MAX - self.epochs[*other]))?; + self.join(owner, other) + }), + } + } + fn usable(&self, owner: u64, slot: usize) -> bool { + self.owners.get(slot) == Some(&owner) + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + } + fn join(&mut self, owner: u64, index: usize) -> Option { + if !self.usable(owner, index) { + return None; + } + if self.refs[index] == 0 { + Some(self.take_idle(index, owner)) + } else { + self.refs[index] += 1; + self.advance(index); + Some(index as u8) } } @@ -501,17 +541,14 @@ impl Pool { } /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { - let state = self.state.lock(); - state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) + self.state.lock().owner_quarantine(owner) == Some(true) } + /// For an owner that may hold `pairs` render pairs. pub(crate) fn acquire_render( self: &Arc, owner: u64, + pairs: usize, request: Option>, deadline_ns: u64, check: impl Fn() -> Result, @@ -532,22 +569,12 @@ impl Pool { if state.exhausted() { return Err(EIO); } - // The first retained pair's quarantine is not bypassed through its + // A retained pair's quarantine is not bypassed through its // healthy sibling. Reclamation wakes this bounded admission wait. - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { + if let Some(spared) = state.owner_quarantine(owner) { // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the // pair to drain until the caller's deadline, which the caller extends while it // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); if now >= quarantine_deadline && !spared { return Err(EIO); } @@ -570,7 +597,7 @@ impl Pool { } else { state.plan(owner, request, ticket) }; - if let Some(slot) = state.pick(owner, plan) { + if let Some(slot) = state.pick(owner, pairs, plan) { state.forget_waiter(ticket.take()); let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -613,6 +640,7 @@ impl Pool { pub(crate) fn try_acquire_render( self: &Arc, owner: u64, + pairs: usize, request: Option>, ) -> Result> { if owner == 0 { @@ -623,7 +651,7 @@ impl Pool { return Ok(None); } let plan = state.plan(owner, request, None); - let Some(slot) = state.pick(owner, plan) else { + let Some(slot) = state.pick(owner, pairs, plan) else { return Ok(None); }; let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); @@ -640,12 +668,7 @@ impl Pool { if owner == 0 || state.refusal(owner).is_err() { return false; } - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { + if state.owner_quarantine(owner).is_some() { return false; } let plan = state.plan(owner, request, None); @@ -735,7 +758,7 @@ mod accounting_tests { fn every_logical_owner_fits_without_speculative_spares() { let mut state = State::new(); for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - let slot = state.acquire(owner).unwrap(); + let slot = state.acquire(owner, 1).unwrap(); assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); state.release(slot); } @@ -746,7 +769,7 @@ mod accounting_tests { #[test] fn close_does_not_reuse_an_active_lease() { let mut state = State::new(); - let slot = state.acquire(1).unwrap(); + let slot = state.acquire(1, 1).unwrap(); state.released |= 1u64 << slot; assert!(!state.reassign_unused_spare(slot, Some(1), None)); state.release(slot); @@ -759,7 +782,7 @@ mod accounting_tests { #[test] fn warming_quarantine_and_exit_hold_exclude_reassignment() { let mut state = State::new(); - let primary = state.acquire(1).unwrap(); + let primary = state.acquire(1, 1).unwrap(); // A retained legacy sibling still obeys all quarantine/kill guards. state.owners[2] = 1; let spare = state.borrow_warm_sibling(1, primary).unwrap(); @@ -805,6 +828,19 @@ impl QueueSlot { pub(crate) fn pool(&self) -> &Arc { &self.pool } + /// Gives back a slot no queue was published in, for a creator that sampled the waiters' + /// domain as `observed` before claiming it. Returns the sample that creator waits on next + /// ([`Availability::notify_own`]): its own return still wakes the other waiters, but not + /// itself. + /// + /// [`Availability::notify_own`]: crate::g17::availability::Availability::notify_own + pub(crate) fn cancel(self, observed: u64) -> u64 { + let this = core::mem::ManuallyDrop::new(self); + // SAFETY: `this` is never used or dropped again, so the pool reference moves out once. + let pool = unsafe { core::ptr::read(&this.pool) }; + pool.queues.fetch_sub(1, Ordering::AcqRel); + pool.waiters.as_ref().map_or(observed, |waiters| waiters.notify_own(observed)) + } } impl Drop for QueueSlot { fn drop(&mut self) { @@ -879,11 +915,20 @@ impl Pool { /// Under the device mutex, which orders every publication: the newest /// published fragment kick of each other pair whose live same-VM render /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + /// + /// That includes the owner's own other pair, for its renders that `prefix` + /// (the render's queue prefix) does not already order: an owner's renders + /// on its two pairs run on different queues (see [`hazard::plan`]). An + /// owner publishes in submission order (the backlog keeps inline + /// publication behind queued commands, and the ordered run_job worker + /// publishes one command at a time), so its earlier renders already carry + /// their fragment kick here. pub(crate) fn publication_barriers( &self, owner: u64, slot: u8, footprint: &Footprint, + prefix: Option, barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], ) -> Result { let state = self.state.lock(); @@ -894,7 +939,7 @@ impl Pool { continue; }; if other == slot - || entry.owner == owner + || (entry.owner == owner && Order::contains(prefix, entry.sequence)) || state.quarantined & (1u64 << other) != 0 || !footprint.same_vm(&entry.footprint) || !footprint.overlaps(&entry.footprint) @@ -1011,6 +1056,35 @@ impl Pool { } } +#[kunit_tests(asahi_g17_queue_slot_cancel)] +mod queue_slot_cancel_tests { + use super::*; + use crate::g17::availability::Availability; + + #[test] + fn a_creator_does_not_wake_itself_by_giving_back_its_slot() -> Result { + let waiters = Availability::new()?; + let pool = Pool::with_waiters(MAX_EXECUTION_CONTEXTS, Some(waiters.clone()), None)?; + let observed = waiters.observe(); + let slot = pool.reserve_queue()?; + let next = slot.cancel(observed); + assert_eq!(pool.queues(), 0); + // The return advanced the domain, so other waiters wake, but the creator's next + // sample is current: its wait registers instead of returning at once. + assert_eq!(next, waiters.observe()); + assert_ne!(next, observed); + assert!(waiters.wait(next)?.is_some()); + // A return by anyone else after the sample still ends the creator's wait. + let observed = waiters.observe(); + let slot = pool.reserve_queue()?; + drop(pool.reserve_queue()?); + assert_eq!(slot.cancel(observed), observed); + assert!(waiters.wait(observed)?.is_none()); + waiters.notify(); + Ok(()) + } +} + #[kunit_tests(asahi_g17_fault_admission)] mod fault_capacity_tests { use super::*; @@ -1019,7 +1093,7 @@ mod fault_capacity_tests { fn a_guilty_failure_makes_a_spared_quarantine_guilty() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); assert!(!pool.spared_quarantined(1)); lease.quarantine(true); @@ -1045,7 +1119,7 @@ mod fault_capacity_tests { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); for owner in 1..=512 { let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); lease.quarantine(false); pool.release_owner(owner); let bit = 1u64 << lease.slot(); @@ -1066,7 +1140,7 @@ mod fault_capacity_tests { #[test] fn published_exit_kill_holds_survive_close_and_drain() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); pool.release_owner(1); drop(lease); @@ -1081,7 +1155,7 @@ mod fault_capacity_tests { #[test] fn consumed_scheduler_release_ends_the_kill_hold() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let lease = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); let slot = lease.slot(); let bit = 1u64 << slot; pool.release_owner(1); @@ -1095,7 +1169,7 @@ mod fault_capacity_tests { // The killed context's pair is never reassigned as a spare, and a new owner takes // a new graph before it. assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); - let next = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + let next = pool.acquire_render(2, 1, None, u64::MAX, || Ok(())).unwrap(); assert_ne!(next.slot(), slot); drop(next); pool.release_retained_owners_after_processor_stop(); @@ -1116,9 +1190,9 @@ mod fault_capacity_tests { // One pair released without a kill. state.killed &= !(1u64 << 7); } - let first = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let first = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); assert_eq!(first.slot(), 7); - let second = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + let second = pool.acquire_render(2, 1, None, u64::MAX, || Ok(())).unwrap(); assert_eq!(usize::from(second.slot()), cfg::RENDER_SLOTS - 1); assert_eq!(pool.state.lock().killed & (1u64 << second.slot()), 0); } @@ -1136,11 +1210,11 @@ mod fault_capacity_tests { } } // With room in the queue table a new owner builds a pair in an unowned slot. - let built = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let built = pool.acquire_render(1, 1, None, u64::MAX, || Ok(())).unwrap(); assert!(usize::from(built.slot()) >= 8); // Without room it takes the killed pair released longest ago. pool.note_build_room(false); - let reused = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + let reused = pool.acquire_render(2, 1, None, u64::MAX, || Ok(())).unwrap(); assert_eq!(reused.slot(), 7); assert_eq!(pool.state.lock().killed & (1u64 << 7), 0); // It claims no sibling: neither another killed pair nor an unowned slot. @@ -1165,12 +1239,164 @@ mod shared_depth_tests { #[test] fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { let mut state = State::new(); - let first = state.acquire(1).unwrap(); - for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } - assert_eq!(state.acquire(1), None); + let first = state.acquire(1, 1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1, 1), Some(first)); } + assert_eq!(state.acquire(1, 1), None); assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); state.release(first); - assert_eq!(state.acquire(1), Some(first)); + assert_eq!(state.acquire(1, 1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_owner_pairs)] +mod owner_pair_tests { + use super::*; + use crate::g17::fw::kick::KickTimestamp; + + fn kick(qid: u8, timestamp: u64) -> KickDependency { + KickDependency::new(qid, KickTimestamp::new(timestamp).unwrap()).unwrap() + } + + #[test] + fn a_full_preferred_pair_moves_to_the_owners_other_pair() { + let mut state = State::new(); + state.owners[1] = 1; + state.owners[2] = 1; + for _ in 0..cfg::RENDER_DEPTH { + assert_eq!(state.pick(1, 1, Plan::Pair(1)), Some(1)); + } + assert!(!state.can_pick(1, Plan::Pair(1))); + assert_eq!(state.pick(1, 1, Plan::Pair(1)), None); + assert!(state.can_pick(1, Plan::Join(1))); + assert_eq!(state.pick(1, 1, Plan::Join(1)), Some(2)); + // With one pair a preference is the pair. + state.owners[2] = 0; + assert!(!state.can_pick(1, Plan::Join(1))); + assert_eq!(state.pick(1, 1, Plan::Join(1)), None); + } + + #[test] + fn any_quarantined_pair_stops_its_owner() { + let mut state = State::new(); + state.owners[1] = 1; + state.owners[2] = 1; + assert!(state.owner_quarantine(1).is_none()); + state.quarantined |= 1u64 << 2; + assert_eq!(state.owner_quarantine(1), Some(false)); + assert!(state.refusal(1).is_err()); + state.spared |= 1u64 << 2; + assert_eq!(state.owner_quarantine(1), Some(true)); + state.quarantined |= 1u64 << 1; + assert_eq!(state.owner_quarantine(1), Some(false)); + } + + #[test] + fn an_uncovered_own_render_on_the_other_pair_is_a_barrier() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let mut footprint = Footprint::new(1, false); + footprint.push_vm(0, 0x1000); + let footprint = Arc::new(footprint, GFP_KERNEL).unwrap(); + { + let mut state = pool.state.lock(); + for (slot, sequence, timestamp) in [(1u8, 4u64, 7u64), (1, 5, 9), (2, 6, 3)] { + state + .footprints + .push( + InFlight { + owner: 1, + sequence, + slot: Some(slot), + ticket: 0, + footprint: footprint.clone(), + fragment: Some(kick(10 + slot, timestamp)), + }, + GFP_KERNEL, + ) + .unwrap(); + } + } + let mut barriers = [KickDependency::ZERO; hazard::PUBLICATION_BARRIERS]; + // On pair 2: pair 1's newest overlapping fragment kick, never its own. + assert_eq!(pool.publication_barriers(1, 2, &footprint, None, &mut barriers).unwrap(), 1); + assert!(barriers[0] == kick(11, 9)); + assert_eq!(pool.publication_barriers(1, 2, &footprint, Some(4), &mut barriers).unwrap(), 1); + assert!(barriers[0] == kick(11, 9)); + // The queue prefix already orders them. + assert_eq!(pool.publication_barriers(1, 2, &footprint, Some(5), &mut barriers).unwrap(), 0); + // Another owner's render is a barrier whatever the prefix. + assert_eq!(pool.publication_barriers(3, 2, &footprint, Some(9), &mut barriers).unwrap(), 1); + assert_eq!(pool.publication_barriers(3, 3, &footprint, Some(9), &mut barriers).unwrap(), 2); + } +} + +#[kunit_tests(asahi_g17_two_pairs)] +mod two_pair_tests { + use super::*; + + #[test] + fn a_two_pair_owner_spreads_its_depth_over_a_surplus_sibling() { + crate::g17::pairs::set_sibling_backlog(0); + let mut state = State::new(); + let first = state.acquire(1, 2).unwrap(); + // The sibling is claimed with the first pair, from surplus. + let sibling = (0..cfg::RENDER_SLOTS) + .find(|slot| *slot != usize::from(first) && state.owners[*slot] == 1) + .unwrap(); + // With the first pair busy, the next render takes the idle sibling, then + // the least-loaded of the two. + assert_eq!(usize::from(state.acquire(1, 2).unwrap()), sibling); + let third = state.acquire(1, 2).unwrap(); + assert!(third == first || usize::from(third) == sibling); + for _ in 3..2 * usize::from(cfg::RENDER_DEPTH) { + assert!(state.acquire(1, 2).is_some()); + } + assert_eq!(state.refs[usize::from(first)], cfg::RENDER_DEPTH); + assert_eq!(state.refs[sibling], cfg::RENDER_DEPTH); + assert_eq!(state.acquire(1, 2), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 2); + crate::g17::pairs::set_sibling_backlog(1); + } + + #[test] + fn a_shallow_two_pair_queue_stays_on_its_busy_pair() { + crate::g17::pairs::set_sibling_backlog(1); + let mut state = State::new(); + let first = state.acquire(1, 2).unwrap(); + // At depth 2 the second render stacks on the busy pair. + assert_eq!(state.acquire(1, 2), Some(first)); + // With a backlog there, the third takes the idle sibling. + let third = state.acquire(1, 2).unwrap(); + assert_ne!(third, first); + assert_eq!(state.owners[usize::from(third)], 1); + // Retired back to one render, the pair keeps the next one. + state.release(first); + state.release(third); + assert_eq!(state.acquire(1, 2), Some(first)); + } + + #[test] + fn a_sibling_needs_room_to_build_and_never_takes_a_spare() { + let mut state = State::new(); + state.build_room = false; + // A released pair of a closed owner is reused for the first pair; no + // other free slot is claimed without room to build. + state.owners[5] = 9; + state.released |= 1u64 << 5; + assert_eq!(state.acquire(1, 2), Some(5)); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + // An unused spare goes only to an owner without a pair. + state.owners[6] = 3; + state.released |= 1u64 << 6; + assert!(!state.reassign_unused_spare(6, Some(3), Some(1))); + assert!(state.reassign_unused_spare(6, Some(3), Some(2))); + } + + #[test] + fn a_one_pair_owner_claims_no_sibling() { + let mut state = State::new(); + let first = state.acquire(1, 1).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + assert_eq!(state.acquire(1, 1), Some(first)); } } @@ -1191,7 +1417,7 @@ mod deadline_tests { state.active = State::MASK; } let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + let result = pool.acquire_render(1, 1, None, start + 20_000_000, || Ok(())); assert!(matches!(result, Err(error) if error == EBUSY)); assert!(crate::g17::now_ns() - start < 500_000_000); assert!(pool.state.lock().footprints.is_empty()); @@ -1209,7 +1435,7 @@ mod deadline_tests { state.active = State::MASK; } let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + let result = pool.acquire_render(1, 1, None, start + 2_000_000_000, || { if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } }); assert!(matches!(result, Err(error) if error == ECANCELED)); @@ -1220,7 +1446,7 @@ mod deadline_tests { #[test] fn cancellation_precedes_available_capacity() { let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + assert!(matches!(pool.acquire_render(1, 1, None, u64::MAX, || Err(ECANCELED)), Err(error) if error == ECANCELED)); assert_eq!(pool.state.lock().active, 0); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 4dab923c9e1bb..d8a1919829340 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -4,6 +4,8 @@ //! installed graph; the device retains it until both firmware processors have stopped. pub(crate) mod exit; +mod activation; +mod configuration; pub(crate) mod retirement; use crate::g17::{ @@ -12,12 +14,13 @@ use crate::g17::{ fw::{ channels::ControlRecord, compute::{ComputeProgram, OperandState, COMPUTE_DESCRIPTOR_SIZE}, + m5::initdata::Board, queue::*, variant::PointerLayout, }, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; -use crate::{hw::t8140::dynamic, mmu}; +use crate::mmu; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; @@ -90,7 +93,7 @@ pub(crate) struct Binding { preempt: mmu::KernelMapping, usage: mmu::KernelMapping, operand: mmu::KernelMapping, - _queue_context: mmu::KernelMapping, + _queue_context: Option, context: Arc, } @@ -106,6 +109,8 @@ impl DescriptorRing { /// Firmware reads every descriptor through a cached kernel mapping, as it /// does for render. The pool keeps that mapping until processor stop; the /// hardware reads the same pages through an uncached canonical low alias. + /// On M5 the kernel mapping, in the upper half every execution context + /// includes, is firmware-only. pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; let object = objects.allocate( @@ -113,12 +118,12 @@ impl DescriptorRing { Family::Descriptor, size, size, - mmu::PROT_GPU_FW_SHARED_RW, + if alloc.uat.is_m5() { mmu::PROT_FW_SHARED_RW } else { mmu::PROT_GPU_FW_SHARED_RW }, CpuMap::WriteCombined, )?; let low = object.alias_in( alloc.uat.kernel_lower_vm(), - dynamic::LOWER, + alloc.uat.kernel_lower_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, )?; @@ -129,14 +134,16 @@ impl DescriptorRing { /// Firmware-shared storage of one compute QID, separate from its common kick ring. /// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). struct Graph { + m5: bool, ring: Option, preempt: KernelObject, usage: KernelObject, operand: KernelObject, support: KernelObject, + m5_timestamps: Option>, graph: PooledObject, - context_low: KernelObject, - context_high: KernelObject, + context_low: Option, + context_high: Option, /// Announcements, which stay uncached for the firmware. records: KernelObject, /// Configuration, predecessor and completion-wait records at the same @@ -148,6 +155,8 @@ struct Graph { pointers: PointerLayout, /// Layout of the operand-state buffer for the GPU's cores. operand_state: &'static OperandState, + /// The M5 board, whose part selects the M5 descriptor words. + board: Option<&'static Board>, program: ComputeProgram, } @@ -158,7 +167,9 @@ impl Graph { context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, + board: Option<&'static Board>, program: ComputeProgram, + m5: bool, ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; @@ -167,12 +178,16 @@ impl Graph { let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; - let mut support = alloc.kernel(0xc000, page, prot, cpu)?; - let support_va = support.gpu_va(); - support.write(0, support_va + page)?; - support.write(0x100, support_va + page + 4)?; - support.write(0x110, 0x50u32)?; - support.write(mmu::UAT_PGSZ + 4, 1u32)?; + // The M5 support page is firmware-only; the GPU has no access to it. + let support_prot = if m5 { mmu::PROT_FW_SHARED_RW } else { prot }; + let mut support = alloc.kernel(if m5 { mmu::UAT_PGSZ } else { 0xc000 }, page, support_prot, cpu)?; + if !m5 { + let support_va = support.gpu_va(); + support.write(0, support_va + page)?; + support.write(0x100, support_va + page + 4)?; + support.write(0x110, 0x50u32)?; + support.write(mmu::UAT_PGSZ + 4, 1u32)?; + } let mut graph = objects.allocate( alloc, Family::QueueState, @@ -183,14 +198,6 @@ impl Graph { )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { - view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + CURSORS as u64, - items_va: graph_va + ITEMS as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }); view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) @@ -199,9 +206,29 @@ impl Graph { *view = PointerBlock::new(&pointers)?; Ok(()) })?; - let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; - let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; + let record = QueueRecordArgs { + pointers_va: graph_va + CURSORS as u64, + items_va: graph_va + ITEMS as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + policy: context.policy(), + }; + // One queue record, the firmware's own: the M5 record is shorter than the G17P one the + // layout's slot holds, and the zeroed layout leaves the rest of the slot zero. + if m5 { + graph.write(0, crate::g17::fw::m5::queue::QueueRecord::new(&record)?)?; + } else { + graph.write(0, QueueRecord::new(&record))?; + } + let (context_low, context_high) = if m5 { (None, None) } else { + (Some(alloc.lower(CONTEXT_SIZE, page, prot, cpu)?), + Some(alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?)) + }; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let m5_timestamps = if m5 { + Some(Arc::new(alloc.kernel(RECORD_SLOTS * 16, page, prot, cpu)?, GFP_KERNEL)?) + } else { None }; let cached_records = objects.allocate( alloc, Family::QueueConfig, @@ -211,11 +238,13 @@ impl Graph { cpu, )?; Ok(Self { + m5, ring: Some(ring), preempt, usage, operand, support, + m5_timestamps, graph, context_low, context_high, @@ -225,6 +254,7 @@ impl Graph { reclaimed: 0, pointers, operand_state, + board, program, }) } @@ -240,20 +270,20 @@ impl Graph { return Err(EFAULT); } let vm = context.vm(); - vm.install_job_context_aliases()?; + if !self.m5 { vm.install_job_context_aliases()?; } let descriptors = self.descriptor_alias(vm)?; - let prot = mmu::PROT_GPU_FW_SHARED_RW; + let prot = if self.m5 { mmu::PROT_GPU_SHARED_RW } else { mmu::PROT_GPU_FW_SHARED_RW }; let page = mmu::UAT_PGSZ as u64; + let private = mmu::client_private_range(self.m5); let preempt = self .preempt - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + .alias_in(vm, private.clone(), page, prot)?; + let usage = self.usage.alias_in(vm, private.clone(), page, prot)?; let operand = self .operand - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let queue_context = self - .context_low - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + .alias_in(vm, private.clone(), page, prot)?; + let queue_context = self.context_low.as_ref() + .map(|context| context.alias_in(vm, private.clone(), page, prot)).transpose()?; Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), @@ -268,6 +298,15 @@ impl Graph { /// Caller has excluded publication and proved the retained graph idle. fn set_owner(&mut self, context: &Context) -> Result { + if self.m5 { + let pointer = self.graph.pointer(0, + core::mem::size_of::())? + .cast::(); + // SAFETY: The checked packed record stays mapped. The device mutex + // and idle witness exclude access by another owner during this update. + return unsafe { (*pointer).set_compute_owner(context.work_head_va(), + context.scheduler_va(), context.owner_pid(), context.policy()) }; + } let pointer = self .graph .pointer(0, core::mem::size_of::())? @@ -313,6 +352,15 @@ impl Graph { /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed /// descriptors at this point. + /// Clears the client-visible state the last owner's work left (preemption + /// buffer, usage page, operand state) before another VM is given views of + /// it. The caller proved the queue idle. + fn clear_client_state(&mut self) { + self.preempt.bytes_mut().fill(0); + self.usage.bytes_mut().fill(0); + self.operand.bytes_mut().fill(0); + fence(Ordering::SeqCst); + } fn vacate_ring(&mut self) -> Option { self.ring.take() } @@ -376,7 +424,8 @@ impl Graph { } use crate::g17::{ - completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, + completion::Deferred, context::WorkStateLease, + freelist::{ComputeCapacity, ComputePool, PoolDescriptor}, job::Packet, kick, qos, recovery, status::VmStatus, }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; @@ -472,6 +521,9 @@ struct Previous> { binding: Option, context: C, publication: Option<(u64, u64)>, + /// The queue moved to another VM. The previous owner's views were unmapped at the move, + /// so a rollback leaves the queue detached instead of restoring them. + moved: bool, } impl Previous { /// The exact drain witness covers the replacement publication even when @@ -505,14 +557,15 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { - let pool = ComputePool::new(alloc, id.qid(), blocks)?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, required: ComputeCapacity, + client: Option<&Arc>) -> Result { + let pool = ComputePool::new(alloc, id.qid(), required, client)?; Ok(Self { pool, id }) } /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { - Ok((self.pool.id(), self.pool.page_list_va()?)) + pub(crate) fn pool_descriptor(&self) -> Result { + self.pool.descriptor() } } @@ -558,12 +611,15 @@ pub(crate) struct Queue { pool_phase: PoolPhase, release_sent_ns: Option, growing: bool, - growth_blocks: usize, + /// What the growth in progress must provide. + growth: Option, /// A killed queue was recycled: its pool/ring must be vacated and re-armed /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). needs_fresh_backing: bool, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, + /// Monotonic time the watchdog quarantined this queue, zero otherwise. + timed_out_at: u64, /// Free-list generation of the last published configuration. free_list_generation: u64, /// `None` after an exited owner was detached; a new owner binds afresh. @@ -579,11 +635,18 @@ pub(crate) struct Queue { replay_retirement: bool, replay_error: Option, replay_prepared: bool, + /// Since when the replay front waits for its render consumers (see compute_replay). + consumer_wait_since: Option, exit: Option, /// Since when the kill of this failed queue's context waits for other contexts' compute, /// and whether that wait was reported. kill_wait: Option<(u64, bool)>, installed: bool, + /// The owner re-arming this vacant queue: only it builds the fresh backing, so concurrent + /// waiters do not each build a pool for the same queue. + rearm_for: Option, + configuration: Option, + m5_install: Option, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, @@ -626,11 +689,13 @@ impl Queue { context: Arc, pointers: PointerLayout, operand_state: &'static OperandState, + board: Option<&'static Board>, program: ComputeProgram, + m5: bool, ) -> Result { let Backing { mut pool, mut kick } = backing; let (active, replays) = (Fifo::new()?, Fifo::new()?); - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, program)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, board, program, m5)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -643,9 +708,10 @@ impl Queue { pool_phase: PoolPhase::Populated, release_sent_ns: None, growing: false, - growth_blocks: 0, + growth: None, needs_fresh_backing: false, released_at: 0, + timed_out_at: 0, free_list_generation: 0, binding: Some(binding), previous: None, @@ -658,9 +724,13 @@ impl Queue { replay_retirement: false, replay_error: None, replay_prepared: false, + consumer_wait_since: None, exit: None, kill_wait: None, installed: false, + rearm_for: None, + configuration: None, + m5_install: None, outer_started: 0, published_config: None, config_cache: None, @@ -750,10 +820,22 @@ impl Queue { !self .context() .is_some_and(|bound| Arc::ptr_eq(bound, context)) - || (self.active.len == 0 && self.replays.len == 0) + || (self.active.len == 0 && self.replays.len == 0 && self.configuration.is_none()) } pub(crate) fn in_flight(&self) -> bool { - self.active.len != 0 && !self.quarantined + (self.active.len != 0 || self.configuration.is_some()) && !self.quarantined + } + /// Per-queue consumer wait (see compute_replay). + pub(crate) fn consumer_wait(&mut self, now: u64) -> u64 { + now.saturating_sub(*self.consumer_wait_since.get_or_insert(now)) + } + pub(crate) fn clear_consumer_wait(&mut self) { + self.consumer_wait_since = None; + } + /// Cold M5 installation publishes one configuration before the first kick. + /// A prepared claim prevents reassignment while these credits are owned. + pub(crate) fn outer_slots(&self) -> u32 { + if self.graph.m5 && self.m5_install.is_none() { 2 } else { 1 } } /// The oldest command is a published long job (ray traversal or AS build) whose /// completion record the driver has not seen yet. @@ -773,7 +855,14 @@ impl Queue { self.active.front().map(|active| crate::g17::timeout::Head { vm: active.packet.context.vm().id(), published: active.published, - finished: active.observed.is_some(), + // An M5 completion record can precede the command's writes: only the stamp + // proves it finished. + finished: if self.graph.m5 { + active.ticket.as_ref().is_some_and(|ticket| self.graph.stamp() + .is_ok_and(|stamp| retirement::stamp_covers(stamp, ticket.kick))) + } else { + active.observed.is_some() + }, failed: self.quarantined, }) } @@ -836,7 +925,7 @@ impl Queue { /// A completion hint can require a second visibility/recordless pass even /// when firmware has no more events to send. Do not poll idle or failed queues. pub(crate) fn retirement_polling(&self) -> bool { - !self.quarantined && self.retire_pending + self.configuration.is_some() || (!self.quarantined && self.retire_pending) } /// A release has no completion event: keep observing the control consumer @@ -991,6 +1080,7 @@ impl Queue { mut complete_qos: impl FnMut(qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { + if recovery_state == 0 { self.retire_configuration(epoch)?; } if self.quarantined || !self.selected(masks) { return Ok(()); } @@ -1038,7 +1128,7 @@ impl Queue { recordless, self.last_end, next_end, - record_aged, + record_aged && !self.graph.m5, )?; if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { retirement::note_stampless(self.qid(), ticket.kick); @@ -1072,10 +1162,17 @@ impl Queue { ); } self.retire_pending = true; - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok([timestamps[0], timestamps[1], 0, 0]), - ))?; + let completion = front.packet.completion.clone(); + let producers = front.packet.check_firmware_producers(); + defer(match producers { + Ok(()) if self.graph.m5 => Deferred::RetiredCompute( + completion, + [timestamps[0], timestamps[1], 0, 0], + front.observed.map(|record| record.timestamps), + ), + Ok(()) => Deferred::Retired(completion, Ok([timestamps[0], timestamps[1], 0, 0])), + Err(error) => Deferred::Retired(completion, Err(error)), + })?; self.retire_pending = false; self.submitted = submitted; let active = self.active.pop().ok_or(EIO)?; @@ -1149,21 +1246,43 @@ impl Queue { )? { return Ok(()); } + let unexecuted = if self.graph.m5 { + // M5 command words stay zero even for executed commands. The + // front needs its exact zero-duration ring record; a later + // command, chained to its predecessor by the kick parent, must + // not show a record with a real duration. A render consumer of + // this work (firmware dependency) is checked by the replay pass: + // it republishes only while every render of the owner is parked + // or idle and none executed since recovery. + self.active.iter().enumerate().all(|(index, active)| { + !active.packet.completion.signalled() + && match active.observed { + // Only reached for spared work quarantined after a + // recovery: every retained command crossed it. + Some(record) => { + record.timestamps[0] != 0 && retirement::unexecuted(record, true) + } + None => index != 0, + } + }) + } else { + !self + .active + .iter() + .any(|active| active.packet.completion.has_firmware_consumer()) + && self.active.iter().all(|active| { + active + .packet + .completion + .replay_witness_words() + .is_some_and(|words| words[0] == words[1]) + }) + }; let candidate = self.spared_deferred && self.spared_quarantine && !self.released && self.owner == Some(newest.owner) - && !self - .active - .iter() - .any(|active| active.packet.completion.has_firmware_consumer()) - && self.active.iter().all(|active| { - active - .packet - .completion - .replay_witness_words() - .is_some_and(|words| words[0] == words[1]) - }); + && unexecuted; if candidate && !replay { return Ok(()); } @@ -1198,6 +1317,7 @@ impl Queue { if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; + self.timed_out_at = 0; self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; @@ -1272,6 +1392,7 @@ impl Queue { /// Caller already stopped both processors. No firmware owner can access retained mappings. pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(configuration) = self.configuration.take() { configuration.release(); } while let Some(packet) = self.replays.front() { defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; drop(self.replays.pop()); @@ -1549,6 +1670,7 @@ impl Queue { return false; } self.installed + && self.configuration.is_none() // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) // A recycled killed queue is offered only once its backing is vacant (re-arm path). @@ -1594,19 +1716,49 @@ impl Queue { binding: None, context: old.ok_or(EIO)?, publication: None, + moved: false, }); } else { - let kick_alias = self.kick.map_client(context.vm())?; - let pool_alias = self.pool.map_client(context.vm())?; - let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; + // Another VM: the pool must be fresh backing (EPERM otherwise). The queue's state + // pages belong to one VM at a time. The previous owner's views (kick ring, + // preemption, usage and operand state, descriptors, queue context) are unmapped + // first, so its GPU work can no longer read or write what the new owner's work + // leaves there; then the pages are cleared and mapped into the new VM. The queue + // is idle (`reusable`) and the previous owner's pool release was consumed (fresh + // backing), so the firmware no longer uses those views. The previous context, + // whose scheduler state the queue may still run until the new owner's first + // command, stays in `Previous`. + if !self.pool.serves(context.vm()) { + return Err(EPERM); + } + drop(self.binding.take()); + let moved = (|| -> Result { + self.graph.clear_client_state(); + self.kick.clear_ring()?; + let kick_alias = self.kick.map_client(context.vm())?; + let pool_alias = self.pool.map_client(context.vm())?; + self.graph.bind(context.clone(), kick_alias, pool_alias) + })(); + let binding = match moved { + Ok(binding) => binding, + Err(error) => { + // Left detached: the record still names the previous owner's pages. + if let Some(old) = old.as_ref() { + self.retained_owner = Some(old.retain_firmware_pages()); + } + self.needs_fresh_backing = true; + return Err(error); + } + }; self.graph.set_owner(&context)?; - let old_binding = self.binding.replace(binding); + self.binding = Some(binding); // A detached queue (exited owner already dropped) has nothing to supersede; // the new owner's record fields replace the retained pages' references. self.previous = old.map(|old| Previous { - binding: old_binding, + binding: None, context: old, publication: None, + moved: true, }); self.retained_owner = None; } @@ -1619,6 +1771,7 @@ impl Queue { if same_vm { 2 } else { 1 }, ); self.owner = Some(owner); + self.rearm_for = None; self.unpublished_binding = true; self.released = false; self.retirement_ready = false; @@ -1635,7 +1788,7 @@ impl Queue { return Ok(true); } self.released = true; - if self.active.len == 0 && !self.quarantined { + if self.active.len == 0 && !self.quarantined && self.configuration.is_none() { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; @@ -1643,7 +1796,13 @@ impl Queue { return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; - if let Some(binding) = previous.binding.take() { + if previous.moved { + // The record names the previous owner again, whose views went at the + // move: the queue is left detached, as for an exited owner. + self.binding = None; + self.retained_owner = Some(previous.context.retain_firmware_pages()); + self.needs_fresh_backing = true; + } else if let Some(binding) = previous.binding.take() { self.binding = Some(binding); } else { self.binding.as_mut().ok_or(EIO)?.context = previous.context; @@ -1681,6 +1840,8 @@ impl Queue { } let binding = self.binding.take()?; self.retained_owner = Some(binding.context.retain_firmware_pages()); + // The backing served the exited VM only: give it back for fresh backing. + self.needs_fresh_backing = true; Some(binding) } /// An idle retained queue whose USC backing may be given back. @@ -1705,12 +1866,33 @@ impl Queue { pub(crate) fn needs_fresh_backing(&self) -> bool { self.needs_fresh_backing } + /// Whether this queue's pool backing may be bound for `vm`. + pub(crate) fn pool_serves(&self, vm: &mmu::Vm) -> bool { + self.pool.serves(vm) + } + /// Gives a released queue's backing back for fresh backing before reuse. + pub(crate) fn require_fresh_backing(&mut self) { + self.needs_fresh_backing = true; + } + /// A pool release is in flight, or a queue waits for one, that ends with a + /// vacant queue ready for re-arming. + pub(crate) fn vacancy_pending(&self) -> bool { + !self.growing && (self.pool_release_pending() + || (self.recycled_pending() && self.pool_phase == PoolPhase::Populated)) + } /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) - /// and every killed command was settled: return the QID to service. The item - /// ring is reset to empty at the firmware's consumer cursor; the USC pool and - /// descriptor ring are vacated and re-armed fresh before the next owner binds. + /// and every killed command was settled. The QID stays killed in the firmware + /// until a queue configuration with its QoS update set binds it to a scheduler + /// page other than the one the firmware has for it. A new owner's first + /// publication carries such a configuration, but its command is retired + /// unexecuted on the still-killed queue, with a zero-duration record. + /// + /// On M5 the queue is installed again instead: the next owner's first + /// publication sends the configuration alone and waits until the firmware + /// consumed it (`install_m5`), so the kill is cleared before any command. The + /// USC pool and descriptor ring are vacated and re-armed fresh as before. pub(crate) fn recycle_killed(&mut self) -> Result { - if !self.installed + if self.configuration.is_some() || !self.installed || !self.exit.as_ref().is_some_and(|exit| exit.released()) || self.active.len != 0 || self.replays.len != 0 @@ -1719,6 +1901,15 @@ impl Queue { return Ok(false); } self.recycle_settled()?; + if self.graph.m5 { + // Start over as a fresh install: no published configuration and kick + // count zero, so the next publication installs the queue first. Kick + // timestamps keep counting, so its stamps stay ordered. + self.m5_install = None; + self.published_config = None; + self.outer_started = 0; + self.ordinal = 0; + } Ok(true) } /// A quarantined queue whose owner closed and whose every retained command was @@ -1750,6 +1941,7 @@ impl Queue { self.exit = None; self.quarantined = false; self.quarantine_error = None; + self.timed_out_at = 0; self.failure_status_pending = false; self.retired_by_teardown = false; self.spared_quarantine = false; @@ -1776,6 +1968,18 @@ impl Queue { pub(crate) fn pool_vacant(&self) -> bool { self.pool_phase == PoolPhase::Vacant } + /// No other owner is re-arming this queue. + pub(crate) fn rearm_open(&self, owner: u64) -> bool { + self.rearm_for.is_none_or(|reserved| reserved == owner) + } + pub(crate) fn reserve_rearm(&mut self, owner: u64) { + self.rearm_for = Some(owner); + } + pub(crate) fn release_rearm(&mut self, owner: u64) { + if self.rearm_for == Some(owner) { + self.rearm_for = None; + } + } /// Publishes the release of this idle queue's free list. The caller holds the /// device mutex and rings the control doorbell afterwards. pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { @@ -1834,7 +2038,7 @@ impl Queue { &mut self, pages: crate::g17::freelist::Pages, ring: DescriptorRing, - ) -> Result<(u16, u64)> { + ) -> Result { if !self.can_rearm() { return Err(EAGAIN); } @@ -1843,10 +2047,10 @@ impl Queue { self.pool_phase = PoolPhase::Populated; self.release_sent_ns = None; self.needs_fresh_backing = false; - Ok((self.pool.id(), self.pool.page_list_va()?)) + self.pool.descriptor() } - pub(crate) fn pool_fits(&self, required: usize) -> bool { - !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits(required) + pub(crate) fn pool_fits(&self, required: ComputeCapacity) -> bool { + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits_capacity(required) } /// Only a capacity increase waits for this queue's previously published work. @@ -1854,7 +2058,7 @@ impl Queue { pub(crate) fn begin_growth( &mut self, init: &crate::g17::initdata::InitData, - blocks: usize, + required: ComputeCapacity, ) -> Result { if self.growing { return Err(EAGAIN); @@ -1881,7 +2085,7 @@ impl Queue { self.release_sent_ns = None; } self.growing = true; - self.growth_blocks = blocks; + self.growth = Some(required); Ok(self.installed) } @@ -1890,9 +2094,10 @@ impl Queue { init: &crate::g17::initdata::InitData, pages: &mut Option, alias: &mut Option, - blocks: usize, ) -> Result { - if !self.growing || blocks < self.growth_blocks { + // The fresh backing must provide what the growth was begun for. + let target = self.growth.filter(|_| self.growing).ok_or(EAGAIN)?; + if !pages.as_ref().ok_or(EINVAL)?.fits_capacity(target) { return Err(EAGAIN); } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { @@ -1909,13 +2114,13 @@ impl Queue { let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. // The old release is consumed; no queue can use this row until commit. - init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + init.install_compute_pool(fresh.descriptor(self.pool.id())?)?; self.pool.replace_pages(pages)?; let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; self.release_sent_ns = None; self.growing = false; - self.growth_blocks = 0; + self.growth = None; Ok(Vacated { _alias: old_alias, _pages: pages.take(), @@ -1939,7 +2144,7 @@ impl Queue { return Ok(None); } self.growing = false; - self.growth_blocks = 0; + self.growth = None; self.release_owner().map(Some) } @@ -1977,6 +2182,7 @@ impl Queue { let result = self.quarantine(ETIMEDOUT, defer); if fresh { self.spared_quarantine = return_spared; + self.timed_out_at = now_ns().max(1); } result } @@ -2167,6 +2373,7 @@ pub(crate) trait Host { fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; + fn submit_m5(&mut self, submission: &crate::g17::fw::m5::queue::HardwareSubmission) -> Result; } impl Queue { @@ -2201,6 +2408,11 @@ impl Queue { }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { + matrix_ops: requested_matrix_ops, + cdm_auxiliary_allocation, + plan, + m5_intersection, + m5_scratch, intersection, scratch, usc_base, @@ -2213,20 +2425,28 @@ impl Queue { else { return Err(EINVAL); }; + let m5_scratch = *m5_scratch; + if !self.graph.m5 && (*requested_matrix_ops || *cdm_auxiliary_allocation != 0) { return Err(EINVAL); } + if self.graph.m5 { + // Validation refuses a sampler heap on M5, and M5 completion waits ride the + // kick's dependencies (publish_compute_inputs). + if *sampler_count != 0 || !native.is_empty() { return Err(EINVAL); } + if plan.is_some_and(|plan| !self.pool.fits(Some(plan.geometry))) { return Err(EINVAL); } + self.install_m5(&packet, host)?; + } let work = packet.completion.work_state()?; let work_word = work.word(); if work_word == u32::MAX { return Err(EINVAL); } let work_node = work.node_va()?; - let timestamp_va = packet.completion.compute_timestamp_va()?; let kick_timestamp = self.kick.timestamp(); let kick_parent = self.kick.parent(); let qid = self.qid(); let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; - let first = !self.installed && self.ordinal == 0; + let first = self.ordinal == 0 && (self.graph.m5 || !self.installed); // The first prototype only waits on an installed consumer channel. // A retry without a published source falls back to completion on CPU. if first && !native.is_empty() { @@ -2239,7 +2459,22 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); - item_capacity(self.graph.cursors()?, entry_signal)?; + if self.graph.m5 { + // The configuration and its notification are the only items an M5 kick appends. + let cursors = self.graph.cursors()?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < 2 { + return Err(ENOSPC); + } + } else { + item_capacity(self.graph.cursors()?, entry_signal)?; + } if !native.is_empty() { let cursors = self.graph.cursors()?; let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; @@ -2247,10 +2482,15 @@ impl Queue { return Err(EAGAIN); } } - let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); + let record_slot = (self.graph.m5 || !first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; } + let timestamp_va = if self.graph.m5 { + packet.completion.bind_m5_compute_timestamps( + self.graph.m5_timestamps.as_ref().ok_or(EINVAL)?, + record_slot.ok_or(EINVAL)? * 16)? + } else { packet.completion.compute_timestamp_va()? }; let args = ComputeArgs { program: self.graph.program, intersection: *intersection, @@ -2290,20 +2530,63 @@ impl Queue { }; self.free_list_generation = u64::from(packet.context.scheduler_generation()); let policy = packet.context.policy(); - let ring = 1u8 << policy.priority(); - let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let activation = activation::Publication::new( + self.outer_started, policy.priority(), self.graph.m5); + let config = self.queue_config(&identity, first, activation.refresh, + args.context_id, policy, &packet)?; let cached = self .config_cache .and_then(|(config, at)| (at == epoch).then_some(config)); - let send_config = first || super::render::publication::required(cached, config); + let send_config = self.graph.m5 || first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; + let matrix_ops = self.graph.m5 && *requested_matrix_ops; + let m5_board = if self.graph.m5 { Some(self.graph.board.ok_or(EINVAL)?) } else { None }; + let m5_args = m5_board.map(|board| { + let tuple = crate::g17::fw::m5::compute::kick_tuple(board); + crate::g17::fw::m5::compute::DescriptorArgs { + registers: crate::g17::fw::m5::compute::RegisterArgs { + matrix_ops, + scratch: m5_scratch, + intersection: *m5_intersection, + auxiliary_allocation: *cdm_auxiliary_allocation, + preempt_va: addresses.preempt, cdm_va: *cdm_va, operand_va: addresses.operand, + usage_va: addresses.usage, free_list_slot: u32::from(self.pool.id()), + context_id: args.context_id, work_state: work_word, + object_ids: 0x0000_0001_0000_0001, + resource_policy: plan.map_or(0xf5_0040_0020, |p| u64::from(p.pressure) << 32 | 0x0040_0020), + preempt_state_offset: tuple.preempt_state_offset, + // The board's core layout sizes the operand state, as on G17P. + operand_region_bytes: self.graph.operand_state.bank_bytes(), + operand_aux_offset: self.graph.operand_state.tail_offset(), + operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or(0x54b, |p| p.resource_flags), + dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base), + auxiliary_control: tuple.auxiliary_control, + }, + descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, + cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], + stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid), + kick_timestamp: kick_timestamp.get(), + completion_ordinal: crate::g17::fw::m5::compute::completion_ordinal(board, self.ordinal), + status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], + shared_control_va: self.pool.control_va(), event_metadata: [0;16], + usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, + }}); + // Explicit dependencies keep the pipelined class; the implicit parent, last, takes its + // own ([`crate::g17::fw::m5::compute::parent_dependency`]). + let mut m5_dependency_types = [crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY; + crate::g17::fw::kick::QID_MAX as usize + 1]; + if let (Some(board), Some(parent)) = (m5_board, m5_dependency_types.get_mut(dependencies.len())) { + *parent = crate::g17::fw::m5::compute::parent_dependency(board, false); + } let kick_args = KickArgs { qid, timestamp: kick_timestamp, parent: kick_parent, + ancestor: None, barriers: dependencies, + dependency_types: self.graph.m5.then_some(&m5_dependency_types[..dependencies.len() + 1]), descriptor_va: descriptor_high, queue_va: self.graph.graph.gpu_va(), qos: KickQos { @@ -2311,9 +2594,17 @@ impl Queue { class: COMPUTE_QOS_CLASS, }, mcache: None, - event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)?, - scs_enabled: scratch.enabled(), + event_mask: match m5_board { + Some(board) => crate::g17::fw::m5::compute::kick_event_mask(board, identity.context)?, + None => COMPUTE_KICK_EVENT_MASK, + }, + register_arrays: if self.graph.m5 { + crate::g17::fw::m5::compute::bindings(descriptor_low, m5_scratch, matrix_ops)? + } else { + ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)? + }, + scs_enabled: if self.graph.m5 { m5_scratch.enabled() } else { scratch.enabled() }, + compute_matrix: matrix_ops, priority: policy.priority(), }; let published = host.next_compute_publication()?; @@ -2336,17 +2627,16 @@ impl Queue { let result = (|| -> Result { work.add_submitted_kicks(1)?; accounted = true; - self.graph - .ring - .as_mut() - .ok_or(EAGAIN)? - .object - .initialize::( - kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write_zeroed(&args), - )?; - if first { - self.graph.context_high.write( + let object = &mut self.graph.ring.as_mut().ok_or(EAGAIN)?.object; + let offset = kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + if let Some(m5_args) = &m5_args { + object.initialize::(offset, + |descriptor| descriptor.write_zeroed(m5_args))?; + } else { + object.initialize::(offset, |descriptor| descriptor.write_zeroed(&args))?; + } + if !self.graph.m5 && first { + self.graph.context_high.as_mut().ok_or(EINVAL)?.write( 0x200, crate::g17::fw::compute::QueueContextItem::new( descriptor_high, @@ -2354,12 +2644,13 @@ impl Queue { qid, ), )?; - } else { + } else if !self.graph.m5 { // The retained context item remains at index one; only its descriptor changes. - self.graph.context_high.write(0x210, descriptor_high)?; + self.graph.context_high.as_mut().ok_or(EINVAL)?.write(0x210, descriptor_high)?; let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } + if self.graph.m5 { let cursors = self.graph.cursors()?; self.graph.recycle(cursors)?; } let config_va = if send_config { Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) } else { @@ -2383,7 +2674,26 @@ impl Queue { let kick_ring_va = self.kick.low_va(); let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; - if first { + if self.graph.m5 { + let target = self.pool.reserve_submission()?; + let qos = host.qos_publish(qos::Owner { qid, qos: identity.qos, data_master: 2 }, + identity.scheduler, policy).or_else(|error| { + self.pool.cancel_unpublished(target)?; Err(error) + })?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; + producer = self.graph.append(cursors.producer, config_va.ok_or(EINVAL)?, + || packet.mark_published()) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + visible = true; + self.unpublished_binding = false; + // This Active and its Packet pins now own all visible records. + if let Some(configuration) = self.configuration.take() { configuration.release(); } + } else if first { host.register_queue(&crate::g17::fw::kick::KsmQueue::new( qid, kick_ring_va, @@ -2468,10 +2778,13 @@ impl Queue { } prepared_kick.commit(); self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; - let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; + let announce_va = if self.graph.m5 { + self.graph.record(record_slot, ANNOUNCE, 0x40, + crate::g17::fw::m5::queue::notification(qid, kick_timestamp)?)? + } else { self.graph.record(record_slot, ANNOUNCE, 0x40, announce)? }; producer = self.graph.append(producer, announce_va, || {})?; fence(Ordering::SeqCst); - if entry_signal { + if entry_signal && !self.graph.m5 { let predecessor_va = self.graph .record(record_slot, PREDECESSOR, 0x400, predecessor)?; @@ -2488,10 +2801,11 @@ impl Queue { self.graph.graph.gpu_va(), qid, producer.try_into()?, - self.outer_started & ring == 0, + activation.first, )?, )?; - self.outer_started |= ring; + let notify_activation = activation.first; + self.outer_started = activation.next_started; self.installed = true; self.published_config = Some(identity); if send_config { @@ -2527,6 +2841,13 @@ impl Queue { // ring itself and delayed dispatch. fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + if self.graph.m5 { + // The firmware work notification must precede a deferred host + // doorbell: an idle-policy hold alone does not wake gated cores. + host.submit_m5(&crate::g17::fw::m5::queue::HardwareSubmission::compute( + qid, self.kick.low_va(), policy.priority(), + first || notify_activation)?)?; + } crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); @@ -2548,6 +2869,61 @@ impl Queue { Ok(()) } + /// The firmware consumes an empty channel installation before host-port + /// configuration. EAGAIN leaves the packet unpublished and is retried by + /// the common runtime with the device mutex released. + fn retire_configuration(&mut self, epoch: u64) -> Result { + let Some(configuration) = self.configuration.as_ref() else { return Ok(()); }; + if !configuration.consumed(self.graph.cursors()?, epoch) { return Ok(()); } + self.configuration.take().ok_or(EIO)?.release(); + if self.ordinal == 0 && self.active.len == 0 && self.replays.len == 0 { + self.retirement_ready = true; + if self.released { self.owner = None; } + } + Ok(()) + } + + fn install_m5(&mut self, packet: &Arc, host: &mut impl Host) -> Result { + let (epoch, state) = host.epoch()?; + if state == 0 { self.retire_configuration(epoch)?; } + if let Some(end) = self.m5_install { + return if self.graph.cursors()?.consumer == end || self.ordinal != 0 { Ok(()) } + else { Err(EAGAIN) }; + } + let context = &packet.context; + let policy = context.policy(); + let config = QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), qid: self.qid(), install: true, + completion_seed: None, context_id: context.id().try_into()?, + free_list: FreeListBinding { va: 0, generation: u64::MAX, slot: u32::MAX }, + scheduler_va: context.scheduler_va(), policy, qos_slot: context.qos_id(), + qos_update: true, owner_pid: context.owner_pid(), context_update: false, + })?; + if self.configuration.is_some() { return Err(EBUSY); } + // A queue reinstalled after a kill restarts at the firmware's consumer. + let start = self.graph.cursors()?.producer; + self.configuration = Some(configuration::Configuration::new( + context, (start + 1) % ITEM_RING_ENTRIES, epoch)?); + let result = (|| { + let va = self.graph.record(None, CONFIG, 0xc0, config)?; + let configuration = self.configuration.as_mut().ok_or(EIO)?; + let producer = self.graph.append(start, va, || configuration.mark_visible())?; + self.m5_install = Some(producer); + self.installed = true; + host.publish_qid(self.kick.id())?; + host.publish_outer(policy.priority(), &crate::g17::fw::channels::WorkSlot::new(DataMaster::Compute, + self.graph.graph.gpu_va(), self.qid(), producer.try_into()?, true)?)?; + self.outer_started = 1 << policy.priority(); + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + Err(EAGAIN) + })(); + if result.is_err() && self.configuration.as_ref().is_some_and(|owner| !owner.visible()) { + self.configuration.take().ok_or(EIO)?.release(); + } + result + } + /// Everything that must hold before a publication changes any state: a healthy device /// epoch, room in the active ring (or this packet at the replay front), the bound /// owner's current context and a clean completion status. Returns the epoch and the @@ -2600,6 +2976,7 @@ impl Queue { &self, identity: &ConfigIdentity, first: bool, + priority_refresh: bool, context_id: u16, policy: crate::g17::fw::queue::Policy, packet: &Arc, @@ -2609,8 +2986,12 @@ impl Queue { kick_ring_va: self.kick.low_va(), kick_ring_fw_va: self.kick.firmware_va(), qid: self.qid(), - install: first, - completion_seed: None, + // An existing M5 queue needs the firmware install/update path to + // replace its private scheduling state when its class changes. + // The ordinary binding fields below are still processed afterward; + // no completion seed is supplied for a retained queue. + install: (first && !self.graph.m5) || priority_refresh, + completion_seed: (first && self.graph.m5).then_some(0), context_id, free_list: FreeListBinding { va: self.pool.control_va(), @@ -2715,6 +3096,7 @@ mod fault_ownership_tests { binding: Some(Lease::new(&used)), context: Lease::new(&used), publication, + moved: false, }); assert_eq!(used.load(Ordering::Relaxed), 2); Previous::retire_witnessed(&mut previous); @@ -2747,6 +3129,7 @@ mod fault_ownership_tests { binding: None, context: Lease::new(&used), publication: Some((9, 3)), + moved: false, }); Previous::retire_witnessed(&mut previous); assert_eq!(used.load(Ordering::Relaxed), 0); @@ -2861,7 +3244,12 @@ impl Queue { self.graph.set_owner(&previous.context)?; } if let Some(mut previous) = self.previous.take() { - if let Some(binding) = previous.binding.take() { + if previous.moved { + // As in `release_owner`: left detached, the new owner's views returned. + retired = self.binding.take(); + self.retained_owner = Some(previous.context.retain_firmware_pages()); + self.needs_fresh_backing = true; + } else if let Some(binding) = previous.binding.take() { retired = self.binding.replace(binding); } else { self.binding.as_mut().ok_or(EIO)?.context = previous.context; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs b/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs new file mode 100644 index 0000000000000..b306bbe820cc0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! A retained M5 channel belongs to its current priority ring, not every ring it +//! has previously used. A priority change must submit it to the new ring again, +//! including when a later owner returns to an earlier priority. + +pub(super) struct Publication { + pub(super) first: bool, + /// Refresh firmware's per-QID scheduling state on a retained class change. + pub(super) refresh: bool, + pub(super) next_started: u8, +} + +impl Publication { + pub(super) fn new(started: u8, priority: u8, m5: bool) -> Self { + let ring = 1u8 << priority; + Self { + first: started & ring == 0, + refresh: m5 && started != 0 && started & ring == 0, + next_started: if m5 { ring } else { started | ring }, + } + } +} + +#[cfg(test)] +mod tests { + use super::Publication; + + #[test] + fn first_publication_activates() { + for priority in 0..4 { + let publication = Publication::new(0, priority, true); + assert!(publication.first); + assert!(!publication.refresh); + assert_eq!(publication.next_started, 1 << priority); + } + } + + #[test] + fn same_priority_keeps_fast_path() { + for priority in 0..4 { + let publication = Publication::new(1 << priority, priority, true); + assert!(!publication.first); + assert!(!publication.refresh); + assert_eq!(publication.next_started, 1 << priority); + } + } + + #[test] + fn returning_to_old_priority_reactivates() { + for old in 0..4 { + for new in 0..4 { + if old == new { continue; } + let changed = Publication::new(1 << old, new, true); + assert!(changed.first); + assert!(changed.refresh); + assert_eq!(changed.next_started, 1 << new); + let returned = Publication::new(changed.next_started, old, true); + assert!(returned.first); + assert!(returned.refresh); + assert_eq!(returned.next_started, 1 << old); + } + } + } + + #[test] + fn non_m5_preserves_historical_membership() { + let changed = Publication::new(1 << 2, 3, false); + assert!(changed.first); + assert!(!changed.refresh); + assert_eq!(changed.next_started, (1 << 2) | (1 << 3)); + let returned = Publication::new(changed.next_started, 2, false); + assert!(!returned.first); + assert!(!returned.refresh); + assert_eq!(returned.next_started, changed.next_started); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs b/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs new file mode 100644 index 0000000000000..7531ee2b0b33d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initial M5 queue installation names physical execution/QoS identities +//! before an Active command exists. Retain the exact tokens independently. + +use super::{Context, Cursors, ITEM_RING_ENTRIES}; +use crate::mmu::ContextPin; +use kernel::{prelude::*, sync::Arc}; + +pub(super) struct Configuration { + context: Arc, + pin: ContextPin, + qos: u8, + target: u32, + epoch: u64, + visible: bool, +} +impl Configuration { + pub(super) fn new(context: &Arc, target: u32, epoch: u64) -> Result { + let pin = context.pin()?; + let qos = match context.pin_qos() { + Ok(qos) => qos, + Err(error) => { context.unpin(pin); return Err(error); } + }; + Ok(Self { context: context.clone(), pin, qos, target, epoch, visible: false }) + } + pub(super) fn mark_visible(&mut self) { self.visible = true; } + pub(super) fn visible(&self) -> bool { self.visible } + pub(super) fn consumed(&self, cursors: Cursors, epoch: u64) -> bool { + self.visible && self.epoch == epoch && self.target < ITEM_RING_ENTRIES + && cursors.count == ITEM_RING_ENTRIES + && cursors.producer == self.target && cursors.consumer == self.target + } + /// Only before visibility, after exact empty-channel consumption, after + /// transfer to the same pinned Active, or after verified processor stop. + pub(super) fn release(self) { + self.context.unpin_qos(self.qos); + self.context.unpin(self.pin); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 719a74787ca70..ace49e373311f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -65,6 +65,9 @@ enum Phase { Published, Acknowledged, Unproven, + /// M5: the full-queue retirement witness covered the killed commands. Cached across + /// collector failures, since `submitted` still counts entries a retry already drained. + Witnessed, Released, } @@ -99,7 +102,7 @@ impl Queue { } pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - && self.released + && (self.released || self.timed_out()) && self.active.len != 0 && self.exit.is_none() && (!self.active_traversal() || self.fault_pending()) @@ -111,6 +114,35 @@ impl Queue { CANCELLED_DELAY_NS } } + /// M5: the watchdog failed this queue's command, and nothing has killed it yet. Its + /// commands may never retire on their own (a dispatch that does not terminate), so a kill + /// of its context treats it as failed work, like a command the firmware reported failed. + pub(crate) fn timed_out(&self) -> bool { + self.graph.m5 + && self.quarantined + && self.quarantine_error == Some(ETIMEDOUT) + && !self.spared_quarantine + && self.exit.is_none() + } + /// M5: a timed-out queue whose owner keeps it open, due for a kill of its context, with + /// how long ago the watchdog failed it. The owner's VM has already failed, so it loses + /// nothing it could still use; without the kill a command that never terminates keeps + /// running, and keeps this queue, until the owner closes. + pub(crate) fn open_timeout_kill(&self, now: u64) -> Option<(Arc, u64)> { + if self.released || self.timed_out_at == 0 || !self.timed_out() { + return None; + } + let context = self.context()?.clone(); + let age = now.saturating_sub(self.timed_out_at); + self.exit_candidate(&context, age).then_some((context, age)) + } + /// An open timed-out queue's kill is due within the exit polling window. + pub(crate) fn open_timeout_polling(&self, now: u64) -> bool { + !self.released + && self.timed_out_at != 0 + && self.timed_out() + && now.saturating_sub(self.timed_out_at) < POLL_NS + } /// A kill of `context` was published through this queue. pub(crate) fn exit_published(&self, context: &Arc) -> bool { self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -196,13 +228,13 @@ impl Queue { let Some(phase) = self.exit.as_ref().map(|exit| exit.phase) else { return Ok(()); }; - if !matches!(phase, Phase::Acknowledged | Phase::Unproven) { + if !matches!(phase, Phase::Acknowledged | Phase::Unproven | Phase::Witnessed) { return Ok(()); } // An unproven kill (acknowledged before the scheduler page showed the killed // state) is re-read on later passes and settles only once the page proves it, // even after its commands were retired by other means. - if self.active.len != 0 || phase == Phase::Unproven { + if (self.active.len != 0 || phase == Phase::Unproven) && phase != Phase::Witnessed { if !recovery_closed { return Ok(()); } @@ -217,7 +249,33 @@ impl Queue { self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; return Ok(()); } - if !self.active.iter().all(|active| { + if self.graph.m5 { + // A kill acknowledgement and the killed flag identify the control + // transaction, not the end of the firmware's reads, and M5 compute + // timestamp words stay zero even for executed work. The commands stay + // until the full-queue retirement witness covers the newest of them. + if self.active.len != 0 { + for active in self.active.iter() { + let Some(ticket) = active.ticket.as_ref() else { + return Ok(()); + }; + if self.valid_ticket(active, ticket).is_err() { + return Ok(()); + } + } + let newest = self.active.back().and_then(|active| active.ticket).ok_or(EIO)?; + if !super::retirement::quarantined_retired( + &newest, + self.graph.cursors()?, + self.witness_stamp()?, + self.active.len as u32, + self.submitted, + )? { + return Ok(()); + } + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Witnessed; + } else if !self.active.iter().all(|active| { active .packet .completion diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 6b1c49e6b4b73..8b3ab4b64dec1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -141,7 +141,20 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { } } -/// A zero-duration record fails before visibility polling. It proves execution did not occur; +/// Firmware retires the compute commands in flight at a restart without executing them and +/// stores a record whose end is at most this many timer ticks after its start (0 or 1 tick +/// observed). The shortest executed command record seen is 18 ticks. +pub(crate) const CANCELLED_TICKS: u64 = 1; + +/// The record shows the command never executed: zero duration, or, for a command that was in +/// flight across a recovery (`crossed`), the restart's cancellation record. +pub(crate) fn unexecuted(record: Observation, crossed: bool) -> bool { + let [start, end] = record.timestamps; + start == end || (crossed && start != 0 && end >= start && end - start <= CANCELLED_TICKS) +} + +/// A record showing the command never executed ([`unexecuted`]) fails before visibility +/// polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has /// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery @@ -162,10 +175,11 @@ pub(crate) fn poll( next_end: Option, record_aged: bool, ) -> Result> { - if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { + let crossed = matches!(recordless, Recordless::CrossedRecovery); + if observed.is_some_and(|record| unexecuted(record, crossed)) { return Err(ENODATA); } - let same_generation = !matches!(recordless, Recordless::CrossedRecovery); + let same_generation = !crossed; let completed = stamp_covers(stamp, ticket.kick) || (same_generation && observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b7037a1c35f75..32a7e9e19fd25 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -149,10 +149,10 @@ impl Queue { }) } - fn ensure_compute(&mut self, scs_pages: u16) -> Result { + fn ensure_compute(&mut self, command: &Validated) -> Result { // Validate every request, including an existing scheduler. Physical // admission happens after its DRM dependencies, in backend publication. - self.backend.ensure_compute(scs_pages)?; + self.backend.ensure_compute(command)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -218,10 +218,7 @@ impl Queue { self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { - return Err(EINVAL); - }; - self.ensure_compute(scratch.pages())?; + self.ensure_compute(&command)?; } // Only convert inputs when no independent host-prefix wait could keep // the retained source bookkeeping queued behind an external signal. @@ -298,8 +295,14 @@ impl Queue { &[] }; let count = frontier.len(); + // Only a command that copies or resets queries can observe timestamps the + // host has not yet published for its predecessors. + let query_access = match &command { + Validated::Compute { query_access, .. } => *query_access, + Validated::Render { .. } => true, + }; let timestamp_prefix = order.wait_through[index]; - let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { + let timestamps = if engine == Engine::Compute && query_access && timestamp_prefix.is_some() { let frontier = self.frontiers[index].as_slice(); let end = frontier .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); @@ -419,7 +422,7 @@ impl Queue { let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { - let payload = self.proof.validate( + let mut payload = self.proof.validate( self.context.vm(), self.range.clone(), &self.window, @@ -430,13 +433,18 @@ impl Queue { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; - let order = Order::new( + if !self.backend.supports_engine(engine) { return Err(ENOTSUPP); } + self.backend.prepare_command(&mut payload)?; + let mut order = Order::new( base, preceding, command.barriers, command.payload.prior(), engine, )?; + if let Some((barriers, prior)) = command.payload.tiling_prefixes() { + order.narrow_tiling(&Order::new(base, preceding, barriers, prior, engine)?)?; + } preceding[engine.index()] += 1; let destinations = match &payload { Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, @@ -482,6 +490,9 @@ impl crate::queue::Queue for Queue { ) -> Result { let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { + if !self.backend.supports_submission() { + return Err(ENOTSUPP); + } if self.context.status().get() != 0 { return Err(EIO); } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 76447a297c5ec..6c14fca21a816 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2,6 +2,9 @@ //! Physical paired tiling and fragment queues and their retained command storage. +pub(in crate::g17) mod generation; +mod configuration; +pub(in crate::g17) mod m5_render_submission; mod observation; pub(crate) use observation::Observed; mod dependencies; @@ -38,14 +41,20 @@ use retirement::{Ticket, Tracker}; /// are built: the covered earlier renders (their fragment completions) and the /// covered compute completion. Which kick waits for the render prefix is the /// command's choice (`RenderPass::fragment_barriers`); compute always orders -/// the tiling kick, since compute output may feed vertex work. +/// the tiling kick, since compute output may feed vertex work. A stage-prefix +/// command (`RenderPass::stage_prefixes`) also names the earlier renders its +/// tiling half waits for (`render_tiling`, whole-render completions) and, when +/// its fragment half waits on later compute work, that compute completion. #[derive(Copy, Clone)] pub(crate) struct RenderDependencies { pub(crate) render: [KickDependency; 2], pub(crate) render_count: usize, + pub(crate) render_tiling: [KickDependency; 2], + pub(crate) render_tiling_count: usize, pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + pub(crate) fragment_compute: Option, /// Fragment kicks of other pairs' overlapping same-VM renders. pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], pub(crate) hazard_count: usize, @@ -55,9 +64,12 @@ impl RenderDependencies { pub(crate) const EMPTY: Self = Self { render: [KickDependency::ZERO; 2], render_count: 0, + render_tiling: [KickDependency::ZERO; 2], + render_tiling_count: 0, tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + fragment_compute: None, hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], hazard_count: 0, }; @@ -76,12 +88,16 @@ pub(crate) struct RenderAddresses { pub(crate) struct PublicationLease { pub(crate) free_list_generation: u64, pub(crate) buffer_token: u64, + /// Hardware context for which the M5 token was selected. + pub(crate) buffer_context: Option, } /// Host services borrow the device-locked firmware state. Resource construction /// and command geometry preparation do not borrow this interface. pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; + /// The host's accepted recovery generation (no device access). + fn generation(&self) -> u64; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; /// Publishes `owners` in order with one pass over the QoS table. fn qos_publish_pair( @@ -91,7 +107,7 @@ pub(crate) trait Host { policy: crate::g17::fw::queue::Policy, ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; - fn install_pair(&mut self, registration: &RenderRegistration) -> Result; + fn install_pair(&mut self, registration: &Registration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; /// The next publication number, shared with compute (timeout ordering). fn next_publication(&mut self) -> Result; @@ -103,6 +119,13 @@ pub(crate) trait Host { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]>; + fn publish_configuration_pair( + &mut self, + priority: u8, + fragment: &WorkSlot, + tiling: &WorkSlot, + prepare: impl FnOnce(), + ) -> Result; fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; fn publish_retained_pair( &mut self, @@ -126,6 +149,9 @@ struct Prepared { packet: Arc, ordinal: u64, mcache: Option, + /// M5 kick register arrays and header flags, encoded with the commands. + register_arrays: Option<[[Option; 4]; 2]>, + kick_flags: Option, } struct Active { /// Publication number, shared with compute (crate::g17::timeout). @@ -150,9 +176,18 @@ struct Active { /// apart before it is republished; a single sample may straddle an update. const ABANDON_GRACE_NS: u64 = 2_000_000; +/// The firmware-specific host transaction; queue ownership remains shared. +pub(crate) enum Registration { + Neo(RenderRegistration), + M5(crate::g17::fw::m5::queue::HardwareSubmission), +} + /// Installed kick backing remains owned even when this pair has no logical /// owner. Commands retain their exact descriptor and allocation generations. pub(crate) struct Pair { + /// The firmware identity of an M5 render pool, pinned while this pair + /// uses the pool. G17P pools have fixed identities and need no pin. + pool_pin: Option, kick_aliases: Option<[Option; 2]>, memory: KBox, kicks: [kick::Queue; 2], @@ -169,6 +204,7 @@ pub(crate) struct Pair { deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -186,6 +222,8 @@ pub(crate) struct Pair { /// Retired, never-executed groups awaiting republication on this pair, in /// their original order. Nothing else is prepared on the pair before them. replay: Tracker>, + /// Recovery generation of this pair's last executed retirement. + executed_generation: Option, /// When the oldest parked group was parked or the last one republished. replay_since: Option, /// Scheduler page of the context of the last published command: the @@ -197,6 +235,19 @@ pub(crate) struct Pair { /// Completion values of the two kicks of the last publication, kept with the configuration /// cache: what each queue holds once that publication completed. published_completions: Option<[u64; 2]>, + /// M5: the queues' firmware identity (logical context, scheduler page, + /// owner) changed since the last configuration firmware consumed. Only + /// this sets the M5 configuration's context and QoS update bits; + /// `context_update` stays the host witness that retires `previous` and + /// gates the completion seed. Cleared once a publication carried it. + identity_update: bool, + /// Owner of the identities named by an M5 cold-install configuration + /// pair, until firmware consumed it or an Active took them over. + configuration: Option, + /// Hardware context named by the configurations firmware last took. + configured_context: Option, + /// QoS slot named by those configurations. + configured_qos: Option, } impl Pair { #[inline(never)] @@ -221,7 +272,12 @@ impl Pair { deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, ) -> Result> { + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS || owner == 0 @@ -238,14 +294,11 @@ impl Pair { } return Err(error); } - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters, variant.programs)?; + let layout = m5_render_submission::layout(&first.command, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; @@ -261,8 +314,10 @@ impl Pair { 0, word, variant, + board, )?; storage.write_init(kernel::try_init!(Self { + pool_pin, kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], @@ -279,6 +334,7 @@ impl Pair { deflake, descriptor_flags, variant, + board, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -290,11 +346,16 @@ impl Pair { previous: None, displaced: None, replay: Tracker::new(), + executed_generation: None, replay_since: None, context_update: true, executed: 0, switch: None, published_completions: None, + identity_update: true, + configuration: None, + configured_context: None, + configured_qos: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -318,15 +379,26 @@ impl Pair { pub(crate) fn buffer_id(&self) -> u8 { self.memory.manager.buffer_id() } + /// Whether the pair's M5 board takes the G17D forms. + fn g17d_forms(&self) -> bool { + self.board.is_some_and(|board| board.g17d_forms()) + } + /// The parameter-buffer state an M5 PB token is bound to. + pub(crate) fn m5_pb_state_va(&self) -> Option { + self.memory.graph.is_m5().then(|| self.memory.manager.state_va()) + } pub(crate) fn manager(&mut self) -> &mut buffer::Manager { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.len() != 0 + self.active.len() != 0 || self.replay.len() != 0 || self.configuration.is_some() } - /// A replay or an abandonment grace period needs the worker to poll. + /// Configuration consumption, a replay or an abandonment grace period + /// needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.configuration.is_some() + || self.replay.len() != 0 + || self.active.iter().any(|a| a.abandoned_since.is_some()) } pub(crate) fn replay_stalled(&self, now: u64) -> bool { self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) @@ -379,6 +451,8 @@ impl Pair { pass, attachments, ray_query, + resources: programs, + plan, .. } = &packet.command else { @@ -387,13 +461,22 @@ impl Pair { if packet.completion.status().get() != 0 { return Err(EIO); } + // The TA mesh rendering buffer is bound only to kicks with an API task + // (object) stage. Resolve it before any ordinal state is claimed so an + // allocation failure fails this kick cleanly. + let mesh_storage = if programs.as_ref().is_some_and(|p| p.uses_task()) { + self.memory.mesh_storage_register(alloc)? + } else { + 0 + }; let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = + m5_render_submission::layout(&packet.command, self.clusters, self.variant.programs)?; let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory @@ -414,10 +497,12 @@ impl Pair { self.memory.lifecycle }; let (scratch, metric) = self.memory.manager.scene_registers(scene)?; - let timestamp_offset = ordinal as usize % memory::RECORDS * 16; + let timestamp_offset = self.memory.timestamp_offset(ordinal); let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; let resources = abi::RenderResources { context_id: packet.context.id().try_into()?, + // On M5 too: a directly owned execution root has one generation, which the + // render tail token names. context_generation: packet.context.generation(), pb_slot: self.buffer_id().into(), free_list_slot: self.free_list.id().into(), @@ -436,6 +521,7 @@ impl Pair { gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, + mesh_storage, fragment_status_va: fragment_status, fragment_status_fw_va: fragment_fw, aux_fb_va: auxiliary, @@ -453,14 +539,14 @@ impl Pair { ordinal, ta: abi::RenderStage { qid: qids[0], - descriptor_va: self.memory.graph.descriptor_client(0, ordinal), + descriptor_va: self.memory.graph.descriptor_client(0, ordinal)?, state_word: word, stamp_va: addresses.stamps[0][0], aux_stamp_va: addresses.stamps[0][1], }, fragment: abi::RenderStage { qid: qids[1], - descriptor_va: self.memory.graph.descriptor_client(1, ordinal), + descriptor_va: self.memory.graph.descriptor_client(1, ordinal)?, state_word: word, stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], @@ -475,24 +561,37 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = + let result = if self.memory.graph.is_m5() { self.memory - .write_descriptors(&args, attachments, packet.context.id().try_into()?); - let mcache = match result { - Ok(mcache) => mcache, + .write_m5_commands( + &args, + &layout, + programs.as_ref().and(*plan), + programs.as_ref().map_or(0, |p| p.mesh_class()), + ) + .map(|(arrays, flags)| (None, Some(arrays), Some(flags))) + } else { + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?) + .map(|mcache| (mcache, None, None)) + }; + let (mcache, register_arrays, kick_flags) = match result { + Ok(encoded) => encoded, Err(error) => { self.memory.graph.release(ordinal)?; - self.memory.generation(ordinal)?.ordinal = None; + self.memory.generation(ordinal)?.clear(); self.memory.target.cancel(ordinal); return Err(error); } }; - self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; + self.memory.clear_timestamps(ordinal)?; self.memory.manager.claim_scene(ordinal, scene)?; self.prepared = Some(Prepared { packet: packet.clone(), ordinal, mcache, + register_arrays, + kick_flags, }); Ok(()) } @@ -502,28 +601,46 @@ impl Pair { /// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the /// tiling kick preserves the TA prefix across physical pairs. Otherwise the render prefix /// orders the tiling kick. A compute dependency always orders the tiling kick. +/// The completion classes name, for M5 kicks, what each barrier waits for: +/// 1 the tiling encoder, 2 a full channel or compute completion, 3 the +/// fragment encoder; the last class is the implicit parent's. struct RenderBarriers { - tiling: [KickDependency; 3], + tiling: [KickDependency; 5], tiling_count: usize, - fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], + fragment: [KickDependency; 4 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, + tiling_types: [u8; 6], + fragment_types: [u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS], } fn render_barriers( fragment_stage: bool, + stage_prefixes: bool, dependencies: &RenderDependencies, own_tiling: KickDependency, ) -> RenderBarriers { - let mut tiling_barriers = [KickDependency::ZERO; 3]; + let fragment_stage = fragment_stage || stage_prefixes; + let mut tiling_barriers = [KickDependency::ZERO; 5]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; + let mut fragment_barriers = [own_tiling; 4 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; + let mut tiling_types = [1; 6]; + let mut fragment_types = [3; 5 + crate::g17::hazard::PUBLICATION_BARRIERS]; + fragment_types[0] = 1; + for dependency in &dependencies.render_tiling[..dependencies.render_tiling_count] { + tiling_barriers[tiling_count] = *dependency; + tiling_types[tiling_count] = 2; + tiling_count += 1; + } for dependency in dependencies.render() { if fragment_stage { fragment_barriers[fragment_count] = *dependency; fragment_count += 1; } else { tiling_barriers[tiling_count] = *dependency; + // A full channel barrier uses the tiling encoder's class 2. + // Class 3 belongs to the fragment encoder. + tiling_types[tiling_count] = 2; tiling_count += 1; } } @@ -540,8 +657,31 @@ fn render_barriers( } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; + tiling_types[tiling_count] = 2; tiling_count += 1; } + if let Some(compute) = dependencies.fragment_compute { + fragment_barriers[fragment_count] = compute; + fragment_types[fragment_count] = 2; + fragment_count += 1; + } + if stage_prefixes { + // Only identical words of the same class are interchangeable. + let mut kept = 0; + for read in 0..tiling_count { + if (0..kept).any(|i| { + tiling_barriers[i].word() == tiling_barriers[read].word() + && tiling_types[i] == tiling_types[read] + }) { + continue; + } + tiling_barriers[kept] = tiling_barriers[read]; + tiling_types[kept] = tiling_types[read]; + kept += 1; + } + tiling_count = kept; + tiling_types[kept] = 1; + } // Only tile stores overlap: the tiling kick stays concurrent. for hazard in &dependencies.hazards[..dependencies.hazard_count] { fragment_barriers[fragment_count] = *hazard; @@ -552,6 +692,8 @@ fn render_barriers( tiling_count, fragment: fragment_barriers, fragment_count, + tiling_types, + fragment_types, } } @@ -571,12 +713,24 @@ impl Pair { channels::WorkSlot, kick::{KickArgs, KickQos}, queue::{ - self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, - TilingBinding, + FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, TilingBinding, }, }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + let m5 = self.memory.graph.is_m5(); + let context_id = packet.context.id(); + let qos_slot = packet.context.qos_id(); + if m5 { + if self.configured_context.is_some_and(|configured| configured != context_id) + || self.configured_qos.is_some_and(|configured| configured != qos_slot) + { + self.identity_update = true; + } + if lease.buffer_context.is_some_and(|bound| bound != context_id) { + return Err(EBUSY); + } + } self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; @@ -591,7 +745,13 @@ impl Pair { let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; - let work_slot = work.render_slot(self.slot)?; + // A channel moves to this pair only after every publication made through it is + // visible on both of its rings (the stamps cover ordinal + 1). + let work_slot = work.render_channel(self.slot, |old, published| { + host.stamps(old).is_ok_and(|stamps| { + stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) + }) + })?; self.memory.bind_work_slot(ordinal, qids, work_slot)?; let payload = work.channel_payload(self.slot, ordinal)?; let stamps = host.stamps(qids)?; @@ -625,16 +785,18 @@ impl Pair { data_master: DataMaster::Fragment as u8, }, ]; + // M5 carries the identity update bits only on a changed identity. + let identity_update = if m5 { self.identity_update } else { self.context_update }; let mut configs = [None; 2]; for stage in 0..2 { - configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { + let args = QueueConfigArgs { target: if stage == 0 { - QueueConfigTarget::Tiling(TilingBinding { + QueueConfigTarget::Tiling(Some(TilingBinding { pb_va: self.memory.manager.state_va(), pb_token: lease.buffer_token, pb_slot: self.buffer_id().into(), fragment_qid: qids[1], - }) + })) } else { QueueConfigTarget::Fragment }, @@ -642,11 +804,20 @@ impl Pair { kick_ring_fw_va: self.kicks[stage].firmware_va(), qid: qids[stage], install: !self.installed[stage], - completion_seed: covered[stage].then(|| { - kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)) - }), + completion_seed: (covered[stage] + && (!m5 || !self.installed[stage] || self.context_update)) + .then(|| { + // The predecessor's timestamp with the stage's scratch payload: the + // previous pass's on fragment; with the G17D forms 0 on tiling, as the + // G17D's tiling queues are seeded (its fragment checks read the paired + // fragment queue's payload, not this one). + let tiling_zero = stage == 0 && m5 && self.g17d_forms(); + kicks[stage].prev().completion_value(if tiling_zero { + 0 + } else { + payload.wrapping_sub(1) + }) + }), context_id: context.id().try_into()?, free_list: FreeListBinding { va: self.free_list.control_va(), @@ -656,12 +827,35 @@ impl Pair { scheduler_va: context.scheduler_va(), policy, qos_slot: qos, - qos_update: self.context_update, + qos_update: identity_update, owner_pid: context.owner_pid(), - context_update: self.context_update, - })?); + context_update: identity_update, + }; + configs[stage] = Some(if m5 { + crate::g17::fw::m5::queue::config( + &args, + crate::g17::fw::m5::queue::ConfigProfile { + soft_fault_enabled: true, + extended: false, + parameter_manager: (stage == 0).then(|| u32::from(self.buffer_id())), + }, + )? + } else { + QueueConfig::new(&args)? + }); } let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; + if m5 && self.registered.is_none() { + if self.installed == [true; 2] { + // The cold-install configuration pair must be consumed first. + let cursors = self.memory.graph.cursors()?; + if cursors.iter().zip(self.cursors).any(|(c, end)| c.read != end) { + return Err(EBUSY); + } + } else { + return self.publish_m5_configuration(&packet, configs, kicks, policy, ring, host); + } + } let cache = if self.recovery_generation == epoch { self.configs } else { @@ -693,10 +887,7 @@ impl Pair { if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); } - let queues = qids.map(|qid| { - self.memory.graph.queues.gpu_va() - + u64::from(qid) * size_of::() as u64 - }); + let queues = qids.map(|qid| self.memory.graph.queue_va(qid)); let Validated::Render { pass, ray_query, .. } = &packet.command @@ -704,13 +895,38 @@ impl Pair { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); + let barriers = + render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling); let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; + // With the G17D forms a fragment kick with history also names its parent's + // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). + let ancestor = if m5 && self.g17d_forms() { + self.kicks[1].ancestor() + } else { + None + }; + let mut fragment_types = [0u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS]; + fragment_types[..barriers.fragment_count + 1] + .copy_from_slice(&barriers.fragment_types[..barriers.fragment_count + 1]); + fragment_types[barriers.fragment_count + 1] = 1; + let fragment_typed = barriers.fragment_count + 1 + usize::from(ancestor.is_some()); + let barrier_types = [ + &barriers.tiling_types[..barriers.tiling_count + 1], + &fragment_types[..fragment_typed], + ]; + let prepared = self.prepared.as_ref().ok_or(EIO)?; + let m5_kick = match (prepared.register_arrays, prepared.kick_flags) { + (Some(arrays), Some(flags)) if m5 => Some((arrays, flags)), + (None, None) if !m5 => None, + _ => return Err(EIO), + }; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], parent: parents[stage], + ancestor: if stage == 1 { ancestor } else { None }, barriers, + dependency_types: m5.then_some(barrier_types[stage]), descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), queue_va: queues[stage], qos: KickQos { @@ -722,36 +938,60 @@ impl Pair { }, }, mcache, - event_mask: abi::RENDER_KICK_EVENT_MASK, + // M5 tiling kicks also retain the last event of the mask. + event_mask: if m5 && stage == 0 { + [0, 0, 0, 1u64 << 62] + } else { + abi::RENDER_KICK_EVENT_MASK + }, register_arrays: arrays, - scs_enabled: ray_query[stage].scratch.enabled(), + scs_enabled: match m5_kick { + Some((_, flags)) => flags.scratch[stage], + None => ray_query[stage].scratch.enabled(), + }, + compute_matrix: m5_kick.is_some_and(|(_, flags)| flags.matrix[stage]), priority: policy.priority(), }; let ta_args = args( 0, tiling_barriers, - abi::TaDescriptor::register_bindings( - self.memory.graph.descriptor_client(0, ordinal), - ray_query[0].scratch.enabled(), - self.variant.programs, - )?, + match m5_kick { + Some((arrays, _)) => arrays[0], + None => abi::TaDescriptor::register_bindings( + self.memory.graph.descriptor_client(0, ordinal)?, + ray_query[0].scratch.enabled(), + self.variant.programs, + )?, + }, None, ); let fragment_args = args( 1, &barriers.fragment[..barriers.fragment_count], - abi::FragmentDescriptor::register_bindings( - self.memory.graph.descriptor_client(1, ordinal), - ray_query[1].scratch.enabled(), - self.variant.programs, - )?, + match m5_kick { + Some((arrays, _)) => arrays[1], + None => abi::FragmentDescriptor::register_bindings( + self.memory.graph.descriptor_client(1, ordinal)?, + ray_query[1].scratch.enabled(), + self.variant.programs, + )?, + }, mcache, ); - let registration = RenderRegistration::new( - (qids[1], self.kicks[1].low_va()), - (qids[0], self.kicks[0].low_va()), - policy.priority(), - )?; + let registration = if m5 { + Registration::M5(crate::g17::fw::m5::queue::HardwareSubmission::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + policy.priority(), + true, + )?) + } else { + Registration::Neo(RenderRegistration::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + policy.priority(), + )?) + }; let [tiling_kick, fragment_kick] = &mut self.kicks; let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; @@ -803,10 +1043,16 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; - let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); - let inner = memory.graph.support.word(0x28000)?; - let next_inner = (next_ordinal as u32).wrapping_mul(2); + // M5 graphs keep no G17P event and inner-producer words. + let legacy_state = if m5 { + None + } else { + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; + let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); + let inner = memory.graph.support.word(0x28000)?; + let next_inner = (next_ordinal as u32).wrapping_mul(2); + Some((event_slot, next_event, inner, next_inner)) + }; let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; let published = host.next_publication()?; @@ -824,7 +1070,11 @@ impl Pair { previous, }, kicks, - context_update: self.context_update, + // On M5 the cold-install configuration may already have + // cleared context_update: the first publication after a + // rebind still retires an unpublished `previous`. + context_update: self.context_update + || (m5 && self.previous.as_ref().is_some_and(|p| p.publication.is_none())), failed: false, drained_result: None, abandoned_since: None, @@ -833,9 +1083,25 @@ impl Pair { .map_err(|_| EBUSY)?; self.prepared = None; packet.mark_published(); - lanes[1].publish_prefix(); + work.channel_published(self.slot, qids, next_ordinal); + // The Active now retains this Packet and its identity and QoS pins; it + // replaces the configuration-only identity owner. + if let Some(configuration) = self.configuration.take() { + configuration.release(); + } + // M5 fills both lanes now; the host-port transaction precedes either + // producer, which publish_inner advances. + if m5 { + for lane in &lanes { + lane.prepare(); + } + } else { + lanes[1].publish_prefix(); + } fragment_kick.commit(); - lanes[1].publish_kick(); + if !m5 { + lanes[1].publish_kick(); + } tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { @@ -860,7 +1126,15 @@ impl Pair { pm_submission.commit(); Ok(()) }, - || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), + || match legacy_state { + Some((event_slot, next_event, inner, next_inner)) => { + lanes[0].publish_tiling(event_slot, next_event, inner, next_inner) + } + None => { + lanes[1].publish_kick(); + lanes[0].publish_kick(); + } + }, ) })(); let outer_next = match result { @@ -903,6 +1177,13 @@ impl Pair { self.recovery_generation = epoch; self.installed = [true; 2]; self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); + if m5 { + // Both configurations of this publication carried any update bits, + // so firmware applies them before this kick. + self.configured_context = Some(context_id); + self.configured_qos = Some(qos_slot); + self.identity_update = false; + } self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -923,6 +1204,90 @@ impl Pair { ) } + /// M5 cold install: the queues consume their initial configuration + /// records, published as one configuration pair, before the first kick + /// pair. Returns EBUSY so the command publishes once they are consumed. + fn publish_m5_configuration( + &mut self, + packet: &Arc, + configs: [crate::g17::fw::queue::QueueConfig; 2], + kicks: [KickTimestamp; 2], + policy: crate::g17::fw::queue::Policy, + ring: u8, + host: &mut impl Host, + ) -> Result { + use crate::g17::fw::queue::ITEM_RING_ENTRIES; + if self.configuration.is_some() { + return Err(EBUSY); + } + let ordinal = self.ordinal; + let qids = self.qids(); + let lanes = [ + publication::Lane::new( + &self.memory.graph, + 0, + ordinal, + self.cursors[0], + configs[0], + true, + kicks[0], + false, + )?, + publication::Lane::new( + &self.memory.graph, + 1, + ordinal, + self.cursors[1], + configs[1], + true, + kicks[1], + false, + )?, + ]; + let ends = self.cursors.map(|base| (base + 1) % ITEM_RING_ENTRIES); + let slot = |stage: usize| -> Result { + WorkSlot::new( + DataMaster::Compute, + self.memory.graph.queue_va(qids[stage]), + qids[stage], + ends[stage].try_into()?, + true, + ) + }; + let (tiling, fragment) = (slot(0)?, slot(1)?); + self.configuration = Some(configuration::Configuration::new( + &packet.context, + ends, + host.generation(), + )?); + let configuration = self.configuration.as_mut().ok_or(EIO)?; + let configured = + host.publish_configuration_pair(policy.priority(), &fragment, &tiling, || { + // Ownership exists before either item producer can become visible. + configuration.mark_visible(); + for lane in &lanes { + lane.publish_prefix(); + } + }); + if let Err(error) = configured { + if self.configuration.as_ref().is_some_and(|owner| !owner.visible()) { + self.configuration.take().ok_or(EIO)?.release(); + } + return Err(error); + } + self.cursors = ends; + self.installed = [true; 2]; + self.configured_context = Some(packet.context.id()); + self.configured_qos = Some(packet.context.qos_id()); + self.configs = [Some(configs[0]), Some(configs[1])]; + self.context_update = false; + self.identity_update = false; + self.outer_started |= ring; + packet.context.mark_published(); + host.notify_render(policy.priority())?; + Err(EBUSY) + } + /// Everything that must hold before a publication changes any state: no alias work /// pending, a healthy device epoch, room in the active ring, this slot's packet, the /// prepared descriptors for this ordinal, a clean completion status and a registration @@ -1039,6 +1404,9 @@ impl retirement::Memory for Observer<'_, H> { fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { self.memory.pass(ordinal) } + fn stage_timestamps(&mut self, ordinal: u64) -> Result<[u64; 4]> { + self.memory.stage_timestamps(ordinal) + } } impl Pair { @@ -1085,14 +1453,16 @@ impl Pair { /// Latches when the front group executed and the firmware consumed its ring entries while /// `stamps` still lag it, and reports whether that has held for /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it - /// on its pass record instead of waiting for stamps no later event will move. + /// on its pass record instead of waiting for stamps no later event will move. Not on M5, + /// whose pass record can precede the group's writes: as for M5 compute, only the stamps + /// return the group's buffers. pub(crate) fn note_stamp_wait( &mut self, stamps: [u32; 2], state: u32, now: u64, ) -> Result { - if self.quarantined || self.prepared.is_some() || state != 0 { + if self.quarantined || self.prepared.is_some() || state != 0 || self.memory.graph.is_m5() { return Ok(false); } let Some(front) = self.active.front_mut() else { @@ -1221,10 +1591,9 @@ impl Pair { /// Sample PM feedback only after this pair's exact retirement. Busy pairs /// keep their current capacity and let firmware request growth as needed. pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { - let Validated::Render { pass, .. } = &packet.command else { - return Err(EINVAL); - }; - let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; + let required = + m5_render_submission::layout(&packet.command, self.clusters, self.variant.programs)? + .tvb_blocks; self.memory .manager .note_required(required, crate::g17::now_ns()); @@ -1394,6 +1763,19 @@ impl Pair { self.retire_front(None, state, Some(observed), host, defer) } + /// The M5 cold-install configuration pair was consumed exactly: its + /// identity owner is released. + fn retire_configuration(&mut self, host: &impl Host) -> Result { + let Some(configuration) = self.configuration.as_ref() else { + return Ok(false); + }; + if !configuration.consumed(self.memory.graph.cursors()?, host.generation()) { + return Ok(false); + } + self.configuration.take().ok_or(EIO)?.release(); + Ok(true) + } + fn retire_front( &mut self, masks: Option<[u64; 2]>, @@ -1401,6 +1783,19 @@ impl Pair { observed: Option, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let configuration_retired = state == 0 && self.retire_configuration(host)?; + let retired = self.retire_front_command(masks, state, observed, host, defer)?; + Ok(retired || configuration_retired) + } + + fn retire_front_command( + &mut self, + masks: Option<[u64; 2]>, + state: u32, + observed: Option, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { if self.quarantined || state != 0 || !self.selected(masks) { return Ok(false); @@ -1478,13 +1873,29 @@ impl Pair { ); defer(Deferred::Retired( front.packet.completion.clone(), - Ok(timestamps), + front.packet.check_firmware_producers().map(|()| timestamps), ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + self.executed_generation = Some(host.generation()); Ok(true) } + /// An executed group of this pair retired in `generation` or later. + pub(crate) fn executed_since(&self, generation: u64) -> bool { + self.executed_generation.is_some_and(|executed| executed >= generation) + } + /// Published work of this pair is still live: in flight and not parked. + pub(crate) fn live_unparked(&self) -> bool { + self.replay.len() == 0 && self.active.len() != 0 + } + /// The oldest parked group precedes a command waiting through `through`. + pub(crate) fn replay_through(&self, through: Option) -> bool { + self.replay + .front() + .is_some_and(|packet| crate::g17::job::Order::contains(through, packet.order.sequence)) + } + /// Every retained group was retired and unlinked by firmware without /// executing (job list empty, stamps cover it, no recovery, fragment end /// never written): after another VM's recovery the firmware drops an @@ -2035,6 +2446,9 @@ impl Pair { /// Both processors have stopped before the caller reaches this method. /// VM ownership settles before quarantined admission is rehabilitated. pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(configuration) = self.configuration.take() { + configuration.release(); + } if let Some(prepared) = self.prepared.as_ref() { defer(Deferred::Unpublished( prepared.packet.completion.clone(), @@ -2116,9 +2530,7 @@ fn install_memory( descriptors.word(offset + 12)?, ]; let previous = words.map(|word| word.load(Ordering::Relaxed)); - let pages: u32 = (memory.manager.blocks() - * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) - .try_into()?; + let pages = memory.manager.page_count()?; let next = crate::g17::fw::buffer::reload_descriptor( previous, memory.manager.page_list_va(), @@ -2158,6 +2570,7 @@ fn install_memory( /// A binding prepared while the registry exclusively lends the idle pair. /// Its frontier prevents a stale off-lock result from replacing another owner. pub(crate) struct PreparedBinding { + pool_pin: Option, previous_context: Arc, previous_owner: Option, ordinal: u64, @@ -2179,8 +2592,14 @@ pub(crate) struct RetiredBinding { buffer_id: u8, publication: Option, ready: bool, + /// An M5 same-context pool rebind: the old pool stays until firmware + /// acknowledged its release. + wait_for_pool_release: bool, } impl RetiredBinding { + pub(crate) fn wait_for_pool_release(&self) -> bool { + self.wait_for_pool_release + } pub(crate) fn context(&self) -> &Arc { &self.context } @@ -2198,6 +2617,7 @@ impl Pair { return; } self.context_update = false; + self.identity_update = false; if let Some(previous) = self.previous.as_mut() { if previous .publication @@ -2326,6 +2746,28 @@ impl Pair { && previous_ready && self.idle_for_context(self.context())?) } + /// An M5 pool may be replaced without changing the pair's live logical + /// owner. Commands already using the pair must drain, and an earlier + /// replacement must have retired, unless nothing was published since it: + /// that binding never reached firmware and is superseded directly. + pub(crate) fn bindable(&self, owner: u64, context: &Arc) -> Result { + if self.memory.graph.is_m5() + && self.owner == Some(owner) + && Arc::ptr_eq(context, self.context()) + { + return Ok(self.displaced.is_none() + && self.previous_supersedable() + && self.idle_for_context(context)?); + } + self.reusable() + } + /// No `previous`, or one that nothing has been published after: a warm + /// (memoryless) replacement with no publication ordinal and not ready. + fn previous_supersedable(&self) -> bool { + self.previous.as_ref().is_none_or(|p| { + p.memory.is_none() && p.publication.is_none() && !p.ready && self.context_update + }) + } /// Logical close leaves the physical channel identity and backing retained. pub(crate) fn release_owner(&mut self) { self.owner = None; @@ -2347,7 +2789,11 @@ impl Pair { pm_generation: Option, first: &Arc, ) -> Result { - if !self.reusable()? + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } + if !self.bindable(owner, &context)? || owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() @@ -2355,8 +2801,15 @@ impl Pair { return Err(EBUSY); } alloc.uat.vm_context_mask(context.vm())?; - let same_vm = Arc::ptr_eq(self.context().status(), context.status()); - let (memory, aliases, kicks) = if same_vm { + // On M5 only the same logical context (a pool rebind) keeps the graph: + // any other context gets its own graph and queue records, and a live + // queue record is never retargeted. + let keep_graph = if self.memory.graph.is_m5() { + Arc::ptr_eq(self.context(), &context) + } else { + Arc::ptr_eq(self.context().status(), context.status()) + }; + let (memory, aliases, kicks) = if keep_graph { if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { return Err(EINVAL); } @@ -2365,10 +2818,8 @@ impl Pair { let metrics = metrics.ok_or(EINVAL)?; let buffer_id = buffer_id.ok_or(EINVAL)?; let pm_generation = pm_generation.ok_or(EINVAL)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = + m5_render_submission::layout(&first.command, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -2385,10 +2836,12 @@ impl Pair { self.ordinal, first.completion.work_state()?.word(), self.variant, + self.board, )?; (Some(memory), Some(persistent), Some(aliases)) }; Ok(PreparedBinding { + pool_pin, previous_context: self.context().clone(), previous_owner: self.owner, ordinal: self.ordinal, @@ -2407,7 +2860,7 @@ impl Pair { alloc: &Allocator<'_>, descriptors: &crate::g17::object::KernelObject, ) -> Result { - if !self.reusable()? + if !self.bindable(binding.owner, &binding.context)? || binding.ordinal != self.ordinal || binding.previous_owner != self.owner || !Arc::ptr_eq(&binding.previous_context, self.context()) @@ -2416,20 +2869,44 @@ impl Pair { return Err(EBUSY); } let foreign = binding.memory.is_some(); + let same_context = Arc::ptr_eq(self.context(), &binding.context); if let Some(memory) = binding.memory.as_mut() { - install_memory( + // A VM holds one graph's pool mappings per slot. On M5 a new + // context of the same VM gets its own graph: move the drained + // outgoing graph's mappings out first, and back if the install fails. + let same_vm = Arc::ptr_eq(self.context().status(), binding.context.status()); + let moved = if self.memory.graph.is_m5() && same_vm { + self.memory.take_pool_mappings() + } else { + None + }; + let installed = install_memory( alloc, descriptors, self.slot, memory, &mut binding.aliases, &mut binding.kicks, - )?; + ); + if let Err(error) = installed { + if let Some((slot, mappings)) = moved { + self.memory.put_pool_mappings(slot, mappings)?; + } + return Err(error); + } + drop(moved); } else { - self.memory.set_owner(self.qids(), &binding.context)?; - } + // The same logical context keeps its M5 queue records: their host + // words already name it, and a rewrite would store firmware-owned + // words back while the previous kick may still execute. + self.memory.set_owner(self.qids(), &binding.context, !same_context)?; + } + let wait_for_pool_release = self.memory.graph.is_m5() + && same_context + && !Arc::ptr_eq(&self.free_list, &binding.pool); let old_context = self.context().clone(); let old_buffer = self.buffer_id(); + self.pool_pin = binding.pool_pin.take(); let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); let old_memory = binding .memory @@ -2442,6 +2919,7 @@ impl Pair { buffer_id: old_buffer, publication: None, ready: foreign, + wait_for_pool_release, }; if foreign { self.displaced = Some(retired); @@ -2455,6 +2933,9 @@ impl Pair { self.graph_started = false; } self.context_update = true; + if foreign || !same_context { + self.identity_update = true; + } if foreign { self.cursors = [0; 2]; self.configs = [None; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs b/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs new file mode 100644 index 0000000000000..856a82b82a767 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initial M5 configuration can reach firmware before an Active command. +//! Its captured identities therefore have an independent owner. No aggregate +//! work-state node is referenced by this configuration-only item prefix. + +use super::{retirement::Cursors, Context}; +use crate::mmu::ContextPin; +use kernel::{prelude::*, sync::Arc}; + +pub(super) struct Configuration { + context: Arc, + pin: ContextPin, + qos: u8, + targets: [u32; 2], + generation: u64, + visible: bool, +} +impl Configuration { + pub(super) fn new(context: &Arc, targets: [u32; 2], generation: u64) -> Result { + let pin = context.pin()?; + let qos = match context.pin_qos() { + Ok(qos) => qos, + Err(error) => { context.unpin(pin); return Err(error); } + }; + Ok(Self { context: context.clone(), pin, qos, targets, generation, visible: false }) + } + pub(super) fn mark_visible(&mut self) { self.visible = true; } + pub(super) fn visible(&self) -> bool { self.visible } + pub(super) fn consumed(&self, cursors: [Cursors; 2], generation: u64) -> bool { + self.visible && self.generation == generation && cursors.iter().zip(self.targets).all(|(cursor, target)| { + cursor.done == target && cursor.read == target && cursor.write == target + }) + } + /// Only before visibility, after exact configuration consumption, after + /// transfer to an Active owning the same pinned Packet, or processor stop. + pub(super) fn release(self) { + self.context.unpin_qos(self.qos); + self.context.unpin(self.pin); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs new file mode 100644 index 0000000000000..d03857cc10a56 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-command tiling, fragment-status and auxiliary storage. Mappings precede +//! their backing owners; a generation is reusable only after paired retirement. + +use crate::g17::{ + buffer::{self, BlockGeometry}, + fw::{ + m5::{initdata::Board, render_registers::ContextStore}, + render::{storage, RenderPass}, + }, + object::{Allocator, CpuMap, KernelObject, Placement}, +}; +use crate::{hw::t8140::{dynamic, resources as cfg}, mmu}; +use kernel::prelude::*; + +/// Offsets within a single client allocation, not fixed GPU addresses. +#[derive(Copy, Clone)] +pub(in crate::g17) struct Layout { + pub(in crate::g17) bytes: usize, + pub(in crate::g17) deflake: usize, + pub(in crate::g17) status: usize, + pub(in crate::g17) auxiliary: usize, + /// The viewport words the host writes before a pass, if it writes them. + pub(in crate::g17) viewport: Option, + pub(in crate::g17) viewport_control: u32, +} +impl Layout { + pub(in crate::g17) const NEO: Self = Self { + bytes: storage::STATE_SIZE, deflake: storage::DEFLAKE, + status: storage::TA_STATUS, auxiliary: storage::AUXILIARY, + viewport: Some(0x900), viewport_control: 0xc00, + }; + + /// M5: the context store at +0 ([`ContextStore`]), then the stage status page and the + /// auxiliary framebuffer, each on its own page above the store's extent. With the + /// two-cluster store the three end at 64 KiB; with the G17D's eight-cluster store at + /// 80 KiB. The host writes the viewport words into the state + /// stream, except with the G17D forms: there the stream is the hardware's, which overwrites it + /// when a pass is stored, and the host leaves the store alone. + const fn m5_for(store: ContextStore, host_viewport: bool) -> Self { + let page = mmu::UAT_PGSZ; + let status = store.extent.div_ceil(page) * page; + let auxiliary = status + page; + Self { + bytes: auxiliary + storage::AUXILIARY_SIZE, deflake: 0, status, auxiliary, + viewport: if host_viewport { Some(store.stream + 0x40) } else { None }, + viewport_control: 0x3000, + } + } + + /// `board`'s M5 layout. + pub(in crate::g17) const fn m5(board: &Board) -> Self { + Self::m5_for(ContextStore::selected(board), !board.g17d_forms()) + } + + pub(in crate::g17) fn prepare(self, state: &mut [u8], pass: &RenderPass) -> Result { + if state.len() < self.bytes { return Err(EINVAL); } + if let Some(viewport_offset) = self.viewport { + let viewport = storage::Viewport::new(pass.width, pass.height)? + .with_control(self.viewport_control); + let offset = self.deflake + viewport_offset; + let view = state.get_mut(offset..offset + size_of::()) + .ok_or(EINVAL)?.as_mut_ptr().cast::(); + if view.align_offset(align_of::()) != 0 { return Err(EINVAL); } + // SAFETY: The complete integer-only view is aligned and in bounds. + // The caller exclusively owns this unpublished command generation. + unsafe { viewport.apply(&mut *view) }; + } + state[self.auxiliary..self.auxiliary + storage::AUXILIARY_SIZE].fill(0); + Ok(()) + } +} + +const _: () = { + let two = Layout::m5_for(ContextStore::of(2), true); + assert!(two.bytes == 0x10000 && two.status == 0x4000 && two.auxiliary == 0x8000); + assert!(matches!(two.viewport, Some(0x9c0))); + let eight = Layout::m5_for(ContextStore::of(8), false); + assert!(eight.bytes == 0x14000 && eight.status == 0x8000 && eight.auxiliary == 0xc000); +}; + +enum Views { + Neo { + deflake: mmu::KernelMapping, + auxiliary: mmu::KernelMapping, + fragment_client: mmu::KernelMapping, + fragment: KernelObject, + }, + M5 { status_fw: mmu::KernelMapping, board: &'static Board }, +} + +pub(in crate::g17) struct Generation { + views: Views, + state: KernelObject, + pub(super) ordinal: Option, +} +impl Generation { + pub(in crate::g17) fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm, + board: Option<&'static Board>) -> Result { + let (range, layout) = if alloc.uat.is_m5() { + (BlockGeometry::M5_COMPACT, Layout::m5(board.ok_or(EINVAL)?)) + } else { (dynamic::CLIENT_LOWER, Layout::NEO) }; + KernelObject::new(alloc.dev, vm, Placement::In(range, cfg::TVB_PAGE), layout.bytes, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined) + } + + #[inline(never)] + pub(in crate::g17) fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject, + board: Option<&'static Board>) -> Result> { + let views = if alloc.uat.is_m5() { + let board = board.ok_or(EINVAL)?; + let layout = Layout::m5(board); + let status_fw = state.map_range(alloc.uat.kernel_vm(), + layout.status..layout.status + mmu::UAT_PGSZ, + alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, 0)?; + Views::M5 { status_fw, board } + } else { + let deflake = buffer::compact_alias(&state, vm, + storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ)?; + let auxiliary = state.map_range(vm, + storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, + cfg::AUXILIARY, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, 0)?; + let fragment = alloc.kernel(mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let fragment_client = buffer::compact_alias(&fragment, vm, 0..mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, mmu::UAT_PGSZ)?; + Views::Neo { deflake, auxiliary, fragment_client, fragment } + }; + Ok(KBox::new(Self { views, state, ordinal: None }, GFP_KERNEL)?) + } + + pub(in crate::g17) fn prepare(&mut self, ordinal: u64, pass: &RenderPass) -> Result { + if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } + let layout = match &mut self.views { + Views::Neo { fragment, .. } => { + fragment.bytes_mut().fill(0); + Layout::NEO + } + Views::M5 { board, .. } => { + let layout = Layout::m5(board); + self.state.bytes_mut()[layout.status..layout.status + mmu::UAT_PGSZ].fill(0); + layout + } + }; + layout.prepare(self.state.bytes_mut(), pass)?; + self.ordinal = Some(ordinal); + Ok(()) + } + + pub(in crate::g17) fn addresses(&self) -> [u64; 5] { + match &self.views { + Views::Neo { deflake, auxiliary, fragment_client, fragment } => [ + deflake.iova(), self.state.gpu_va() + storage::TA_STATUS as u64, + fragment_client.iova(), fragment.gpu_va(), auxiliary.iova(), + ], + Views::M5 { status_fw, board } => { + let base = self.state.gpu_va(); + let layout = Layout::m5(board); + [base + layout.deflake as u64, base + layout.status as u64, + base + layout.status as u64 + 8, status_fw.iova() + 8, + base + layout.auxiliary as u64] + } + } + } + + pub(in crate::g17) fn retire(&mut self, ordinal: u64) -> Result { + if self.ordinal != Some(ordinal) { return Err(EIO); } + self.ordinal = None; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs new file mode 100644 index 0000000000000..333846d3f83c6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 render admission, target layout and command inputs on the shared submission path. + +use kernel::prelude::*; +use crate::g17::fw::m5::{render as cmd, render_registers as regs}; +use crate::g17::{buffer::target, command::Validated, fw::render::{RenderArgs, RenderPass, RegisterPrograms}}; + +/// Compressed single-layer MSAA ZLS admission: D32 depth with optional plain +/// stencil, and the compressed Z/S form. +pub(crate) fn depth_msaa_compression_enabled() -> bool { + crate::module_parameters::m5_depth_msaa_compression.value() == 1 +} + +pub(crate) fn validate(pass: &RenderPass) -> Result { + // The hardware carries the sample stride in eight bits. The 64-byte limit + // belonged to the first qualified profile, not the M5 imageblock. + let single_sample = pass.samples_log2 == 0 + && matches!((pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16)) + && pass.sample_size % 8 == 0 + && u32::from(pass.sample_size) * u32::from(pass.utile_width) + * u32::from(pass.utile_height) <= 65536 + && (pass.sample_size != 0 || pass.layers == 1); + // ZLS addresses/actions and page strides are sample-independent. + // Retain the checked plain layout; compression is not admitted here, + // except in the layered compressed Z/S form (every enabled Z/S load or + // store compressed, a compression address per compressed plane, depth + // and stencil strides as supplied by userspace). The same form in + // single-layer 2x/4x MSAA depth/stencil passes is admitted only with the + // opt-in. + let plain_zls = pass.zls_control == 0 + || (pass.zls_control & 0x154 == 0 + && pass.depth.compression_va == 0 && pass.stencil.compression_va == 0); + let multisample = (plain_zls || layered_compressed_zls(pass) + || (depth_msaa_compression_enabled() && (single_layer_compressed_depth_zls(pass) + || single_layer_compressed_zls(pass)))) + && matches!(pass.samples_log2, 1..=3) + && matches!((pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16)) + && pass.sample_size >= 8 && pass.sample_size % 8 == 0 + && (u32::from(pass.utile_width) * u32::from(pass.utile_height) + << pass.samples_log2) <= 2048 + && (u32::from(pass.sample_size) * u32::from(pass.utile_width) + * u32::from(pass.utile_height) << pass.samples_log2) <= 65536; + if !(single_sample || multisample) + || (pass.zls_control != 0 && pass.depth.va == 0 && pass.stencil.va == 0) { + return Err(EOPNOTSUPP); + } + Ok(()) +} + +/// Layered multisample depth/stencil in the compressed Z/S form. +fn layered_compressed_zls(pass: &RenderPass) -> bool { + pass.layers > 1 && compressed_zls_form(pass) +} + +/// Single-layer 2x/4x depth/stencil in the compressed Z/S form, the same form +/// as the layered one, compressed stencil included. +fn single_layer_compressed_zls(pass: &RenderPass) -> bool { + pass.layers == 1 && matches!(pass.samples_log2, 1 | 2) && compressed_zls_form(pass) +} + +/// Multisample depth/stencil in the compressed Z/S form: each enabled load or +/// store of a plane is compressed (ZLS bits: Z load 15 / compress 2, S load 14 +/// / compress 4, Z store 19 / compress 6, S store 18 / compress 8; the depth +/// format bits 25:26 are allowed), a plane +/// with any compression has a compression address, and no other ZLS bits. +fn compressed_zls_form(pass: &RenderPass) -> bool { + const Z_LOAD: u64 = 1 << 15; + const S_LOAD: u64 = 1 << 14; + const Z_STORE: u64 = 1 << 19; + const S_STORE: u64 = 1 << 18; + const Z_LOAD_COMPRESS: u64 = 1 << 2; + const S_LOAD_COMPRESS: u64 = 1 << 4; + const Z_STORE_COMPRESS: u64 = 1 << 6; + const S_STORE_COMPRESS: u64 = 1 << 8; + /// Depth format (bits 25:26) is a property of the plane, not of layering. + const Z_FORMAT: u64 = 3 << 25; + const KNOWN: u64 = Z_LOAD | S_LOAD | Z_STORE | S_STORE | Z_LOAD_COMPRESS | S_LOAD_COMPRESS + | Z_STORE_COMPRESS | S_STORE_COMPRESS | Z_FORMAT; + let control = pass.zls_control; + let z_compressed = control & (Z_LOAD_COMPRESS | Z_STORE_COMPRESS) != 0; + let s_compressed = control & (S_LOAD_COMPRESS | S_STORE_COMPRESS) != 0; + control != 0 + && control & !KNOWN == 0 + && (control & Z_LOAD == 0 || control & Z_LOAD_COMPRESS != 0) + && (control & S_LOAD == 0 || control & S_LOAD_COMPRESS != 0) + && (control & Z_STORE == 0 || control & Z_STORE_COMPRESS != 0) + && (control & S_STORE == 0 || control & S_STORE_COMPRESS != 0) + && (!z_compressed || (pass.depth.va != 0 && pass.depth.compression_va != 0)) + && (!s_compressed || (pass.stencil.va != 0 && pass.stencil.compression_va != 0)) + && (z_compressed || pass.depth.compression_va == 0) + && (s_compressed || pass.stencil.compression_va == 0) +} + +/// Single-layer D32 depth compression, with optional plain S8. +/// The compressed Z/S form is checked separately. Both depth compression bits +/// are required because partial rendering forces depth load and store even +/// when one of those operations is disabled in the final render pass. +fn single_layer_compressed_depth_zls(pass: &RenderPass) -> bool { + const Z_LOAD: u64 = 1 << 15; + const S_LOAD: u64 = 1 << 14; + const Z_STORE: u64 = 1 << 19; + const S_STORE: u64 = 1 << 18; + const Z_COMPRESS: u64 = (1 << 2) | (1 << 6); + const KNOWN: u64 = Z_LOAD | S_LOAD | Z_STORE | S_STORE | Z_COMPRESS; + pass.layers == 1 + && matches!(pass.samples_log2, 1 | 2) + && pass.zls_control & !KNOWN == 0 + && pass.zls_control & Z_COMPRESS == Z_COMPRESS + && pass.depth.va != 0 && pass.depth.compression_va != 0 + && pass.stencil.compression_va == 0 + && (pass.zls_control & (S_LOAD | S_STORE) == 0 || pass.stencil.va != 0) +} + +pub(crate) fn parameter_geometry(blocks: usize) -> Result { + let mut info = core::mem::MaybeUninit::::zeroed(); + // SAFETY: The complete sysinfo is initialized and writable. + unsafe { kernel::bindings::si_meminfo(info.as_mut_ptr()); } + // SAFETY: The object was initialized before si_meminfo filled it. + let info = unsafe { info.assume_init() }; + let ram = (info.totalram as u64).checked_mul(u64::from(info.mem_unit)).ok_or(EOVERFLOW)?; + let initial = blocks.checked_mul(0x80000).ok_or(EOVERFLOW)?; + crate::g17::buffer::BlockGeometry::m5(ram, initial as u64) +} + +/// The target layout of a render: G17P's for a command without an M5 plan. +pub(crate) fn layout(command: &Validated, clusters: u32, programs: RegisterPrograms) + -> Result { + let Validated::Render { pass, plan, .. } = command else { return Err(EINVAL); }; + let layout = target::Layout::new(pass, clusters, programs)?; + if plan.is_none() { return Ok(layout); } + validate(pass)?; + // One target group per cluster unless the pass asks for few primitives. + // Both target storage and the segment controls use the physical cluster count. + let geometry = parameter_geometry(layout.tvb_blocks)?; + // The auxiliary tables cover the complete parameter inventory, including + // future growth, rather than just the initially resident physical blocks. + let bytes = geometry.capacity.checked_mul(geometry.block_bytes).ok_or(EOVERFLOW)?; + let groups = if pass.few_primitives { 1 } else { clusters }; + layout.with_m5_slices(pass, clusters, groups, 64, bytes) +} + +pub(crate) fn controls( + pass: &RenderPass, + board: &'static crate::g17::fw::m5::initdata::Board, +) -> Result { + validate(pass)?; + let stage = |resource_flags| regs::M5RenderStageControls { + scs: crate::g17::render_resources::StageScratch::NONE, + resource_flags, resource_control: 0x200000001, ptc_max: 96, work_key: 0, + }; + Ok(regs::M5RenderControls { + board, + fragment_event_control: regs::fragment_event_control(true, 1), + ta: stage(0x4054b), fragment: stage(0x104054b), + ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff0200, fragment_tile_flags: 0x200003717f, + store_pipeline_2_bind: pass.eot_rsrc_spec << 32, + store_pipeline_2: pass.usc_base + u64::from(pass.eot_offset & !7), + partial_store_pipeline_2_bind: pass.partial_eot_rsrc_spec << 32, + partial_store_pipeline_2: pass.usc_base + u64::from(pass.partial_eot_offset & !7), + partial_tile_flags: 0x200000, + aux_fb_flags: 0x10c000 | u64::from(pass.few_primitives) + | (u64::from(pass.depth_bias_is_int) << 18), + aux_fb_control: 0x100000, + }) +} + +pub(crate) fn inputs(args: &RenderArgs<'_>, usage: [u64; 2]) -> Result { + let r = args.resources; + let mut fields = cmd::M5RenderCommandInputs::default(); + let token = crate::g17::fw::m5::render_tail::context_token(u32::from(r.context_id), + u64::from(r.context_generation)).ok_or(EINVAL)?; + fields.ta.context_token = token; + fields.fragment.context_token = token; + fields.ta.resolved_tail_value = usage[0]; + fields.fragment.resolved_tail_value = Some(usage[1]); + fields.scheduler_record = r.kick_record_va; + // The compiler's external-write class of the render's graphics programs. + fields.ta.shader_write_class = cmd::shader_write_class(args.pass.shader_writes_memory); + fields.ta.sequence = args.ordinal.checked_mul(2).and_then(|n| n.checked_add(1)).ok_or(EOVERFLOW)?; + fields.fragment.sequence = args.ordinal.checked_mul(2).ok_or(EOVERFLOW)?; + fields.fragment.pass = 1; + fields.ta.predecessor_id = r.ta_ids.predecessor; + fields.fragment.predecessor_id = r.fragment_ids.predecessor; + fields.fragment.fragment_word_n = 1; + fields.fragment.fragment_flag_c = 1; + fields.fragment.binding_pairs[22][1] = 0x04040404; + // Pass policy is filled by apply_pass alongside its register copies. + fields.fragment.empty_tile_flags = [0; 4]; + fields.ta.timestamp_va = r.timestamp_va; + fields.ta.diagnostic_ta_timestamp_upper = r.timestamp_va + 8; + fields.fragment.timestamp_va = [r.timestamp_va + 16, r.timestamp_va + 24]; + let stamp = (args.ordinal as u32).wrapping_add(1).wrapping_shl(8); + fields.ta.linked_3d = cmd::M5TaLinked3d { + queue_index: u32::from(args.fragment.qid), stamp, ..Default::default() + }; + let completion = |stage: &crate::g17::fw::render::RenderStage, key| cmd::M5StageCompletion { + stamp_addresses: [stage.stamp_va, stage.aux_stamp_va], stamp, + queue_index: u32::from(stage.qid), ordinal: args.ordinal as u32, + key, ..Default::default() + }; + fields.ta.completion = completion(&args.ta, u64::from(r.ta_ids.current)); + fields.fragment.completion = completion(&args.fragment, u64::from(r.fragment_ids.current)); + fields.ta.pb_slot = r.pb_slot; + fields.ta.pb_state = r.pb_va; + fields.fragment.pm_state = r.pb_va; + fields.ta.pm_scene = r.scene_va; + fields.fragment.pm_scene = r.scene_va; + fields.ta.scene_end_va = r.scene_end_va; + fields.fragment.scene_end_va = r.scene_end_va; + fields.ta.uma_state = r.free_list_control_va; + fields.fragment.uma_state = r.free_list_control_va; + fields.ta.tail_flag = 1; + fields.fragment.tail_flag = 1; + Ok(fields) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 90a11ca977684..65615e1087037 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -19,6 +19,7 @@ use crate::{ }; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; +use crate::g17::fw::m5::initdata::Board; pub(super) const RECORDS: usize = 32; pub(super) const TA_BYTES: usize = 0x14000; @@ -31,6 +32,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// Usage records of one M5 command, both stages. +const M5_USAGE_BYTES: usize = 0x40; +const M5_USAGE_PAIR_BYTES: usize = 2 * M5_USAGE_BYTES; /// The lead command of a stage (InitBM before tiling), after the stage's configuration record /// in its slot ([`crate::g17::fw::Variant::lead_commands`]). pub(super) const LEAD: usize = 0xc0; @@ -50,7 +54,7 @@ impl Aliases { vm: &mmu::Vm, source: core::ops::Range, ) -> Result { - let mut range = dynamic::LOWER; + let mut range = alloc.uat.kernel_lower_dynamic_range(); let size = (source.end - source.start) as u64; loop { // Descriptor aliases are read-only to GPU consumers; the canonical @@ -91,7 +95,8 @@ impl Aliases { /// Stable queue records and rotating descriptors/configuration records. pub(super) struct Graph { - aliases: [Aliases; 2], + /// One alias of all M5 command storage, or the G17P tiling and fragment ones. + aliases: [Option; 2], pub(super) descriptors: PooledObject, pub(super) queues: PooledObject, pub(super) config: PooledObject, @@ -99,8 +104,15 @@ pub(super) struct Graph { pub(super) owners: [Option; RECORDS], /// Host words of the pointer blocks in this firmware build. pub(super) pointers: fw::variant::PointerLayout, + m5: bool, } impl Graph { + pub(super) fn is_m5(&self) -> bool { self.m5 } + + /// Out of line: inlined, its temporaries stay reserved in `Memory::new`'s frame while the + /// rest of the pair (`buffer::Manager::new` and its allocations) is built below it, which + /// overflowed the submit ioctl's kernel stack. + #[inline(never)] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -109,21 +121,30 @@ impl Graph { pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; + let m5 = alloc.uat.is_m5(); + // M5 selects cached firmware attributes at initial mapping creation; + // write-combined CPU stores also support the uncached capacity fallback. + let cpu = if m5 { CpuMap::WriteCombined } else { CpuMap::WriteBack }; + let bytes = if m5 { RECORDS * fw::m5::render_assembly::COMMAND_STORAGE_BYTES } + else { TA_BYTES + FRAGMENT_BYTES }; + // The GPU reads command storage only through the read-only aliases below. On M5 the + // kernel mapping, in the upper half every execution context includes, is firmware-only. let descriptors = pool.allocate( alloc, Family::Descriptor, - TA_BYTES + FRAGMENT_BYTES, - TA_BYTES + FRAGMENT_BYTES, - prot, - CpuMap::WriteBack, + bytes, + bytes, + if m5 { mmu::PROT_FW_SHARED_RW } else { prot }, + cpu, )?; - let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; - let fragment = Aliases::new( + let ta = Aliases::new(&descriptors, alloc, context.vm(), + 0..if m5 { bytes } else { TA_BYTES })?; + let fragment = if m5 { None } else { Some(Aliases::new( &descriptors, alloc, context.vm(), TA_BYTES..TA_BYTES + FRAGMENT_BYTES, - )?; + )?) }; let mut queues = pool.allocate( alloc, Family::QueueState, @@ -138,27 +159,33 @@ impl Graph { 0x8000, 0x8000, prot, - CpuMap::WriteBack, + cpu, )?; let support = alloc.kernel( - storage::SUPPORT_SIZE, + if m5 { mmu::UAT_PGSZ } else { storage::SUPPORT_SIZE }, mmu::UAT_PGSZ as u64, prot, CpuMap::WriteCombined, )?; let address = queues.gpu_va(); for stage in 0..2 { - queues.write( - qids[stage] as usize * size_of::(), - queue::QueueRecord::new(&queue::QueueRecordArgs { - pointers_va: address + POINTERS[stage] as u64, - items_va: address + ITEMS[stage] as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }), - )?; + let args = queue::QueueRecordArgs { + pointers_va: address + POINTERS[stage] as u64, + items_va: address + ITEMS[stage] as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + policy: context.policy(), + }; + if m5 { + queues.write(qids[stage] as usize * size_of::(), + fw::m5::queue::QueueRecord::new(&args)?)?; + } else { + queues.write( + qids[stage] as usize * size_of::(), + queue::QueueRecord::new(&args), + )?; + } queues.initialize::(POINTERS[stage], |view| { *view = queue::PointerBlock::new(&pointers)?; Ok(()) @@ -166,33 +193,67 @@ impl Graph { } queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; Ok(Self { - aliases: [ta, fragment], + aliases: [Some(ta), fragment], descriptors, queues, config, support, owners: [None; RECORDS], pointers, + m5, }) } - pub(super) fn offset(stage: usize, ordinal: u64) -> usize { + pub(super) fn queue_offset(&self, qid: u8) -> usize { + usize::from(qid) * if self.m5 { size_of::() } + else { size_of::() } + } + pub(super) fn queue_va(&self, qid: u8) -> u64 { + self.queues.gpu_va() + self.queue_offset(qid) as u64 + } + pub(super) fn offset(&self, stage: usize, ordinal: u64) -> usize { let slot = ordinal as usize % RECORDS; - if stage == 0 { + if self.m5 { + slot * fw::m5::render_assembly::COMMAND_STORAGE_BYTES + + stage * fw::m5::render_assembly::FRAGMENT_OFFSET + } else if stage == 0 { slot * abi::TA_DESCRIPTOR_SIZE } else { TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE } } pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { - self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 + self.descriptors.gpu_va() + self.offset(stage, ordinal) as u64 } - pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { + pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> Result { + if self.m5 { + return Ok(self.aliases[0].as_ref().ok_or(EIO)?.client.iova() + + self.offset(stage, ordinal) as u64); + } let stride = if stage == 0 { abi::TA_DESCRIPTOR_SIZE } else { abi::FRAGMENT_SLOT_SIZE }; - self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 + Ok(self.aliases[stage].as_ref().ok_or(EIO)?.client.iova() + + (ordinal as usize % RECORDS * stride) as u64) + } + /// The command storage of `ordinal` on M5: both stages' commands. + pub(super) fn m5_storage(&mut self, ordinal: u64) -> Result<&mut [u8]> { + if !self.m5 || !self.owns(ordinal) { return Err(EINVAL); } + let start = self.offset(0, ordinal); + self.descriptors.bytes_mut() + .get_mut(start..start + fw::m5::render_assembly::COMMAND_STORAGE_BYTES) + .ok_or(EINVAL) + } + /// Reset the usage entries owned by this descriptor generation. They stay + /// resident with the graph until both stages release the descriptor slot. + pub(super) fn m5_usage(&mut self, ordinal: u64) -> Result<[u64; 2]> { + if !self.m5 || !self.owns(ordinal) { return Err(EINVAL); } + let start = ordinal as usize % RECORDS * M5_USAGE_PAIR_BYTES; + self.support.bytes_mut().get_mut(start..start + M5_USAGE_PAIR_BYTES) + .ok_or(EINVAL)?.fill(0); + let address = self.support.gpu_va() + start as u64; + Ok([address, address + M5_USAGE_BYTES as u64]) } pub(super) fn publish_recovery_targets(&self) -> Result { let Some(target) = self.pointers.recovery_target else { @@ -247,8 +308,24 @@ struct Compact { address: u64, _backing: KernelObject, } +/// The mesh storage chunk: chunk bases are 64 KiB aligned and the storage +/// register counts 64 KiB chunks. +const MESH_STORAGE_CHUNK: usize = 0x10000; + impl Compact { + /// M5 shared-RW object in the compact window with an explicit VA alignment. + fn new_aligned(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize, align: u64) -> Result { + let size = size.checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ - 1); + let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + let mapping = backing.map_range(vm, 0..size, buffer::BlockGeometry::M5_COMPACT, + align, mmu::PROT_GPU_SHARED_RW, mmu::UAT_PGSZ)?; + if mapping.iova() % align != 0 { return Err(EINVAL); } + Ok(Self { address: mapping.iova(), mapping: Some(mapping), _backing: backing }) + } fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { + if alloc.uat.is_m5() { + return Self::new_aligned(alloc, vm, size, cfg::TVB_PAGE); + } let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; let mapping = buffer::compact_alias( &backing, @@ -266,14 +343,65 @@ impl Compact { } } -pub(super) struct Generation { +/// Per-command state of a pair: the G17P one below or the M5 one +/// ([`super::generation::Generation`]). +pub(super) enum Generation { + Neo(KBox), + M5(KBox), +} +impl Generation { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm, board: Option<&'static Board>) + -> Result { + if alloc.uat.is_m5() { + super::generation::Generation::allocate_state(alloc, vm, board) + } else { + NeoGeneration::allocate_state(alloc, vm) + } + } + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject, + board: Option<&'static Board>) -> Result { + Ok(if alloc.uat.is_m5() { + Self::M5(super::generation::Generation::new(alloc, vm, state, board)?) + } else { + Self::Neo(NeoGeneration::new(alloc, vm, state)?) + }) + } + pub(super) fn ordinal(&self) -> Option { + match self { + Self::Neo(generation) => generation.ordinal, + Self::M5(generation) => generation.ordinal, + } + } + /// Gives up the claim of an unpublished or retired command. + pub(super) fn clear(&mut self) { + match self { + Self::Neo(generation) => generation.ordinal = None, + Self::M5(generation) => generation.ordinal = None, + } + } + /// `viewport` applies to the G17P deflake buffer ([`abi::Deflake::viewport`]). + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass, viewport: bool) -> Result { + match self { + Self::Neo(generation) => generation.prepare(ordinal, pass, viewport), + Self::M5(generation) => generation.prepare(ordinal, pass), + } + } + pub(super) fn addresses(&self) -> [u64; 5] { + match self { + Self::Neo(generation) => generation.addresses(), + Self::M5(generation) => generation.addresses(), + } + } +} + +pub(super) struct NeoGeneration { deflake: mmu::KernelMapping, fragment_client: mmu::KernelMapping, state: KernelObject, fragment: KernelObject, pub(super) ordinal: Option, } -impl Generation { +impl NeoGeneration { fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { KernelObject::new( alloc.dev, @@ -368,21 +496,28 @@ impl Generation { } pub(super) struct Memory { - _fragment_view: mmu::KernelMapping, + _fragment_view: Option, pub(super) graph: Graph, scratch: Compact, discard: Compact, + /// M5 mesh rendering buffer of the queue's task-stage kicks. Shaders read + /// its per-workgroup 64 KiB chunk base, so it is mapped in the context VM. + mesh_storage: Option, + mesh_storage_chunks: u32, pub(super) target: KBox, - generations: [Option>; DEPTH], + generations: [Option; DEPTH], generation_count: usize, pub(super) timestamps: KernelObject, pub(super) manager: buffer::Manager, pub(super) lifecycle: [fw::ObjectIds; 2], // The extra client views retain their allocation positions even though // kick entries consume the register arrays embedded in the descriptors. - registers: KernelObject, + registers: Option, + m5_workspace: Option>, pub(super) context: Arc, pub(super) installed_pool: Option, + /// The M5 board ([`crate::g17::Shared::m5_board`]); `None` off M5. + pub(super) board: Option<&'static Board>, } impl Memory { #[inline(never)] @@ -402,16 +537,31 @@ impl Memory { ordinal: u64, word: u32, variant: &'static fw::Variant, + board: Option<&'static Board>, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); + let m5 = alloc.uat.is_m5(); + if m5 != layout.is_m5() { return Err(EINVAL); } let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; - let state = Generation::allocate_state(alloc, vm)?; - let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; - let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; - let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + let state = Generation::allocate_state(alloc, vm, board)?; + let (tvb, mappings, scratch, discard) = if m5 { + let geometry = super::m5_render_submission::parameter_geometry(layout.tvb_blocks)?; + let bytes = fw::m5::pool::parameter_scratch_stride(geometry.capacity, 0) + .map_err(|_| EINVAL)?.checked_mul(fw::m5::pool::SCENE_TABLE_ENTRIES) + .ok_or(EOVERFLOW)?; + let (tvb, mappings) = buffer::Tvb::with_geometry(alloc.dev, vm, layout.tvb_blocks, geometry)?; + let scratch = Compact::new(alloc, vm, bytes)?; + let discard = Compact::new(alloc, vm, 0x8000)?; + (tvb, mappings, scratch, discard) + } else { + let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; + let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; + let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + (tvb, mappings, scratch, discard) + }; let count = DEPTH; let initial = if ordinal == 0 { 0 @@ -419,11 +569,11 @@ impl Memory { (ordinal ^ 1) as usize % count }; let mut generations = core::array::from_fn(|_| None); - generations[initial] = Some(Generation::new(alloc, vm, state)?); + generations[initial] = Some(Generation::new(alloc, vm, state, board)?); for offset in 1..count { let index = (initial + offset) % count; - let state = Generation::allocate_state(alloc, vm)?; - generations[index] = Some(Generation::new(alloc, vm, state)?); + let state = Generation::allocate_state(alloc, vm, board)?; + generations[index] = Some(Generation::new(alloc, vm, state, board)?); } let timestamps = alloc.kernel( mmu::UAT_PGSZ, @@ -447,36 +597,47 @@ impl Memory { let predecessor = crate::g17::ids::OBJECTS.object()?; let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; let support_va = graph.support.gpu_va(); - graph.support.initialize::(0, |view| { - view.init( - support_va, - alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, - low::RENDER_RUN_LIST, - low::RENDER_PAGE_LIST, - word, - ) - })?; - let registers = KernelObject::new( + if !m5 { + graph.support.initialize::(0, |view| { + view.init( + support_va, + alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, + low::RENDER_RUN_LIST, + low::RENDER_PAGE_LIST, + word, + ) + })?; + } + let registers = if m5 { None } else { Some(KernelObject::new( alloc.dev, vm, crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), mmu::UAT_PGSZ, mmu::PROT_GPU_SHARED_RO, CpuMap::WriteCombined, - )?; - let fragment_view = graph.descriptors.map_range( + )?) }; + let m5_workspace = if m5 { + let mut workspace = KBox::::new_uninit(GFP_KERNEL)?; + // SAFETY: Workspace is integer-only and has a valid zero representation. + unsafe { core::ptr::write_bytes(workspace.as_mut_ptr(), 0, 1); } + // SAFETY: The entire value was initialized above. + Some(unsafe { workspace.assume_init() }) + } else { None }; + let fragment_view = if m5 { None } else { Some(graph.descriptors.map_range( vm, TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RO, 0, - )?; + )?) }; Ok(( storage.write_init(kernel::try_init!(Self { graph, scratch, discard: discard, + mesh_storage: None, + mesh_storage_chunks: 0, target, generations, generation_count: count, @@ -485,35 +646,83 @@ impl Memory { lifecycle: [tiling, fragment], _fragment_view: fragment_view, registers: registers, + m5_workspace, context, installed_pool: None, + board, }))?, mappings, )) } + /// The mesh storage register value of a task-stage kick. The buffer is + /// allocated on the queue's first such kick and kept for the queue's lifetime, + /// so other contexts pay nothing. A failed allocation leaves no partial state. + /// A zero chunk parameter disables the binding. + pub(super) fn mesh_storage_register(&mut self, alloc: &Allocator<'_>) -> Result { + if self.mesh_storage.is_none() { + let chunks = crate::module_parameters::m5_mesh_storage_chunks.value(); + if chunks == 0 { return Ok(0); } + if chunks > 0xffff { return Err(EINVAL); } + let bytes = (chunks as usize).checked_mul(MESH_STORAGE_CHUNK).ok_or(EOVERFLOW)?; + let storage = Compact::new_aligned(alloc, self.context.vm(), bytes, + MESH_STORAGE_CHUNK as u64)?; + self.mesh_storage = Some(storage); + self.mesh_storage_chunks = chunks; + } + let storage = self.mesh_storage.as_ref().ok_or(ENOMEM)?; + Ok((storage.address & 0xffff_ffff_ffff) | (u64::from(self.mesh_storage_chunks) << 48)) + } pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { self.generations[ordinal as usize % self.generation_count] - .as_deref_mut() + .as_mut() .ok_or(EIO) } pub(super) fn owns(&self, ordinal: u64) -> bool { self.graph.owns(ordinal) && self.generations[ordinal as usize % self.generation_count] .as_ref() - .is_some_and(|generation| generation.ordinal == Some(ordinal)) + .is_some_and(|generation| generation.ordinal() == Some(ordinal)) + } + /// M5 commands record four stage timestamps, G17P ones two. + pub(super) fn timestamp_offset(&self, ordinal: u64) -> usize { + ordinal as usize % RECORDS * if self.graph.m5 { 32 } else { 16 } + } + pub(super) fn clear_timestamps(&mut self, ordinal: u64) -> Result { + let at = self.timestamp_offset(ordinal); + if self.graph.m5 { self.timestamps.write(at, [0u64; 4]) } + else { self.timestamps.write(at, [0u64; 2]) } } pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { if !self.owns(ordinal) { return Err(EIO); } - let offset = ordinal as usize % RECORDS * 16; + let offset = self.timestamp_offset(ordinal); + Ok([ + self.timestamps.dword(offset)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + if self.graph.m5 { 24 } else { 8 })? + .load(Ordering::Relaxed), + ]) + } + /// Export the four M5 stage words only after the observer has acquired + /// both completion stamps. Keep the two-word G17P representation intact. + pub(super) fn stage_timestamps(&self, ordinal: u64) -> Result<[u64; 4]> { + if !self.graph.m5 { + let [start, end] = self.pass(ordinal)?; + return Ok([start, start, start, end]); + } + if !self.owns(ordinal) { + return Err(EIO); + } + let offset = self.timestamp_offset(ordinal); Ok([ self.timestamps.dword(offset)?.load(Ordering::Relaxed), self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 16)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 24)?.load(Ordering::Relaxed), ]) } pub(super) fn owners(&self) -> [Option; DEPTH] { - core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) + core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal())) } pub(super) fn retire(&mut self, ordinal: u64) -> Result { if !self.owns(ordinal) { @@ -522,7 +731,7 @@ impl Memory { self.target.retire(ordinal)?; self.manager.retire_scene(ordinal)?; self.graph.release(ordinal)?; - self.generation(ordinal)?.ordinal = None; + self.generation(ordinal)?.clear(); Ok(()) } } @@ -547,7 +756,7 @@ impl Memory { let record = ordinal as usize % RECORDS; let config = size_of::(); let ranges = [ - (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.descriptors, self.graph.offset(0, ordinal), size_of::()), (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), ]; @@ -566,6 +775,42 @@ impl Memory { } } + /// Encodes both M5 commands of `args.ordinal` into its command storage and + /// returns the kicks' register arrays and header flags. + pub(super) fn write_m5_commands( + &mut self, + args: &abi::RenderArgs<'_>, + layout: &target::Layout, + plan: Option, + mesh_class: u8, + ) -> Result<([[Option; 4]; 2], fw::kick::StageKickFlags)> { + use super::m5_render_submission as render_submission; + use fw::m5::{render_assembly, render_registers}; + if !self.graph.m5 { + return Err(EINVAL); + } + let mut controls = render_submission::controls(args.pass, self.board.ok_or(EINVAL)?)?; + if let Some(stages) = plan.and_then(|p| p.stages) { + controls.ta.scs = stages[0].scs; + controls.fragment.scs = stages[1].scs; + controls.ta.resource_flags = stages[0].resource_flags; + controls.ta.ptc_max = stages[0].ptc_max; + controls.fragment.resource_flags = stages[1].resource_flags; + controls.fragment.ptc_max = stages[1].ptc_max; + } + let usage = self.graph.m5_usage(args.ordinal)?; + let mut inputs = render_submission::inputs(args, usage)?; + inputs.ta.shifted_usage_hint = fw::m5::render::mesh_class_usage_hint(mesh_class); + render_registers::apply_pass(args, &controls, &mut inputs)?; + let extra = self.target.m5_extra(layout)?; + let low = self.graph.descriptor_client(0, args.ordinal)?; + let workspace = self.m5_workspace.as_deref_mut().ok_or(EIO)?; + let encoded = render_assembly::encode_into(self.graph.m5_storage(args.ordinal)?, low, + args, &controls, extra, render_registers::FragmentDispatch { words: [0; 2] }, + &inputs, workspace)?; + Ok((encoded.rce, encoded.kick_flags)) + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -576,7 +821,7 @@ impl Memory { .graph .descriptors .pointer( - Graph::offset(0, args.ordinal), + self.graph.offset(0, args.ordinal), size_of::(), )? .cast::(); @@ -584,7 +829,7 @@ impl Memory { .graph .descriptors .pointer( - Graph::offset(1, args.ordinal), + self.graph.offset(1, args.ordinal), size_of::(), )? .cast::(); @@ -626,15 +871,19 @@ impl Memory { if !self.owns(ordinal) { return Err(EIO); } + if self.graph.m5 { + return fw::m5::render_assembly::link_completions( + self.graph.m5_storage(ordinal)?, kicks, payload, fragment_busy); + } let tiling = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); let fragment = self .graph .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? + .pointer(self.graph.offset(1, ordinal), size_of::())? .cast::(); // SAFETY: The complete views were bounds/alignment checked during // preparation. The unchanged full ordinal still exclusively owns both @@ -655,7 +904,7 @@ impl Memory { let ta = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); // SAFETY: The view was checked during preparation and the ordinal still owns it; only // integer fields are read. @@ -677,21 +926,49 @@ impl Memory { impl Memory { /// Preserve the initial mapping order while moving VM-lifetime mappings /// out of the graph. Reserve all host storage before moving any mapping. + /// Moves this drained graph's pool mappings out of its VM slot, so a new + /// M5 graph of the same VM can install on the pair (see bind_owner). + pub(super) fn take_pool_mappings(&mut self) -> Option<(u8, mmu::RenderPoolMappings)> { + let slot = self.installed_pool?; + let id = u64::from(self.lifecycle[0].predecessor); + let mappings = self.context.vm().take_render_pool_mappings(slot, id)?; + self.installed_pool = None; + Some((slot, mappings)) + } + /// Returns mappings moved out by `take_pool_mappings` after a failed install. + pub(super) fn put_pool_mappings( + &mut self, + slot: u8, + mappings: mmu::RenderPoolMappings, + ) -> Result { + self.context.vm().put_render_pool_mappings(slot, mappings)?; + self.installed_pool = Some(slot); + Ok(()) + } + pub(super) fn take_persistent( &mut self, tvb: &mut KVec, ) -> Result> { if self.scratch.mapping.is_none() || self.discard.mapping.is_none() - || self.registers.gpu_va() == 0 + || (!self.graph.m5 && self.registers.as_ref().is_none_or(|r| r.gpu_va() == 0)) { return Err(EIO); } + if self.mesh_storage.as_ref().is_some_and(|m| m.mapping.is_none()) { + return Err(EIO); + } let mut aliases = KVec::new(); - aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.reserve(tvb.len().checked_add(4).ok_or(EOVERFLOW)?, GFP_KERNEL)?; aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; + if let Some(storage) = &mut self.mesh_storage { + aliases.push(storage.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + } + if let Some(registers) = &mut self.registers { + aliases.push(registers.take_mapping()?, GFP_KERNEL)?; + } for mapping in core::mem::take(tvb) { aliases.push(mapping, GFP_KERNEL)?; } @@ -704,6 +981,9 @@ impl Memory { if slot >= 4 || !self.owns(ordinal) { return Err(EINVAL); } + if self.graph.m5 { + return fw::m5::render_assembly::bind_work_slot(self.graph.m5_storage(ordinal)?, slot); + } let queues = [ self.graph .queues @@ -715,12 +995,12 @@ impl Memory { let ta = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); let fragment = self .graph .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? + .pointer(self.graph.offset(1, ordinal), size_of::())? .cast::(); for word in queues { word.store(slot, Ordering::Relaxed); @@ -738,7 +1018,40 @@ impl Memory { /// Retargets both retired queue records to `context`. A context of another /// scheduling profile also replaces the priority and policy words, after /// the owner fields; the same profile leaves them untouched. - pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { + /// `rewrite`: store the M5 records even when their host-owned words + /// already name `context`; without it that no-op read-modify-write, which + /// would also store back firmware-owned words of a possibly running queue, + /// is skipped. G17P always retargets. + pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context, rewrite: bool) -> Result { + if self.graph.m5 { + let records = qids.map(|qid| self.graph.queues.pointer(self.graph.queue_offset(qid), + size_of::())); + let [first, second] = records; + let pointers = [first?, second?].map(|p| p.cast::()); + // SAFETY: Both complete records are in bounds. The caller owns + // retired queues exclusively; build both updates before storing. + unsafe { + let mut values = pointers.map(|p| p.read_unaligned()); + if !rewrite { + let mut named = true; + for value in &values { + named &= value.names_owner(context.work_head_va(), context.scheduler_va(), + context.owner_pid(), context.policy())?; + } + if named { + return Ok(()); + } + } + for value in &mut values { + value.set_compute_owner(context.work_head_va(), context.scheduler_va(), + context.owner_pid(), context.policy())?; + } + for (pointer, value) in pointers.into_iter().zip(values) { + pointer.write_unaligned(value); + } + } + return Ok(()); + } let offsets = qids.map(|qid| usize::from(qid) * size_of::()); let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; // Check every field before the first update, retaining the required diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c..83a4bcb2f3022 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -2,10 +2,9 @@ //! Installed kick storage retained after a logical scheduler release is acknowledged. -use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; +use super::{install_memory, memory, Pair, RetiredBinding, Tracker}; use crate::g17::{ buffer, - command::Validated, context::Context, freelist::RenderPool, job::Packet, @@ -20,7 +19,7 @@ use kernel::{prelude::*, sync::Arc}; /// context awaiting an executed replacement keeps its separate retirement owner. pub(crate) struct ParkedPair { kicks: Option<[kick::Queue; 2]>, - _scheduler: Arc, + _scheduler: Arc, ids: [kick::Id; 2], slot: u8, ordinal: u64, @@ -28,6 +27,7 @@ pub(crate) struct ParkedPair { deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, installed: [bool; 2], recovery_generation: u64, registered: Option, @@ -38,10 +38,13 @@ pub(crate) struct ParkedPair { /// owner) keeps it, so its next owner change still waits while that /// scheduler is released. executed: u64, + /// The VM of the context that last ran on these rings. + vm: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. pub(crate) struct Prepared { + pool_pin: Option, memory: Option>, pool: Arc, aliases: Option>, @@ -83,6 +86,7 @@ impl Pair { /// under the same device mutex. Moving these owners performs no firmware operation. pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { let context = self.memory.context.clone(); + let vm = context.vm().id(); let scheduler = context.scheduler_owner(); let buffer_id = self.buffer_id(); let parked = ParkedPair { @@ -95,12 +99,14 @@ impl Pair { deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, + board: self.board, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, executed: self.executed, + vm, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -109,12 +115,27 @@ impl Pair { buffer_id, publication: None, ready: true, + wait_for_pool_release: false, }; (parked, retired) } } impl ParkedPair { + /// Clears both kick rings when the pair goes to a context of another VM than the one + /// that last ran on it, so the new owner cannot read the entries the previous owner's + /// commands left. The pair is parked, so the firmware consumed every entry. + pub(crate) fn clear_for_vm(&mut self, vm: u64) -> Result { + if self.vm == vm { + return Ok(()); + } + for queue in self.kicks.as_mut().ok_or(EBUSY)?.iter_mut() { + queue.clear_render_ring()?; + } + self.vm = vm; + Ok(()) + } + /// The release of `scheduler` has been consumed: a later owner change no /// longer has to wait for it. pub(crate) fn release_consumed(&mut self, scheduler: u64) { @@ -143,19 +164,24 @@ impl ParkedPair { pm_generation: u64, first: &Arc, ) -> Result { + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { return Err(EINVAL); } let kicks = self.kicks.as_ref().ok_or(EBUSY)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = super::m5_render_submission::layout( + &first.command, + self.clusters, + self.variant.programs, + )?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -172,8 +198,10 @@ impl ParkedPair { self.ordinal, word, self.variant, + self.board, )?; Ok(Prepared { + pool_pin, memory: Some(memory), pool: free_list, aliases: Some(persistent), @@ -221,6 +249,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + pool_pin: next.pool_pin.take(), graph_started: false, kick_aliases: None, memory, @@ -237,6 +266,7 @@ impl ParkedPair { deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, + board: self.board, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -248,11 +278,16 @@ impl ParkedPair { previous: self.previous.take(), displaced: None, replay: Tracker::new(), + executed_generation: None, replay_since: None, context_update: true, executed: self.executed, switch: None, published_completions: None, + identity_update: true, + configuration: None, + configured_context: None, + configured_qos: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index d65c40c7cfade..838d3b17bab86 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -5,6 +5,7 @@ use super::{memory, retirement::Window}; use crate::g17::fw::{ + m5, kick::KickTimestamp, queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, }; @@ -27,6 +28,12 @@ pub(in crate::g17::queue) fn required(previous: Option, config: Que || previous.map(identity) != Some(identity(config)) } +/// The kick announcement record: the G17P one, or the M5 notification. +enum Announcement { + Neo(KickAnnounce), + M5(m5::queue::Notification), +} + pub(super) struct Lane<'a> { items: [&'a AtomicU64; 4], producer: &'a AtomicU32, @@ -34,8 +41,8 @@ pub(super) struct Lane<'a> { config: QueueConfig, config_destination: *mut u64, config_present: bool, - announce: KickAnnounce, - announce_destination: *mut KickAnnounce, + announce: Announcement, + announce_destination: *mut u8, prefix: usize, prefix_end: u32, pub(super) window: Window, @@ -55,7 +62,9 @@ impl<'a> Lane<'a> { timestamp: KickTimestamp, lead: bool, ) -> Result { - let count = 2 + u32::from(present) + u32::from(lead); + let m5 = graph.is_m5(); + // M5 items carry no descriptor: [configuration], notification. + let count = u32::from(present) + u32::from(lead) + if m5 { 1 } else { 2 }; let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -69,8 +78,8 @@ impl<'a> Lane<'a> { } let announce_destination = graph .queues - .pointer(announce_offset, size_of::())? - .cast::(); + .pointer(announce_offset, if m5 { size_of::() } + else { size_of::() })?; let index = |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; let items = [ @@ -91,7 +100,9 @@ impl<'a> Lane<'a> { if lead { push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); } - push(descriptor); + if !m5 { + push(descriptor); + } if present { push(graph.config.gpu_va() + config_offset as u64); } @@ -105,16 +116,20 @@ impl<'a> Lane<'a> { config, config_destination, config_present: present, - announce: KickAnnounce::new( - config.qid.try_into()?, - if stage == 0 { - DataMaster::Tiling - } else { - DataMaster::Fragment - }, - timestamp, - config.priority as u8, - )?, + announce: if m5 { + Announcement::M5(m5::queue::notification(config.qid.try_into()?, timestamp)?) + } else { + Announcement::Neo(KickAnnounce::new( + config.qid.try_into()?, + if stage == 0 { + DataMaster::Tiling + } else { + DataMaster::Fragment + }, + timestamp, + config.priority as u8, + )?) + }, announce_destination, prefix: count as usize - 1, prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, @@ -146,9 +161,21 @@ impl<'a> Lane<'a> { // SAFETY: The full record is inside the retained queue graph and is // exclusively owned by this ordinal. Publication follows its copy. unsafe { - self.announce_destination.write_unaligned(self.announce); + match &self.announce { + Announcement::Neo(value) => self.announce_destination + .cast::().write_unaligned(*value), + Announcement::M5(value) => self.announce_destination + .cast::().write_unaligned(*value), + } } } + /// Fill the retained records without advancing the firmware-visible cursor. + /// M5's host-port transaction must occur between this and publish_kick. + pub(super) fn prepare(&self) { + self.items(0..self.prefix + 1); + self.config(); + self.announce(); + } pub(super) fn publish_prefix(&self) { self.items(0..self.prefix); self.config(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 4377bbfe3e311..19a2caabe68e1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -39,7 +39,7 @@ pub(crate) struct Window { impl Window { pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { - if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { + if base >= capacity || !(1..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { return Err(EINVAL); } Ok(Self { @@ -230,8 +230,7 @@ pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> R if !observation.complete { return Ok(Poll::Pending); } - let [start, end] = memory.pass(ticket.ordinal)?; - Ok(Poll::Complete([start, start, start, end])) + Ok(Poll::Complete(memory.stage_timestamps(ticket.ordinal)?)) } /// Outstanding publications span less than half the 24-bit stamp sequence. @@ -255,6 +254,11 @@ pub(crate) trait Memory { fn snapshot(&mut self, ordinal: u64) -> Result; fn stamps(&mut self) -> Result<[u32; 2]>; fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; + /// Four stage timestamps: M5 records them, G17P derives them from its pass. + fn stage_timestamps(&mut self, ordinal: u64) -> Result<[u64; 4]> { + let [start, end] = self.pass(ordinal)?; + Ok([start, start, start, end]) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -314,8 +318,7 @@ fn complete_poll( // Refresh this generation's pass after observing the post-maintenance // stamps. The pre-stamp snapshot alone cannot authorize export. fence(Ordering::Acquire); - let [start, end] = memory.pass(ticket.ordinal)?; - return Ok(Poll::Complete([start, start, start, end])); + return Ok(Poll::Complete(memory.stage_timestamps(ticket.ordinal)?)); } ticket.previous = snapshot.cursors; Ok(Poll::Pending) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index d7f8b46eda039..efeb0e15f00aa 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -11,7 +11,7 @@ mod attribution; mod slot; mod safety; pub(crate) use safety::{note_retired, since_retired}; -pub(crate) use attribution::Sources; +pub(crate) use attribution::{Sources, PAGE_FAULT}; use attribution::Attribution; pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { @@ -50,8 +50,8 @@ pub(super) struct Entries { } impl Entries { - /// Returns false at the first invalid record: the table is a packed list and - /// records after it are left over from an earlier restart. + /// Counts a valid record. Returns false for an invalid one, which ends the walk on G17P + /// and is skipped on M5 (see `read_entries`). pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { return false; @@ -134,6 +134,8 @@ pub(super) trait Memory { fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); + /// The M5 firmware's recovery table, whose stamp form is not the G17P one. + fn is_m5(&self) -> bool; } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -304,7 +306,14 @@ pub(super) fn service( let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); let (cancelled, blame, leftover) = classify(host, entries, request.blamed, attribution); - if listed != 0 { + if listed != 0 && host.is_m5() { + // The M5 table only classifies (read_entries): each command is retired from its own + // queue's records, replayed when it never ran. + pr_warn!( + "G17: GPU recovery: {} commands were in flight; each is retired from its own queue's records\n", + listed + ); + } else if listed != 0 { pr_warn!( "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", listed, @@ -331,17 +340,26 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error, leftover }) } -/// Walks the packed recovery table up to its first invalid record and marks -/// every listed command interrupted. A failed read keeps what was read so far. +/// Walks the recovery table. G17P's is a packed list: records after its first invalid one are +/// left over from an earlier restart, so the walk ends there. M5's need not be packed, so all +/// of its records are walked and invalid ones skipped; they only classify the recovery, while +/// on G17P every listed command is marked interrupted. A failed read keeps what was read so far. fn read_entries(host: &mut H) -> Entries { let mut entries = Entries::default(); + let m5 = host.is_m5(); for index in 0..RECOVERY_RECORDS { let Ok((flags, stamp)) = host.entry(index) else { break; }; if !entries.include(flags) { + if m5 { + continue; + } break; } + if m5 { + continue; + } let qid = ((flags >> 4) & 0x7f) as u8; match host.note_interrupted(qid, stamp) { Interrupted::Exact => { @@ -459,6 +477,7 @@ mod tests { } impl Memory for Halted { + fn is_m5(&self) -> bool { false } fn recovery_state(&self) -> Result { Ok(self.state.get()) } fn host_recovery(&self) -> Result { Ok(0) } fn fault_sources(&self) -> Result { @@ -513,6 +532,96 @@ mod tests { fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } } + /// A halted recovery table holding `records` (index, flags); every other record is invalid. + /// Records the QIDs whose commands the walk marks interrupted. + struct Table { + recovery: State, + m5: bool, + records: [(usize, u32); 5], + marked: u128, + } + + impl Table { + fn new(m5: bool) -> Self { + const VALID: u32 = 1; + Self { + recovery: State::new(), + m5, + records: [ + (0, VALID | (12 << 4)), + (1, VALID | u32::from(GUILTY) | (17 << 4)), + // Index 2 is invalid: G17P's list ends here, and what follows is left over. + (3, VALID | u32::from(GUILTY | CANCELLED) | (40 << 4)), + (RECOVERY_RECORDS - 1, VALID | u32::from(GUILTY) | (41 << 4)), + (RECOVERY_RECORDS, VALID | u32::from(GUILTY) | (42 << 4)), + ], + marked: 0, + } + } + } + + impl Memory for Table { + fn is_m5(&self) -> bool { self.m5 } + fn recovery_state(&self) -> Result { Ok(3) } + fn host_recovery(&self) -> Result { Ok(0) } + fn fault_sources(&self) -> Result { Err(EIO) } + fn transition(&self, _expected: u32, _next: u32) -> Result { Err(EIO) } + fn clear_timestamps(&self) -> Result { Err(EIO) } + fn entry(&self, index: usize) -> Result<(u32, u32)> { + assert!(index < RECOVERY_RECORDS); + let flags = self.records.iter().find(|(at, _)| *at == index).map_or(0, |(_, flags)| *flags); + Ok((flags, index as u32)) + } + fn wait_tick(&self) {} + } + + impl Queues for Table { + fn render_scheduler_active(&self) -> bool { true } + fn render_pass_started(&self, _qid: u8, _stamp: u64) -> Option { None } + fn publish_recovery_targets(&self) -> Option { None } + fn oldest_spared(&self, _qid: u8) -> Option { None } + fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} + fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn note_interrupted(&mut self, qid: u8, _stamp: u32) -> Interrupted { + self.marked |= 1u128 << qid; + Interrupted::Exact + } + fn settle_blamed_render(&mut self, _qid: u8, _error: Error) {} + fn settle_cancelled_renders(&mut self, _qids: u128) {} + fn rescan_compute(&mut self) -> Result { Ok(()) } + fn release_retired_compute(&mut self) {} + } + + impl Host for Table { + fn recovery(&self) -> &State { &self.recovery } + fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } + } + + #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + let mut host = Table::new(false); + let entries = read_entries(&mut host); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + assert_eq!(entries.exact, 2); + assert_eq!(entries.exact_qids, (1u128 << 12) | (1u128 << 17)); + // Leftover records blame and cancel nothing and mark no bystander's command. + assert_eq!(host.marked, (1u128 << 12) | (1u128 << 17)); + } + + #[test] + fn m5_recovery_table_is_walked_past_invalid_records() { + let mut host = Table::new(true); + let entries = read_entries(&mut host); + assert_eq!(entries.listed, 4); + assert_eq!(entries.guilty, (1u128 << 17) | (1u128 << 40) | (1u128 << 41)); + assert_eq!(entries.cancelled, 1u128 << 40); + // The M5 table only classifies. + assert_eq!((entries.exact, entries.fallback, entries.exact_qids), (0, 0, 0)); + assert_eq!(host.marked, 0); + } + #[test] fn pass_evidence_is_frozen_before_the_first_acknowledgement() { let mut host = Halted::new(true); diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index fdf0cd62cbd07..9c7f42c839881 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -4,7 +4,8 @@ pub(super) const SLOTS: usize = 16; const STAMP_MASK: u64 = (1 << 40) - 1; -const PAGE_FAULT: u32 = 3; +/// Firmware recovery reason for a GPU MMU fault. +pub(crate) const PAGE_FAULT: u32 = 3; #[derive(Clone, Copy, Default)] pub(super) struct Source { diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index 5c2ad346c9890..8b15fdd03e3a7 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -140,17 +140,6 @@ mod tests { assert!(storm.forgive()); } #[test] - fn recovery_table_ends_at_its_first_invalid_record() { - use crate::g17::recovery::Entries; - let mut entries = Entries::default(); - assert!(entries.include(1 | (12 << 4))); - assert!(entries.include(1 | 2 | (17 << 4))); - assert!(!entries.include(0)); - assert_eq!(entries.listed, 2); - assert_eq!(entries.guilty, 1u128 << 17); - assert_eq!(entries.cancelled, 0); - } - #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); diff --git a/drivers/gpu/drm/asahi/g17/render_resources.rs b/drivers/gpu/drm/asahi/g17/render_resources.rs new file mode 100644 index 0000000000000..fb9f6a2bfd504 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/render_resources.rs @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Immutable linked-program storage, shared by commands in one submission. + +use super::fw::m5::{ + pool::PoolGeometry, + pool_budget::{History, RenderBudget, Sizes}, + render_demand::{ComputeStorage, FragmentStorage, RenderDemand, TileStorage, VertexStorage}, +}; +use super::fw::m5::compute::{ScratchRequest, IntersectionTables}; +use crate::util::Reader; +use kernel::{prelude::*, uapi}; + +// SAFETY: The UAPI record contains only fixed-width unsigned integers. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_render_program {} +// SAFETY: The versioned record contains only unsigned integer storage. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_render_program_v2 {} + +pub(crate) struct Programs { + entries: KVec, +} +impl Programs { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_render_program = reader.read()?; + Self::validate(&entry)?; + entries.push(Self::wrap(entry), GFP_KERNEL)?; + } + Ok(Self { entries }) + } + + pub(crate) fn from_entries(values: &[uapi::drm_asahi_render_program]) -> Result { + if values.is_empty() { return Err(EINVAL); } + let mut entries = KVec::with_capacity(values.len(), GFP_KERNEL)?; + for entry in values { + Self::validate(entry)?; + entries.push(Self::wrap(*entry), GFP_KERNEL)?; + } + Ok(Self { entries }) + } + + fn wrap(legacy: uapi::drm_asahi_render_program) -> uapi::drm_asahi_render_program_v2 { + uapi::drm_asahi_render_program_v2 { legacy, threads: 0, usage_flags: 0, + scs_layout: 0, auxiliary_allocation: 0, reserved: 0, + intersection_pool: 0, intersection_pool_bytes: 0, + intersection_first: 0, intersection_count: 0, + intersection_default_program: 0, intersection_default_control: 0 } + } + + pub(crate) fn parse_v2(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_render_program_v2 = reader.read()?; + let mesh = entry.usage_flags & (uapi::DRM_ASAHI_RENDER_PROGRAM_MESH + | uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT) != 0; + let task = entry.usage_flags & crate::file::DRM_ASAHI_RENDER_PROGRAM_TASK != 0; + if entry.reserved != 0 || entry.usage_flags & !0xf != 0 + || entry.usage_flags & 6 == 6 || entry.legacy.flags != 0 + || (task && entry.usage_flags & uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT == 0) + || entry.legacy.stage > 1 || (mesh && (entry.legacy.stage != 0 + || entry.threads == 0 || entry.legacy.auxiliary_count != 0 + || entry.legacy.imageblock_bytes != 0 || entry.legacy.multisample_bytes != 0)) + || (!mesh && entry.threads != 0) { return Err(EINVAL); } + if !mesh { Self::validate(&entry.legacy)?; } + let scratch = ScratchRequest::new(entry.scs_layout)?; + let intersection = Self::intersection(&entry)?; + if scratch.enabled() != (entry.usage_flags & uapi::DRM_ASAHI_RENDER_PROGRAM_SCS != 0) + || scratch.enabled() != intersection.enabled() + || (!scratch.enabled() && (entry.scs_layout != 0 || entry.auxiliary_allocation != 0)) { + return Err(EINVAL); + } + entries.push(entry, GFP_KERNEL)?; + } + let programs = Self { entries }; + // A render kick has one SCS layout and one intersection-table binding + // per engine, even when several linked programs contribute demand. + programs.stage_scs()?; + Ok(programs) + } + + fn intersection(e: &uapi::drm_asahi_render_program_v2) -> Result { + IntersectionTables::new(e.intersection_pool, e.intersection_pool_bytes, + e.intersection_first, e.intersection_count, + e.intersection_default_program, e.intersection_default_control) + } + + /// Any linked program runs as a hardware mesh or object (task) stage. + pub(crate) fn uses_mesh(&self) -> bool { + self.entries.iter().any(|e| e.usage_flags + & (uapi::DRM_ASAHI_RENDER_PROGRAM_MESH | uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT) != 0) + } + + /// An API task (object) stage is linked. Only such kicks get the TA mesh + /// rendering buffer: only the object→mesh TA program binds it, never the + /// mesh-single, grid or indirect ones. + pub(crate) fn uses_task(&self) -> bool { + self.entries.iter().any(|e| e.usage_flags & crate::file::DRM_ASAHI_RENDER_PROGRAM_TASK != 0) + } + + /// 0 ordinary, 1 hardware mesh (including the mesh-only passthrough object + /// stage), 2 API task (object) → mesh. + pub(crate) fn mesh_class(&self) -> u8 { + if self.uses_task() { 2 } + else if self.uses_mesh() { 1 } + else { 0 } + } + + pub(crate) fn stage_scs(&self) -> Result<[StageScratch; 2]> { + let mut stages = [StageScratch::NONE; 2]; + for e in &self.entries { + if e.scs_layout == 0 { continue; } + let next = StageScratch { scratch: ScratchRequest::new(e.scs_layout)?, + intersection: Self::intersection(e)?, auxiliary_allocation: e.auxiliary_allocation }; + let out = &mut stages[e.legacy.stage as usize]; + if out.scratch.enabled() && *out != next { return Err(EINVAL); } + *out = next; + } + Ok(stages) + } + + fn validate(entry: &uapi::drm_asahi_render_program) -> Result { + if entry.stage > 1 || entry.flags != 0 + || (entry.stage == 0 && (entry.auxiliary_count != 0 + || entry.threadgroup_bytes != 0 || entry.imageblock_bytes != 0 + || entry.multisample_bytes != 0)) { return Err(EINVAL); } + Ok(()) + } + + fn demands(&self, mut demand: RenderDemand) -> Result<(RenderDemand, [RenderDemand; 2])> { + let mut stages = [demand; 2]; + stages[0].include_vertex(VertexStorage::default()).map_err(|_| EINVAL)?; + let empty = FragmentStorage::default().parameters().map_err(|_| EINVAL)?; + let no_tile = TileStorage::default().parameters(false).map_err(|_| EINVAL)?; + stages[1].include_fragment(empty, no_tile).map_err(|_| EINVAL)?; + demand.include_vertex(VertexStorage::default()).map_err(|_| EINVAL)?; + demand.include_fragment(empty, no_tile).map_err(|_| EINVAL)?; + for e in &self.entries { + let entry = &e.legacy; + if entry.stage == 0 { + let storage = VertexStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + }; + if e.usage_flags & 6 != 0 { + let storage = ComputeStorage { + primary_bytes: storage.primary_bytes, secondary_bytes: storage.secondary_bytes, + linked_bytes: storage.linked_bytes, minimum_primary_bytes: storage.minimum_primary_bytes, + threadgroup_bytes: entry.threadgroup_bytes, threads: e.threads, + }; + demand.include_mesh(storage, 96, 114688).map_err(|_| EINVAL)?; + stages[0].include_mesh(storage, 96, 114688).map_err(|_| EINVAL)?; + } else { + demand.include_vertex(storage).map_err(|_| EINVAL)?; + stages[0].include_vertex(storage).map_err(|_| EINVAL)?; + } + } else { + let fragment = FragmentStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + auxiliary_count: entry.auxiliary_count, + }.parameters().map_err(|_| EINVAL)?; + let tile = TileStorage { + threadgroup_bytes: entry.threadgroup_bytes, + imageblock_bytes: entry.imageblock_bytes, + multisample_bytes: entry.multisample_bytes, + }.parameters(false).map_err(|_| EINVAL)?; + demand.include_fragment(fragment, tile).map_err(|_| EINVAL)?; + stages[1].include_fragment(fragment, tile).map_err(|_| EINVAL)?; + } + } + Ok((demand, stages)) + } +} + +/// Frozen device topology and shared-queue policy. No per-command allocation. +#[derive(Clone, Copy)] +pub(crate) struct M5Policy { + demand: RenderDemand, + budgets: [RenderBudget; 2], + compute_budgets: [RenderBudget; 2], +} + +#[derive(Clone, Copy)] +pub(crate) struct Plan { + pub(crate) sizes: Sizes, + pub(crate) geometry: PoolGeometry, + pub(crate) resource_flags: u64, + pub(crate) pressure: u32, + pub(crate) stages: Option<[StagePlan; 2]>, +} + +#[derive(Clone, Copy)] +pub(crate) struct StagePlan { + pub(crate) scs: StageScratch, + pub(crate) resource_flags: u64, + pub(crate) ptc_max: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct StageScratch { + pub(crate) scratch: ScratchRequest, + pub(crate) intersection: IntersectionTables, + pub(crate) auxiliary_allocation: u32, +} +impl StageScratch { + pub(crate) const NONE: Self = Self { scratch: ScratchRequest::NONE, + intersection: IntersectionTables::NONE, auxiliary_allocation: 0 }; +} + +impl M5Policy { + pub(crate) fn new(id: &crate::regs::G17Id) -> Result { + // Configured G17S context storage per enabled core. This is separate + // from the final per-kick PTC cap selected during command preparation. + const CONTEXT_FLOOR: u32 = 96; + let nominal = id.num_cores.checked_mul(id.num_clusters).ok_or(EOVERFLOW)?; + Ok(Self { + demand: RenderDemand::new(&id.core_masks, nominal, CONTEXT_FLOOR) + .map_err(|_| EINVAL)?, + compute_budgets: [ + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 3).map_err(|_| EINVAL)?, + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 2).map_err(|_| EINVAL)?, + ], + budgets: [ + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 3).map_err(|_| EINVAL)?, + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 4).map_err(|_| EINVAL)?, + ], + }) + } + + pub(crate) fn plan_compute(&self, programs: &ComputePrograms, priority: u8) -> Result { + self.plan_compute_scs(programs, priority, 0, 0) + } + + pub(crate) fn plan_compute_scs(&self, programs: &ComputePrograms, priority: u8, + scs_layout: u64, auxiliary_pages: u32) -> Result { + let scratch = super::fw::m5::compute::ScratchRequest::new(scs_layout)?; + // UMA pool class 1 is realtime; class 2 covers all other + // public priorities. Its resident-kick count also depends on engine. + let (budget, active_kicks) = match priority { + 0 => (&self.compute_budgets[0], 3), + 1..=3 => (&self.compute_budgets[1], 2), + _ => return Err(EINVAL), + }; + let mut demand = self.demand; + for entry in &programs.entries { + demand.include_compute(ComputeStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + threadgroup_bytes: entry.threadgroup_bytes, + threads: entry.threads, + }, 96, 114688).map_err(|_| EINVAL)?; + } + let request = budget.compute_scs_requirements( + demand.requirements().map_err(|_| EINVAL)?, scratch.pages(), auxiliary_pages) + .map_err(|_| EINVAL)?; + let sizes = budget.sizes(request, History::default()).map_err(|_| EINVAL)?; + Ok(Plan { + geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, + resource_flags: demand.resource_flags().map_err(|_| EINVAL)?, + pressure: (demand.pressure() / active_kicks).clamp(96, 512), + stages: None, + }) + } + + pub(crate) fn plan(&self, programs: &Programs, priority: u8) -> Result { + let (budget, active_kicks) = match priority { 0 => (&self.budgets[0], 3), 1..=3 => (&self.budgets[1], 4), + _ => return Err(EINVAL) }; + let (demand, stage_demands) = programs.demands(self.demand)?; + let scs = programs.stage_scs()?; + let stage = |d: RenderDemand, scs| -> Result { Ok(StagePlan { + scs, + resource_flags: d.resource_flags().map_err(|_| EINVAL)?, + ptc_max: (d.pressure() / active_kicks).clamp(96, 512), + }) }; + let request = budget.render_scs_requirements( + demand.requirements().map_err(|_| EINVAL)?, + scs[0].scratch.pages().max(scs[1].scratch.pages()), + scs[0].auxiliary_allocation.max(scs[1].auxiliary_allocation)).map_err(|_| EINVAL)?; + let sizes = budget.sizes(request, History::default()) + .map_err(|_| EINVAL)?; + Ok(Plan { + geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, + resource_flags: demand.resource_flags().map_err(|_| EINVAL)?, + pressure: demand.pressure(), + stages: Some([stage(stage_demands[0], scs[0])?, stage(stage_demands[1], scs[1])?]), + }) + } +} + +// SAFETY: The record contains only fixed-width unsigned integers. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_compute_program {} + +pub(crate) struct ComputePrograms { + entries: KVec, +} +impl ComputePrograms { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_compute_program = reader.read()?; + if entry.flags != 0 || entry.threads == 0 { return Err(EINVAL); } + entries.push(entry, GFP_KERNEL)?; + } + Ok(Self { entries }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 543612cd211a6..a3bafd969981d 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -5,7 +5,7 @@ //! Writers and readers hold the device mutex; reads never power the GPU up. use super::{ - fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, + fw::channels::EVENT_WORDS, Role, }; use kernel::prelude::*; @@ -19,6 +19,9 @@ struct Sample { words: [u32; EVENT_WORDS], } +/// One sample per statistics tag a family's 64-bit mask can name. +const TAGS: usize = u64::BITS as usize; + pub(super) struct Reports { samples: KVec, } @@ -26,25 +29,27 @@ pub(super) struct Reports { impl Reports { pub(super) fn new() -> Result { // Allocate before firmware starts; receive and debugfs paths allocate nothing. - let mut samples = KVec::with_capacity(64, GFP_KERNEL)?; - for _ in 0..64 { + let mut samples = KVec::with_capacity(2 * TAGS, GFP_KERNEL)?; + for _ in 0..2 * TAGS { samples.push(Sample::default(), GFP_KERNEL)?; } Ok(Self { samples }) } + /// `allowed` is the family's statistics tag mask (`InitData::statistics_types`). pub(super) fn record( &mut self, role: Role, words: [u32; EVENT_WORDS], host_ns: u64, + allowed: u64, admitted: bool, ) { let tag = words[0]; - if tag >= u32::BITS || STATISTICS_TYPES & (1 << tag) == 0 { + if tag >= u64::BITS || allowed & (1u64 << tag) == 0 { return; } - let sample = &mut self.samples[role as usize * 32 + tag as usize]; + let sample = &mut self.samples[role as usize * TAGS + tag as usize]; sample.count = sample.count.saturating_add(1); sample.admitted = sample.admitted.saturating_add(u64::from(admitted)); sample.last_admitted = admitted; @@ -66,8 +71,8 @@ impl Reports { writeln!( f, "role={} tag={:#x} count={} host_ns={} admitted={} last_admitted={} words={:08x?}", - index / 32, - index % 32, + index / TAGS, + index % TAGS, sample.count, sample.host_ns, sample.admitted, @@ -131,25 +136,30 @@ impl kernel::debugfs::Writer for PowerView { mod tests { use super::*; + /// The G17 family's statistics tags, as `InitData::statistics_types` gives them. + const G17: u64 = super::super::fw::channels::STATISTICS_TYPES as u64; + /// M5's, which add tag 32. + const M5: u64 = super::super::fw::m5::STATISTICS_TYPES; + #[test] fn retains_all_words_and_separates_role_and_tag() { let mut reports = Reports::new().unwrap(); let mut words = [0; EVENT_WORDS]; words[0] = 16; words[17] = 0xdeadbeef; - reports.record(Role::Primary, words, 100, false); - reports.record(Role::Primary, words, 101, true); - reports.record(Role::Secondary, words, 102, false); + reports.record(Role::Primary, words, 100, G17, false); + reports.record(Role::Primary, words, 101, G17, true); + reports.record(Role::Secondary, words, 102, G17, false); assert_eq!(reports.samples[16].count, 2); assert_eq!(reports.samples[16].host_ns, 101); assert_eq!(reports.samples[16].words, words); assert_eq!(reports.samples[16].admitted, 1); assert!(reports.samples[16].last_admitted); - assert_eq!(reports.samples[48].count, 1); - assert_eq!(reports.samples[48].admitted, 0); - assert!(!reports.samples[48].last_admitted); + assert_eq!(reports.samples[TAGS + 16].count, 1); + assert_eq!(reports.samples[TAGS + 16].admitted, 0); + assert!(!reports.samples[TAGS + 16].last_admitted); words[0] = 17; - reports.record(Role::Primary, words, 103, true); + reports.record(Role::Primary, words, 103, G17, true); assert_eq!(reports.samples[17].count, 1); assert_eq!(reports.samples[16].count, 2); } @@ -157,11 +167,25 @@ mod tests { #[test] fn rejects_invalid_tags_before_indexing() { let mut reports = Reports::new().unwrap(); - for tag in [0, 15, 29, 31, 32, u32::MAX] { + for tag in [0, 15, 29, 31, 32, 63, 64, u32::MAX] { let mut words = [0; EVENT_WORDS]; words[0] = tag; - reports.record(Role::Primary, words, 1, true); + reports.record(Role::Primary, words, 1, G17, true); } assert!(reports.samples.iter().all(|s| s.count == 0)); } + + #[test] + fn follows_the_family_mask() { + let mut reports = Reports::new().unwrap(); + let mut words = [0; EVENT_WORDS]; + words[0] = 32; + reports.record(Role::Secondary, words, 1, G17, true); + assert_eq!(reports.samples[TAGS + 32].count, 0); + reports.record(Role::Secondary, words, 2, M5, true); + assert_eq!(reports.samples[TAGS + 32].count, 1); + words[0] = 64; + reports.record(Role::Secondary, words, 3, u64::MAX, true); + assert_eq!(reports.samples.iter().filter(|s| s.count != 0).count(), 1); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 9a96e1adbe654..94ed6fc391aa1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -174,6 +174,8 @@ pub(super) struct Registry { pub(super) compute_building: bool, compute_scheduler_building: bool, replay_outer_wait: Option, + /// A compute replay waits for its owner's renders to park or idle (compute_replay). + replay_consumer_wait: Option, pub(super) observations: compute::retirement::Batch, /// Completion-ring slot where the next compute scan starts. completion_cursor: usize, @@ -211,7 +213,7 @@ impl Registry { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, + metrics: Arc::new(MetricsIds::for_firmware(variant.m5_commands)?, GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, @@ -226,6 +228,7 @@ impl Registry { compute_building: false, compute_scheduler_building: false, replay_outer_wait: None, + replay_consumer_wait: None, observations: compute::retirement::Batch::new()?, completion_cursor: 0, completion_seen: { @@ -264,8 +267,12 @@ impl Registry { /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only - /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + /// when no populated idle queue of the owner's VM exists. A populated pool + /// serves one VM: a queue whose backing served another VM is not bound. + /// An open owner's idle channel serving another VM is shared for fresh + /// backing; while any queue waits for such a release, none is shared, and + /// the requester waits for the vacant queue. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { @@ -283,7 +290,35 @@ impl Registry { if self.qids.compute_fits(self.admission.queues() as usize) { return Ok(Rebind::Fresh); } + if self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.vacancy_pending()) + { + return Err(ENOSPC); + } let teardown = &self.teardown; + let vm = context.vm(); + // A released channel whose backing served another VM goes first: vacate it, then + // re-arm it fresh for this one. + let foreign = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.reusable() && !queue.pool_vacant() && !queue.pool_serves(vm)) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| (queue.released_at(), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + if let Some(qid) = foreign { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .require_fresh_backing(); + return Err(ENOSPC); + } let idle = self .compute .iter() @@ -291,29 +326,36 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.shareable()) .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + // A channel serving the requester's VM first, then the best fit, then the longest idle. + .map(|queue| ((!queue.pool_serves(vm), !queue.pool_fits(required), queue.last_published()), queue.qid())) .min_by_key(|(key, _)| *key) .map(|(_, qid)| qid); let Some(qid) = idle else { return Err(ENOSPC); }; admission.check()?; - let shared = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid) - .ok_or(EIO)? - .share_idle()?; - if !shared { - return Err(ENOSPC); + { + let mut queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)?; + if !queue.share_idle()? { + return Err(ENOSPC); + } + if !queue.pool_serves(vm) { + // Its backing holds another VM's scratch: vacate it, then re-arm fresh. + queue.require_fresh_backing(); + return Err(ENOSPC); + } } Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) } /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks /// only whether some channel is free for a new owner. - fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { - if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + fn compute_readiness(&self, owner: u64, query: Option<(&Arc, super::freelist::ComputeCapacity)>) -> Readiness { + if let (Some((context, required)), Some(queue)) = (query, self.compute(owner).filter(|_| owner != 0)) { return if queue.spared_pending() { Readiness::Progress } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { @@ -344,7 +386,7 @@ impl Registry { /// Binds `owner` to a reusable channel whose last owner released it, or /// names a vacant one to re-arm. None if no channel is reusable. - fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; for mut queue in self @@ -362,14 +404,24 @@ impl Registry { continue; } if queue.pool_vacant() { - vacant.get_or_insert(queue.qid()); + if queue.rearm_open(owner) { + vacant.get_or_insert(queue.qid()); + } continue; } - if !queue.pool_fits(required) { continue; } + if !queue.pool_fits(required) || !queue.pool_serves(context.vm()) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Some(Rebind::Done)); } - Ok(vacant.map(Rebind::Rearm)) + let Some(qid) = vacant else { return Ok(None) }; + // Reserved for this owner until it binds the queue or gives up: other owners wait for + // a channel instead of each building a pool for the same queue. + self.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .reserve_rearm(owner); + Ok(Some(Rebind::Rearm(qid))) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -379,7 +431,7 @@ impl Registry { qid: u8, pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, - ) -> Result<(u16, u64)> { + ) -> Result { let mut queue = self .compute .iter_mut() @@ -390,6 +442,16 @@ impl Registry { queue.rearm(pages, ring) } + /// Ends `owner`'s re-arm reservation of `qid` when it did not bind the queue. + pub(super) fn release_rearm(&mut self, qid: u8, owner: u64) { + if let Some(mut queue) = self.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + { + queue.release_rearm(owner); + } + } + pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; let mut queue = self @@ -618,6 +680,8 @@ pub(super) enum Readiness { Channel, /// Retirement-bound capacity or reopened publication. Progress, + /// M5: no unclaimed room on the shared work ring. + Outer, } /// Outcome of offering an owner a retained compute queue. @@ -677,11 +741,21 @@ pub(super) struct Backend { /// channel from being shared away between prepare_job and run_job. #[pin] claimed: kernel::sync::Mutex>>, + /// M5: this owner's claims on the work ring M5 TA, fragment and compute share, one per + /// engine, taken while preparing a publication and consumed by it + /// ([`Backend::claim_outer`], [`Backend::publish_claimed`]). + #[pin] + outer_compute: kernel::sync::Mutex>, + #[pin] + outer_render: kernel::sync::Mutex>, /// Render pair admission, shared by every logical queue of the session. admission: Arc, /// The render pair this owner last published to without the device mutex /// (zero for none): a hint, revalidated by every lend. direct_slot: core::sync::atomic::AtomicU32, + /// Render pairs this owner may hold, fixed at creation with its scheduler's + /// credits and its render pool's sizing. + pairs: u8, } impl Backend { @@ -719,6 +793,8 @@ impl Backend { { return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); } + // The new context may reuse a pooled scheduler page: clear released QoS rows first. + firmware.queues.qos_ids.forget_released(|ids| firmware.port.qos_forget(ids)); let attempt = if firmware.uat.execution_contexts_used() < crate::mmu::MAX_EXECUTION_CONTEXTS { // Creators hold the device mutex, so neither check above can // have changed; the slot only fails on a broken invariant. @@ -798,8 +874,11 @@ impl Backend { closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + outer_compute <- kernel::new_mutex!(None, "g17::Backend::outer_compute"), + outer_render <- kernel::new_mutex!(None, "g17::Backend::outer_render"), admission, direct_slot: core::sync::atomic::AtomicU32::new(0), + pairs: super::pairs::per_owner(), }), GFP_KERNEL, )?, @@ -810,6 +889,11 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. fn shared_compute(&self, admission: &Admission<'_>) -> Result { + // The shared aperture is G17P compute state. M5 compute queues need only + // their own pools, and the objects it would install are not M5's. + if self.context.is_m5() { + return Ok(()); + } loop { admission.check()?; let lease = self.shared.preparations.enter_until(admission)?; @@ -952,7 +1036,7 @@ impl Backend { Ok(()) } - fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + fn grow_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; { let state = self.shared.state.lock(); @@ -965,8 +1049,9 @@ impl Backend { let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; - let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); - let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); + let mut pages = Some(super::freelist::Pages::for_compute(&alloc, required, + self.context.vm().memory_client())?); + let mut alias = Some(pages.as_mut().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { @@ -992,7 +1077,7 @@ impl Backend { firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } } - queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) + queue.finish_growth(&firmware.init, &mut pages, &mut alias).map(Some) })(); deferred.finish(); match result { @@ -1033,7 +1118,53 @@ impl Backend { result } - fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + /// Re-arms the vacant queue `qid` reserved for this owner with fresh zeroed backing, built + /// without the device mutex, and binds it. Some(true): bound; Some(false): retry (the + /// preparation epoch changed or the queue is no longer re-armable); None: no memory for + /// the backing now. + fn rearm_vacant( + &self, + qid: u8, + required: super::freelist::ComputeCapacity, + alloc: &super::object::Allocator<'_>, + objects: &Arc, + lease: &super::preparation::Lease, + admission: &Admission<'_>, + ) -> Result> { + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::for_compute(alloc, required, + self.context.vm().memory_client())?; + Ok((pages, compute::DescriptorRing::new(alloc, objects)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => return Ok(None), + result => result?, + }; + let mut state = self.shared.state.lock(); + admission.check()?; + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Ok(Some(false)); + } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // Leave rejected backing local so it is dropped after the device guard, not in + // rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm() && queue.rearm_open(self.owner)); + if !ready { drop(state); return Ok(Some(false)); } + let descriptor = match firmware.queues.rearm_compute(qid, pages, ring) { + Err(error) if error == EAGAIN => return Ok(Some(false)), + result => result?, + }; + // The firmware consumed this slot's release before the backing + // was dropped; the row is rewritten before the queue is kicked. + firmware.init.install_compute_pool(descriptor)?; + firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; + Ok(Some(true)) + } + + fn build_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; self.admission_open()?; loop { @@ -1110,47 +1241,28 @@ impl Backend { let reservation = match reservation { Ok(reservation) => reservation, Err(qid) => { - // Fresh zeroed backing is built without the device mutex; the - // queue may have been taken or re-armed by another owner since. - let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) - })(); - let (pages, ring) = match built { - Err(error) if error == ENOSPC => { + // bind_reusable reserved `qid` for this owner: any outcome but binding it + // ends the reservation, so another owner may re-arm it. + let result = self.rearm_vacant(qid, required, &alloc, &objects, &lease, admission); + if !matches!(result, Ok(Some(true))) { + if let Some(firmware) = self.shared.state.lock().as_deref_mut() { + firmware.queues.release_rearm(qid, self.owner); + } + } + match result? { + Some(true) => return Ok(()), + Some(false) => continue, + None => { drop(lease); self.wait_channels(observed, admission, true)?; continue; } - result => result?, - }; - let mut state = self.shared.state.lock(); - admission.check()?; - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - continue; } - if let Err(error) = self.admission_open() { drop(state); return Err(error); } - // A competing ready owner may have taken/rearmed it while - // backing was built off-lock. Leave rejected backing local - // so it is dropped after the device guard, not in rearm(). - let ready = firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.qid() == qid && queue.can_rearm()); - if !ready { drop(state); continue; } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { - Err(error) if error == EAGAIN => continue, - result => result?, - }; - // The firmware consumed this slot's release before the backing - // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; - firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; - return Ok(()); } }; let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; + let pool = compute::PoolBacking::new(&alloc, reservation.id, required, + self.context.vm().memory_client())?; { let mut state = self.shared.state.lock(); admission.check()?; @@ -1160,13 +1272,15 @@ impl Backend { } self.admission_open()?; firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; + firmware.init.install_compute_pool(pool.pool_descriptor()?)?; } let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease, admission)?; + // It is G17P compute state; M5 has none. + if !self.context.is_m5() { + self.compute_scheduler(&alloc, &lease, admission)?; + } let queue = compute::Queue::new( &alloc, &objects, @@ -1175,7 +1289,9 @@ impl Backend { self.context.clone(), self.shared.variant.pointers, self.shared.operand_state, + self.shared.m5_board, self.shared.variant.compute, + alloc.uat.is_m5(), )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); @@ -1320,8 +1436,9 @@ impl DeferredBatch { Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => { completion.note_input_error(*error); }, - Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => - completion.note_source_retired(), + Deferred::Retired(completion, Ok(_)) + | Deferred::RetiredCompute(completion, _, _) + | Deferred::Drained(completion, Ok(_)) => completion.note_source_retired(), _ => {}, } self.values.push_within_capacity(value).map_err(|_| EIO)?; @@ -1330,6 +1447,7 @@ impl DeferredBatch { // succeeds; failure fences and rejected events are not retirement. if let Some( Deferred::Retired(completion, _) + | Deferred::RetiredCompute(completion, _, _) | Deferred::Drained(completion, _) | Deferred::Release(completion), ) = self.values.last() @@ -1351,6 +1469,7 @@ impl DeferredBatch { let mut changed = false; for value in &self.values { if let Deferred::Retired(completion, Ok(_)) + | Deferred::RetiredCompute(completion, _, _) | Deferred::Drained(completion, Ok(_)) = value { if let Err(error) = completion.check_source_inputs() { @@ -1381,7 +1500,9 @@ impl DeferredBatch { } struct ComputeHost<'a> { + shared: &'a Arc, init: &'a super::initdata::InitData, + regs: &'a crate::regs::G17Resources, port: &'a super::port::Port, outer: Option>, effort: &'a mut super::power::Effort, @@ -1399,6 +1520,10 @@ impl compute::Host for ComputeHost<'_> { )) } fn prepare_compute_shared(&mut self) -> Result { + // The global compute state is G17P's; M5 has none to activate. + if self.init.is_m5() { + return Ok(()); + } self.init.activate_compute() } fn qos_publish( @@ -1418,16 +1543,32 @@ impl compute::Host for ComputeHost<'_> { fn register_queue(&mut self, queue: &super::fw::kick::KsmQueue) -> Result { self.port.register_queues(core::slice::from_ref(queue)) } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + if self.init.is_m5() { + return self.init.publish_work(priority, slot).map(|_| ()); + } self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { + // Every M5 pipe-2 doorbell (0x83) is noted in the firmware's submission history first, + // as the render doorbell notes its pair. + if self.init.is_m5() && message >> super::MSG_TYPE_SHIFT == 0x83 && message & 3 == 2 { + if let Err(error) = self.init.note_m5_submission_history() { + pr_warn!("asahi: M5 submission history skipped: {:?}\n", error); + } + } self.port.notify(message) } fn note_submission(&mut self) -> Result { + self.shared.note_submitter(); + self.shared.note_gpu_activity(); self.effort.submit(); + self.effort.note_waiters(super::now_ns(), super::fence::waiters()); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) } + fn submit_m5(&mut self, submission: &super::fw::m5::queue::HardwareSubmission) -> Result { + self.regs.submit_m5_work(submission) + } } /// Publication into a bound, installed queue through the port alone. The caller checked the @@ -1474,6 +1615,10 @@ impl compute::Host for DirectHost<'_> { fn note_submission(&mut self) -> Result { Ok(()) } + fn submit_m5(&mut self, _submission: &super::fw::m5::queue::HardwareSubmission) -> Result { + // M5 never publishes without the device mutex (see compute_direct). + Err(EINVAL) + } } impl super::Firmware { @@ -1502,6 +1647,10 @@ impl super::Firmware { let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; let mut native: KVec = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + // M5 shared-ring queues have no completion-wait records: the consumer's kick entry names + // the source's kick, and the firmware orders it. + let m5 = self.init.is_m5(); + let mut waited: u128 = 0; for source in &sources.sources { if source.submission.source_status()? { continue; } let found = self.queues.compute.iter_mut().flatten() @@ -1514,6 +1663,12 @@ impl super::Firmware { let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { return Err(EAGAIN); }; + waited |= 1u128 << dependency.qid(); + if m5 { + merged.insert(sequence, dependency)?; + holds.push(hold, GFP_KERNEL)?; + continue; + } // The completion-wait record orders the consumer behind its // source; naming the source in the kick entry as well only adds // a second, slower wait and a predecessor record. @@ -1531,7 +1686,7 @@ impl super::Firmware { if packet.is_published() { for queue in self.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + .filter(|queue| waited & (1u128 << queue.qid()) != 0) { for source in &sources.sources { queue.note_input_consumer(source); } } @@ -1554,8 +1709,19 @@ impl super::Firmware { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // reservation; dropping it unused returns the work-ring slot. - let outer = self.port.reserve_compute(packet.context.policy().priority())?; + // reservation; dropping it unused returns the work-ring slot. M5 compute takes its + // slot on the work ring M5 render shares, under this mutex (InitData::publish_work): + // a full ring is refused here, before a queue install or kick changes any state. + let outer = if self.init.is_m5() { + if !self.init.work_ready(packet.context.policy().priority(), + super::fw::queue::DataMaster::Compute)? + { + return Err(EAGAIN); + } + None + } else { + Some(self.port.reserve_compute(packet.context.policy().priority())?) + }; let queues = &mut self.queues; let mut queue = queues .compute @@ -1574,9 +1740,11 @@ impl super::Firmware { return Err(EAGAIN); } let mut host = ComputeHost { + shared: &self.primary.state.shared, init: &self.init, + regs: &self.bridge.regs, port: &self.port, - outer: Some(outer), + outer, effort: &mut self.effort, qids: &mut queues.qids, generation: self.recovery.generation(), @@ -1594,7 +1762,7 @@ impl Registry { /// Everything ensure_compute establishes for `owner`: the shared compute /// objects mapped into its VM, and its queue bound to `context` with a /// pool that fits and no spared witness to release. - fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + fn compute_bound(&self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity) -> Result { let Some(queue) = self.compute(owner) else { return Ok(false); }; @@ -1754,9 +1922,20 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + /// The compute pool capacity `command` needs, in its family's terms: the M5 + /// plan's geometry, or the G17P block count of its SCS request. + pub(super) fn compute_capacity(&self, command: &super::command::Validated) -> Result { + use super::freelist::ComputeCapacity; + let super::command::Validated::Compute { scratch, plan, .. } = command else { return Err(EINVAL); }; + if self.context.is_m5() { + return Ok(ComputeCapacity::M5(plan.map(|plan| plan.geometry))); + } + super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) + .map(ComputeCapacity::Blocks) + } + + fn ensure_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages)?; let mut deferred = DeferredBatch::publication()?; loop { admission.check()?; @@ -1794,10 +1973,13 @@ impl Backend { fn compute_direct( &self, packet: &Arc, - required: usize, + required: super::freelist::ComputeCapacity, deferred: &mut DeferredBatch, ) -> Option { - if packet.completion.sources().is_some() + // M5 compute is a shared-ring submission on the ring M5 render shares: it is published only + // under the device mutex, never through the classic compute ring producer. + if self.context.is_m5() + || packet.completion.sources().is_some() || self.context.render_seen() || packet.cancel_requested() || packet.completion.status().get() != 0 @@ -1815,6 +1997,7 @@ impl Backend { if !port.note_submission() { return None; } + self.shared.note_gpu_activity(); match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { Ok(_) => {} Err(error) if error == EBUSY => return None, @@ -1854,26 +2037,96 @@ impl Backend { { return None; } - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()).ok()?; + let required = self.compute_capacity(&packet.command).ok()?; + let outer = self.outer_domain(); loop { let channels = self.shared.channels.observe(); let progress = self.shared.progress.observe(); - let (domain, observed) = match self.compute_readiness(packet, required) { - Readiness::Ready => return None, - Readiness::Channel => (&self.shared.channels, channels), - Readiness::Progress => (&self.shared.progress, progress), + let claimed = outer.as_ref().map(|domain| domain.observe()); + let (domain, observed) = match (self.compute_readiness(packet, required), &outer, claimed) { + (Readiness::Ready, ..) => return None, + (Readiness::Channel, ..) => (&self.shared.channels, channels), + (Readiness::Progress, ..) => (&self.shared.progress, progress), + (Readiness::Outer, Some(domain), Some(observed)) => (domain, observed), + (Readiness::Outer, ..) => return None, }; - self.shared.queue_events(); match domain.wait(observed) { - Ok(Some(fence)) => return Some(fence), + Ok(Some(fence)) => { + // Registered first: the event worker polls outer consumption only for waiters. + self.shared.queue_events(); + return Some(fence); + } Ok(None) => continue, Err(_) => return None, } } } - fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + /// M5: the domain outer-ring claims wait on. None off M5 or without firmware. + fn outer_domain(&self) -> Option> { + self.shared.state.lock().as_deref().and_then(|firmware| firmware.init.outer_available().ok()) + } + + /// M5: holds this owner's claim on the shared work ring for the next compute or render + /// publication, reserving one under the device mutex unless one with credits left is + /// held. False: no unclaimed room; wait on [`Self::outer_domain`] from a sample taken + /// before this call. Always true off M5. + fn claim_outer(&self, firmware: &super::Firmware, engine: crate::g17::job::Engine) -> Result { + if !firmware.init.is_m5() { + return Ok(true); + } + let (held, count) = match engine { + crate::g17::job::Engine::Compute => ( + &self.outer_compute, + firmware.queues.compute(self.owner).map_or(2, |queue| queue.outer_slots()), + ), + // Tiling and fragment: one slot each. + crate::g17::job::Engine::Render => (&self.outer_render, 2), + }; + let mut held = held.lock(); + if held.as_ref().is_some_and(|claim| claim.remaining() != 0) { + return Ok(true); + } + match firmware.init.reserve_outer(self.context.policy().priority(), count) { + Ok(claim) => { + // Replaces only a spent claim, whose drop notifies nobody. + *held = Some(claim); + Ok(true) + } + Err(error) if error == EBUSY => Ok(false), + Err(error) => Err(error), + } + } + + /// Publishes into this owner's compute channel, lending the held outer claim's credits. + fn publish_claimed( + &self, + firmware: &mut super::Firmware, + packet: &Arc, + dependencies: &[super::fw::kick::KickDependency], + deferred: &mut DeferredBatch, + ) -> Result { + let mut held = self.outer_compute.lock(); + let _scope = (*held) + .as_mut() + .filter(|claim| claim.remaining() != 0) + .map(|claim| firmware.init.outer_scope(claim)) + .transpose()?; + firmware.publish_compute(self.owner, packet.clone(), dependencies, deferred) + } + + /// Returns an engine's unspent outer claim after its publication, cancellation or close. + /// Outside the device mutex: returned credits notify waiters. + pub(super) fn release_outer(&self, engine: crate::g17::job::Engine) { + let held = match engine { + crate::g17::job::Engine::Compute => &self.outer_compute, + crate::g17::job::Engine::Render => &self.outer_render, + }; + let claim = held.lock().take(); + drop(claim); + } + + fn compute_readiness(&self, packet: &Arc, required: super::freelist::ComputeCapacity) -> Readiness { match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { Err(error) if error == EBUSY => return Readiness::Progress, Err(_) => return Readiness::Ready, @@ -1894,7 +2147,15 @@ impl Backend { }); if bound { self.hold_claim(Some(cell)); - return Readiness::Ready; + if !self.context.is_m5() { + return Readiness::Ready; + } + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + return match self.claim_outer(firmware, crate::g17::job::Engine::Compute) { + Ok(false) => Readiness::Outer, + _ => Readiness::Ready, + }; } } } @@ -1916,7 +2177,13 @@ impl Backend { { return Readiness::Progress; } - firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + match firmware.queues.compute_readiness(self.owner, Some((&self.context, required))) { + Readiness::Ready => match self.claim_outer(firmware, crate::g17::job::Engine::Compute) { + Ok(false) => Readiness::Outer, + _ => Readiness::Ready, + }, + other => other, + } } /// Replaces the prepared packet's channel claim; None releases it after publication, @@ -1995,10 +2262,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Option { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { - return Some(Err(EINVAL)); - }; - let required = match super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) { + let required = match self.compute_capacity(&packet.command) { Ok(required) => required, Err(error) => return Some(Err(error)), }; @@ -2012,8 +2276,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; + let required = self.compute_capacity(&packet.command)?; if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { return Err(render::FALLBACK); } @@ -2046,8 +2309,11 @@ impl Backend { { return Err(render::FALLBACK); } + if !self.claim_outer(firmware, crate::g17::job::Engine::Compute)? { + return Err(render::FALLBACK); + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + match self.publish_claimed(firmware, packet, dependencies.as_slice(), deferred) { Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { Err(render::FALLBACK) } @@ -2089,8 +2355,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; + let required = self.compute_capacity(&packet.command)?; if let Some(result) = self.compute_direct(packet, required, deferred) { return result; } @@ -2101,7 +2366,7 @@ impl Backend { // publication after recovery both signal `progress`. let mut bound = true; let mut demand = None; - let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let deadline = now_ns().saturating_add(super::preparation::QUEUE_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } self.admission_open()?; @@ -2110,9 +2375,11 @@ impl Backend { }; let admission = Admission::new(deadline, &check); let progress = &self.shared.progress; + let outer = self.outer_domain(); loop { admission.check()?; let observed = progress.observe(); + let claimed = outer.as_ref().map(|domain| domain.observe()); if !self.sources_published(packet)? { let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { firmware.recovery.pending() @@ -2130,7 +2397,7 @@ impl Backend { // admission and publication without changing this logical queue. if !bound { demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); - Backend::ensure_compute(self, scratch.pages(), &admission)?; + Backend::ensure_compute(self, required, &admission)?; admission.check()?; } let lease = self.shared.preparations.enter_until(&admission)?; @@ -2178,14 +2445,19 @@ impl Backend { drop(lease); continue; } + let held = self.claim_outer(firmware, crate::g17::job::Engine::Compute)?; + if let (false, Some(domain), Some(claimed)) = (held, &outer, claimed) { + // The shared work ring has no unclaimed room: consumption or returned claims + // free it. + drop(state); + drop(lease); + self.shared.queue_events(); + self.wait_release(domain, claimed, &admission)?; + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; admission.check()?; - let result = firmware.publish_compute( - self.owner, - packet.clone(), - dependencies.as_slice(), - deferred, - ); + let result = self.publish_claimed(firmware, packet, dependencies.as_slice(), deferred); if !packet.is_published() && (matches!(result, Err(EAGAIN)) || (matches!(result, Err(EBUSY)) @@ -2301,6 +2573,17 @@ impl super::Shared { state.take() }; if let Some(mut firmware) = firmware { + if !firmware.init.has_engine_retirement_witness() { + // Engines may still write into the session's pages after the processors + // stop, and nothing here proves they have stopped: keep the session. Its + // callbacks run this module's code: the start pinned the module, so this + // only confirms the pin. + if super::retain_session().is_err() { + dev_err!(self.dev.as_ref(), "GPU session kept without a module pin\n"); + } + core::mem::forget(firmware); + return; + } firmware.finish_stopped_queues(); // RTKit destruction still dereferences the mailbox provider. drop(firmware); @@ -2355,7 +2638,7 @@ impl super::Firmware { // states) still ends a scheduler wait that has no timeout of its own. if freed || (shared.channels.has_waiters() - && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + && self.queues.compute_readiness(0, None) == Readiness::Ready) { shared.channels.notify(); } @@ -2528,7 +2811,7 @@ mod shared_capacity_tests { assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = registry.admission.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); leases.push(lease, GFP_KERNEL).unwrap(); } @@ -2551,7 +2834,7 @@ mod shared_capacity_tests { let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = registry.admission.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); leases.push(lease, GFP_KERNEL).unwrap(); } @@ -2570,7 +2853,7 @@ mod shared_capacity_tests { for client in 0..4u64 { for queue in 0..9u64 { let owner = 1 + client * 9 + queue; - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + let lease = registry.admission.acquire_render(owner, 1, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); leases.push(lease, GFP_KERNEL).unwrap(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 10085c3fcb922..c83edeef6ba1a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -8,7 +8,7 @@ use crate::g17::{ job::{self, Engine, Packet}, }; use core::sync::atomic::Ordering; -use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; +use kernel::{dma_fence::{Fence, RawDmaFence}, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { @@ -94,6 +94,8 @@ impl Backend { // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); self.hold_claim(None); + self.release_outer(Engine::Compute); + self.release_outer(Engine::Render); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -116,6 +118,15 @@ impl Backend { } impl job::Backend for Backend { + fn creator_cluster_scheduling(&self) -> bool { + self.shared.follows_submitter() + } + fn fence_queue(&self) -> Arc { + self.shared.fence_queue() + } + fn render_pairs(&self) -> u32 { + u32::from(self.pairs) + } fn input_sources(&self, inputs: &[crate::file::SyncItem]) -> Option> { @@ -169,16 +180,49 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } + fn supports_submission(&self) -> bool { + self.shared.submission_enabled + } + + fn supports_engine(&self, engine: crate::g17::job::Engine) -> bool { + !self.shared.firmware_test || engine == crate::g17::job::Engine::Compute + } + + fn prepare_command(&self, command: &mut crate::g17::command::Validated) -> Result { + use crate::g17::command::Validated; + let Some(policy) = self.shared.render_policy else { return Ok(()) }; + let priority = self.context.policy().priority(); + match command { + Validated::Render { resources, plan, .. } => { + let programs = resources.as_deref().ok_or(EINVAL)?; + *plan = Some(policy.plan(programs, priority)?); + // The M5 target layout refuses the passes the M5 path cannot run. Derive it + // here, so that SUBMIT returns the refusal before anything is queued. + crate::g17::queue::render::m5_render_submission::layout( + command, self.shared.clusters, self.shared.variant.programs)?; + } + Validated::Compute { resources, plan, m5_scratch, cdm_auxiliary_allocation, .. } => { + if let Some(programs) = resources { + *plan = Some(policy.plan_compute_scs(programs, priority, m5_scratch.layout(), + *cdm_auxiliary_allocation)?); + } else if m5_scratch.enabled() { + // SCS needs the dispatch storage declarations to size its pool. + return Err(EINVAL); + } + } + } + Ok(()) + } + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { - crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages) + crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages, self.pairs) } - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, command: &crate::g17::command::Validated) -> Result { // A reverse timeline chain must be submitted before the host signal. // Validate here, but do not consume physical capacity behind unsignalled // DRM dependencies. compute_packet admits it when the job can run. - crate::g17::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages) - .map(|_| ()) + self.compute_capacity(command).map(|_| ()) } fn fail_vm(&self, error: Error) { @@ -253,6 +297,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { self.hold_claim(None); } + self.release_outer(packet.engine()); self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -284,6 +329,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { self.hold_claim(None); } + self.release_outer(packet.engine()); if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -295,6 +341,33 @@ impl job::Backend for Backend { if result.is_err() { self.stop_after_error(); } + // drm_sched frees the job as soon as this returns while the job's + // completion callback is still on its scheduler fence, so the fence + // must be signalled by now even when the queue could not settle the + // packet (a busy or failed runtime, a settle racing another queue's + // timeout). A completion its physical queue still owns keeps the VM + // pinned until the queue retires it; that later completion is a no-op. + if !packet.completion.signalled() { + packet.completion.fail_while_owned(ECANCELED); + } + // Another thread may have claimed the completion and not yet signalled. + let fence = packet.completion.scheduler_fence(); + // SAFETY: the retained fence stays alive across this bounded wait, + // which holds no device lock. + unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(1000) as _, + ) + }; + } + + /// Not on M5: a job that does not finish within its watchdog interval fails then, however + /// recently other work retired. Its compute kill then follows the first expiry. + fn renews_shared(&self) -> bool { + let state = self.shared.state.lock(); + (*state).as_deref().is_none_or(|firmware| !firmware.init.is_m5()) } fn long_job_beside(&self, _packet: &Arc) -> bool { @@ -324,6 +397,14 @@ impl job::Backend for Backend { if let Some(deferred) = deferred.as_mut() { firmware.scan_renders(deferred)?; } + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init, renewals) { + if renewals == 0 { + dev_info!(self.shared.dev.as_ref(), + "Render watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", + self.owner, older, own); + } + return Ok(true); + } // Borrowed-graph and queued-retirement deferrals share the renewal bound. if timeout::transaction_grace(renewals) && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed53..dfa343a9ac275 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,6 +20,14 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init, renewals) { + if renewals == 0 { + dev_info!(self.shared.dev.as_ref(), + "Compute watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", + self.owner, older, own); + } + return Ok(Decision::Defer); + } // Each queue is locked on its own: release this one before visiting the others. let Some(own) = firmware.queues.compute(self.owner) .filter(|queue| queue.can_classify_timeout(packet)) @@ -49,6 +57,23 @@ impl Backend { matches!(result, Ok(Decision::Defer)) } + /// The older node of the packet's context ahead of its own, with its own word, when the + /// firmware's progress word moved within the watchdog's interval, the packet's VM has no + /// failed work and the job's deferrals are within budget + /// ([`crate::g17::timeout::behind_own_head`]). The word is read at every poll. Firmware + /// without the word renews nothing by this rule. + pub(super) fn behind_own_head(&self, packet: &Arc, + init: &crate::g17::initdata::InitData, renewals: u8) -> Option<(u32, u32)> { + let work = packet.completion.work_state().ok()?; + let older = work.older_node(); + let progressing = init.m5_progress().is_ok_and(crate::g17::timeout::note_progress); + let status = packet.completion.status(); + let failed = status.get() != 0 || status.reported_error() != 0; + crate::g17::timeout::behind_own_head(packet.is_published(), older, progressing, + failed, renewals) + .then(|| (older.unwrap_or(0), work.word())) + } + /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 9e12842f41c02..4b311f31982e1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,7 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, - timeout::LIVENESS_NS, + timeout::{LIVENESS_NS, RECOVERY_WAIT_MS}, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; @@ -143,6 +143,9 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { + // Released contexts' QoS rows stop naming their scheduler pages at the next event + // turn, and at the latest before a new context is created. + self.queues.qos_ids.forget_released(|ids| self.port.qos_forget(ids)); let recovery_closed = self.init.recovery_state()? == 0; let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { @@ -201,6 +204,28 @@ impl Firmware { ); } } + // M5: a command the watchdog failed is killed once its quarantine delay has passed, + // also while its owner keeps the queue open (`Queue::open_timeout_kill`). + for index in 0..self.queues.compute.len() { + let due = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| queue.open_timeout_kill(now)); + let Some((context, age)) = due else { + continue; + }; + if !self.queues.render.teardown_idle(&context)? { + continue; + } + if let Err(error) = self.publish_compute_exit(&context, age, now, &mut pressed) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit request failed: {:?}\n", + context.id(), + error + ); + } + } Ok(()) } @@ -242,7 +267,7 @@ impl Firmware { } return Ok(()); } - } else if self.kill_unsafe(context, now) { + } else if self.kill_unsafe(context, age, now) { return Ok(()); } let (owner, target) = { @@ -326,8 +351,11 @@ impl Firmware { /// retired on the firmware's fault record, so no compute queue is left to carry a kill, /// yet those pairs are released only after one: without this the context, its pairs and /// its memory stay until the processors stop. + /// + /// M5 has none: it issues a context kill only from a compute queue's exit, so a + /// render-only M5 context is never killed. fn render_kill_candidate(&self, index: usize, context: &Arc, age: u64) -> bool { - render_kill_eligible( + !self.init.is_m5() && render_kill_eligible( age, self.queues.teardown.render_kill_published(index) || context.kill_published(), self.queues.render.executed_slots(context) != 0, @@ -428,8 +456,11 @@ impl Firmware { /// minutes of a five-lane load with 67 faults), even with nothing freed before the kill's /// acknowledgement; such a kill takes the next moment without foreign compute instead. /// Render work is not waited for. A GPU that retired nothing for [`LIVENESS_NS`] has - /// stopped. + /// stopped. The wait is Neo's; M5 publishes its kills at once. fn foreign_compute_running(&self, context: &Arc, now: u64) -> bool { + if self.init.is_m5() { + return false; + } let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); !stalled && self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).any(|queue| { @@ -438,13 +469,15 @@ impl Firmware { } /// A compute command of `context` failed and the firmware holds commands behind it that it - /// never runs. `publish_compute_exit` is only reached once `teardown_idle` found all of - /// the context's render pairs idle; the pairs behind such a failure become idle when + /// never runs, or, on M5, the watchdog failed one whose commands may never retire + /// (`Queue::timed_out`). `publish_compute_exit` is only reached once `teardown_idle` found + /// all of the context's render pairs idle; the pairs behind such a failure become idle when /// their commands have executed (`Pair::firmware_drained`), so by then nothing of the /// context runs any more. fn compute_failed(&self, context: &Arc) -> bool { self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).any(|queue| { - queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && queue.fault_pending() + queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + && (queue.fault_pending() || queue.timed_out()) }) } @@ -456,9 +489,13 @@ impl Firmware { /// kill proceeds. A context whose compute failed ([`Self::compute_failed`]) /// is not left alone, since its held commands never retire: its kill waits /// only for other contexts' compute ([`Self::foreign_compute_running`]). - fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + /// + /// On M5 the wait ends [`RECOVERY_WAIT_MS`] after the context closed (`age`): a closed + /// context's command that never terminates would otherwise hold its queue, pairs and + /// memory for as long as other clients keep the GPU busy. + fn kill_unsafe(&self, context: &Arc, age: u64, now: u64) -> bool { let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); - if stalled { + if stalled || (self.init.is_m5() && age >= u64::from(RECOVERY_WAIT_MS) * 1_000_000) { return false; } self.queues.render.foreign_in_flight(context) @@ -480,7 +517,7 @@ impl Firmware { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_polling(now)) + .any(|queue| queue.exit_polling(now) || queue.open_timeout_polling(now)) { return true; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 32b9acef94c0f..0aca0ec0c8ffe 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -6,6 +6,9 @@ use super::DeferredBatch; use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; +/// Bound for a compute replay waiting on its owner's render abandonment proof. +const CONSUMER_WAIT_NS: u64 = 300_000_000; + impl Firmware { /// Called only from an outer worker or client submission, never from event /// dispatch or a recovery callback nested within publication. All candidate @@ -18,6 +21,7 @@ impl Firmware { if state != 0 { return Ok(()); } + self.queues.replay_consumer_wait = None; let commands = self .queues .compute @@ -53,7 +57,51 @@ impl Firmware { else { break; }; - let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; + // A render consumer named this command through a firmware dependency. + // Republish only while no render of the owner has executed since this recovery + // (none consumed the dead token), after every live render of the owner is + // parked for replay; the render replay then depends on the new command. A + // parked render this command waits through is republished first, so the new + // command depends on it again. + let generation = self.recovery.generation(); + let parked_prefix = owner.is_some_and(|owner| { + self.queues.render.owner_parked_through(owner, packet.order.wait_through[0]) + }); + let (executed, live) = if packet.completion.has_firmware_consumer() { + owner.map_or((true, false), |owner| self.queues.render.owner_render_state(owner, generation)) + } else { + (false, false) + }; + let must_wait = parked_prefix || (!executed && live); + let now = super::now_ns(); + let waited = if must_wait { + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .consumer_wait(now) + } else { + 0 + }; + if must_wait && waited < CONSUMER_WAIT_NS { + // The event worker polls while this waits. + self.queues.replay_consumer_wait = Some(now); + return Ok(()); + } + let consumer = (executed || must_wait).then_some(ENODATA); + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .clear_consumer_wait(); + // M5 compute takes its slot on the work ring M5 render shares, whose unclaimed + // room the publication checks (InitData::work_ready). + let priority = packet.context.policy().priority(); + let outer_ready = if self.init.is_m5() { + self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Compute)? + } else { + self.port.compute_ready(priority)? + }; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -66,7 +114,9 @@ impl Firmware { self.queues.replay_outer_wait = None; } - let result = if !outer_ready { + let result = if let Some(error) = consumer { + Err(error) + } else if !outer_ready { // This replay has no live firmware ownership. Persistent // backpressure uses the existing failed-replay retirement. Err(EBUSY) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4af2704be0b44..5114465145081 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -30,6 +30,9 @@ impl event::Transport for Service<'_> { fn roles(&self) -> &'static [Role] { self.firmware.roles() } + fn reports_after_signal(&self) -> bool { + self.firmware.init.is_m5() + } fn trace(&mut self, role: Role) { let firmware = &mut *self.firmware; firmware @@ -130,6 +133,9 @@ impl recovery::Host for Service<'_> { } impl recovery::Memory for Service<'_> { + fn is_m5(&self) -> bool { + self.firmware.init.is_m5() + } fn recovery_state(&self) -> Result { self.firmware.init.recovery_state() } @@ -137,7 +143,30 @@ impl recovery::Memory for Service<'_> { self.firmware.init.host_recovery() } fn fault_sources(&self) -> Result { - self.firmware.init.fault_sources() + let mut mmu_fault = false; + if self.firmware.init.is_m5() { + let registers = self.firmware.bridge.regs.m5_fault_registers(); + pr_err!("asahi: recovery fault registers: {:x?}\n", registers); + // MMU fault status: valid (bit 0), reason (1..3, 0 = unmapped), read + // (4), level (7..8), requestor (9..16), VM slot (17..22); the address + // word holds the faulting address shifted right by 6. + if let Ok([_, status, address, ..]) = registers { + if status & 1 != 0 { + mmu_fault = true; + let access = if status & 0x10 != 0 { "read" } else { "write" }; + pr_err!("asahi: recovery: GPU MMU fault: {} reason {} at {:#x}, level {}, requestor {:#x}, VM slot {}\n", + access, (status >> 1) & 7, + (address & ((1 << 42) - 1)) << 6, (status >> 7) & 3, (status >> 9) & 0xff, + (status >> 17) & 0x3f); + } + } + } + let mut sources = self.firmware.init.fault_sources()?; + // M5 reads no firmware reason word; a latched MMU fault is reason 3. + if mmu_fault && sources.reason.is_none() { + sources.reason = Some(recovery::PAGE_FAULT); + } + Ok(sources) } fn transition(&self, expected: u32, next: u32) -> Result { @@ -301,6 +330,21 @@ impl Firmware { Ok(()) } + /// Drains every role's trace ring, then their statistics rings. On M5 the event worker calls + /// this after it has signalled the completions of its pass: the reports are not scheduling + /// inputs. + fn drain_reports(&mut self) { + let dev = &self.primary.state.shared.dev; + for &role in self.roles() { + self.rings.drain_trace(dev, &self.init, role); + } + for &role in self.roles() { + self.observe_statistics_power(); + self.rings.drain_statistics(&self.primary.state.shared.dev, &self.init, role); + self.observe_statistics_power(); + } + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -440,6 +484,10 @@ impl crate::g17::Shared { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } + if firmware.init.is_m5() { + // The completions above are signalled; the reports that arrived with them come after. + firmware.drain_reports(); + } if recover { firmware.service_runtime_recovery(deferred)?; if !firmware.recovery.pending() { @@ -518,9 +566,14 @@ impl crate::g17::Shared { self.queue_reclaims(); } } + // The firmware may consume M5 work-ring slots without a completion interrupt: while + // a preparation waits for unclaimed room, every pass polls the consumers. + let outer = firmware.init.poll_outer_progress()?; let backpressured = firmware.render_control_backpressured(); let polling = backpressured + || firmware.init.outer_waiting() || firmware.queues.replay_outer_wait.is_some() + || firmware.queues.replay_consumer_wait.is_some() || firmware.queues.render.replay_polling() || firmware.queues.teardown.polling(now) || firmware.compute_exit_polling(now) @@ -570,6 +623,10 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Outside the device mutex: the notification can run DMA-fence callbacks. + if let Some(domain) = outer { + domain.notify(); + } // Firmware ring room, consumed releases and teardown steps have no // event of their own: recheck render publishers' conditions every pass. if self.renders.has_waiters() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index fc2e01be761ae..4b62d96d7cd3c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -524,6 +524,8 @@ impl Registry { struct Host<'a> { firmware: &'a mut crate::g17::Firmware, registration_powered: bool, + /// An M5 pair's installation, submitted with its first outer publication. + m5_registration: Option, } impl Host<'_> { fn release_registration(&mut self) { @@ -542,6 +544,9 @@ impl Host<'_> { } } impl crate::g17::queue::render::Host for Host<'_> { + fn generation(&self) -> u64 { + self.firmware.recovery.generation() + } fn epoch(&self) -> Result<(u64, u32)> { Ok(( self.firmware.recovery.generation(), @@ -571,8 +576,18 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. - fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + /// M5's host command port installs and kicks the pair in one transaction: + /// its registration is retained until the outer publication, without the + /// core-power hold. + fn install_pair(&mut self, registration: &crate::g17::queue::render::Registration) -> Result { use crate::g17::fw::variant::QueueRegistration; + let registration = match registration { + crate::g17::queue::render::Registration::M5(submission) => { + self.m5_registration = Some(*submission); + return Ok(()); + } + crate::g17::queue::render::Registration::Neo(registration) => registration, + }; match self.firmware.variant.registration { QueueRegistration::Firmware => return Ok(()), QueueRegistration::Mailbox => { @@ -611,12 +626,44 @@ impl crate::g17::queue::render::Host for Host<'_> { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]> { + if self.firmware.init.is_m5() { + let submission = match self.m5_registration.take() { + Some(submission) => submission, + None => crate::g17::fw::m5::queue::HardwareSubmission::advance_pair( + fragment.qid, + tiling.qid, + )?, + }; + return self.firmware.submit_m5_render_pair( + priority, + &[copy_slot(fragment), copy_slot(tiling)], + &submission, + commit, + late_tiling, + true, + ); + } self.firmware .init .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) } + fn publish_configuration_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + prepare: impl FnOnce(), + ) -> Result { + let pair = self.firmware.init.m5_work_pair(priority, fragment, tiling)?; + pair.prepare(); + prepare(); + pair.publish(); + Ok(()) + } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { - self.release_registration(); + if !self.firmware.init.is_m5() { + self.release_registration(); + } let result = self.firmware.queues.render.pools.activate(id, generation); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); @@ -639,16 +686,28 @@ impl crate::g17::queue::render::Host for Host<'_> { } result } - /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. M5 renders + /// use the common compute work ring, pipe 2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); - crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { - self.firmware.bridge.regs.g17_core_power() - }); - let result = self - .firmware - .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); + let m5 = self.firmware.init.is_m5(); + if m5 { + // A paired publication is one tiling and one fragment submission. + for _ in 0..2 { + if let Err(error) = self.firmware.init.note_m5_submission_history() { + pr_warn!("asahi: M5 submission history skipped: {:?}\n", error); + } + } + } else { + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); + } + let result = self.firmware.primary.notify( + (0x83 << crate::g17::MSG_TYPE_SHIFT) + | u64::from(priority) << 2 + | if m5 { 2 } else { 0 }, + ); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } @@ -684,6 +743,18 @@ impl crate::g17::queue::render::Host for Host<'_> { } } +/// The M5 pair submission takes both work slots as one array. +fn copy_slot(slot: &crate::g17::fw::channels::WorkSlot) -> crate::g17::fw::channels::WorkSlot { + crate::g17::fw::channels::WorkSlot { + unk_00: slot.unk_00, + queue_va: slot.queue_va, + data_master: slot.data_master, + items_end: slot.items_end, + qid: slot.qid, + first: slot.first, + } +} + impl crate::g17::Firmware { /// The pair is returned even when transport, accounting or collection fails. /// No off-lock operation or callback which can re-enter the device runs here. @@ -703,6 +774,7 @@ impl crate::g17::Firmware { let mut host = Host { firmware: self, registration_powered: false, + m5_registration: None, }; let result = operation(&mut pair, &mut host); host.release_registration(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index e150a10e2c9bc..b31a37dae1fd1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -10,6 +10,14 @@ impl Registry { &self, owner: u64, packet: &Arc, + ) -> Result { + self.render_prefix_at(owner, packet, packet.order.wait_through[0]) + } + pub(in crate::g17::runtime) fn render_prefix_at( + &self, + owner: u64, + packet: &Arc, + prefix: Option, ) -> Result { let mut frontier = Frontier::new(); let compute = packet.engine() == crate::g17::job::Engine::Compute; @@ -18,7 +26,7 @@ impl Registry { pair.collect_dependencies( owner, packet.completion.status(), - packet.order.wait_through[0], + prefix, &mut frontier, compute, )?; @@ -26,7 +34,7 @@ impl Registry { dependencies.collect( owner, packet.completion.status(), - packet.order.wait_through[0], + prefix, &mut frontier, compute, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index 409b948ee6aef..c0bb025d2f639 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -45,6 +45,9 @@ struct DirectHost<'a> { } impl crate::g17::queue::render::Host for DirectHost<'_> { + fn generation(&self) -> u64 { + self.generation + } fn next_publication(&mut self) -> Result { Ok(self.port.next_publication()) } @@ -65,7 +68,7 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.port.qos_cancel(publication) } - fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + fn install_pair(&mut self, _registration: &crate::g17::queue::render::Registration) -> Result { Err(EAGAIN) } fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { @@ -86,6 +89,16 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { self.port .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) } + fn publish_configuration_pair( + &mut self, + _priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + _prepare: impl FnOnce(), + ) -> Result { + // Only M5 publishes configuration pairs, never on this path. + Err(EINVAL) + } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { // The lease leaves preparation when the pair is returned; nothing can // observe it in between because the pair is lent. @@ -186,6 +199,7 @@ impl crate::g17::Firmware { let lease = PublicationLease { free_list_generation: preparation.generation, buffer_token, + buffer_context: None, }; let pair = match self.queues.render.take_pair(slot, owner, packet) { Ok(pair) => pair, @@ -263,8 +277,10 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Option { // A compute prefix needs the compute queue's state; leave it to the - // device-mutex path. - if packet.is_published() + // device-mutex path. M5 renders publish through the device mutex + // only: their host-port transaction is part of the publication. + if packet.context.is_m5() + || packet.is_published() || packet.cancel_requested() || packet.completion.status().get() != 0 || packet.order.wait_through[1].unwrap_or(0) != 0 @@ -281,6 +297,7 @@ impl Backend { if !port.note_submission() { return None; } + self.shared.note_gpu_activity(); match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Render) { Ok(_) => {} Err(error) if error == EBUSY => return None, @@ -320,6 +337,7 @@ impl Backend { self.owner, slot, footprint, + packet.order.wait_through[0], &mut dependencies.hazards, )?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 277534328406e..032a73759733b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -251,7 +251,7 @@ impl crate::g17::Shared { manager.prepare_growth( &dev, &vm, - crate::g17::buffer::firmware_growth_target(before), + manager.firmware_target(), None, ) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 1e3354d90dbfb..ab94764580521 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -122,7 +122,17 @@ impl Pools { } fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + if pool.is_m5() { + let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool.m5_descriptor()?; + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + return Ok(()); + } + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; pool.repopulate()?; let descriptor = pool .page_pool_state()? @@ -240,8 +250,13 @@ impl Pools { /// A detached pool may capture firmware counters only after its last pair /// stopped using this registration. Callers check all retained pair owners. pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.is_m5() { + let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + return pool.store_m5_descriptor(words.map(|word| word.load(Ordering::Relaxed))); + } let state = pool.page_pool_state()?; - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; fence(Ordering::Acquire); let descriptor = PagePoolDescriptor { page_list: words[0].load(Ordering::Relaxed), @@ -256,14 +271,12 @@ impl Pools { } } -fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { - let offset = Pools::slot(id)? * size_of::(); - Ok([ - table.dword(offset)?, - table.dword(offset + 8)?, - table.dword(offset + 16)?, - table.dword(offset + 24)?, - ]) +/// The `N` descriptor words of pool `id`: four on G17P, eight on M5. +fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; N]> { + let offset = Pools::slot(id)? * N * size_of::(); + // Check the full row before constructing the infallible array of views. + table.pointer(offset, N * size_of::())?; + Ok(core::array::from_fn(|i| table.dword(offset + i * 8).unwrap())) } impl Pools { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 62ca7895a86f8..0f9153c5938a7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -6,6 +6,7 @@ use super::{Config, Reservation}; use crate::g17::{ buffer::MetricsLease, + freelist::{PoolPin, RenderPool}, preparation::Lease, job::Packet, object::{Allocator, KernelObject, Pool}, @@ -44,6 +45,28 @@ impl BuildResources { fn metrics(&self) -> Result { MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) } + /// The render pool geometry the command needs: M5 commands carry a plan. + fn pool_plan(&self, packet: &Packet) -> Result> { + let crate::g17::command::Validated::Render { plan, .. } = &packet.command else { + return Err(EINVAL); + }; + if self.uat.is_m5() != plan.is_some() { + return Err(EINVAL); + } + Ok(*plan) + } + /// M5 maps the render's own metrics entry into the context's VM. Called during + /// off-lock preparation, after the context's bindings exist. + fn bind_metrics( + &self, + context: &crate::g17::context::Context, + metrics: &mut MetricsLease, + ) -> Result { + if self.uat.is_m5() { + metrics.map_client(context.vm())?; + } + Ok(()) + } } #[expect( @@ -78,6 +101,148 @@ struct BuildContext<'a> { } impl super::super::Backend { + /// Reserves scarce M5 pool identities before the pair build transfers a + /// physical render pair: the owner's fitting pool, or the context's pool + /// for the command's plan, pinned. The pin is held across the build; the + /// pair takes its own. + fn claim_render_pool( + &self, + first: &Arc, + lease: &Lease, + ) -> Result<(Arc, Option)> { + let dev = self.shared.drm()?; + let (resources, existing) = { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + let resources = BuildResources::new(firmware)?; + let geometry = resources.pool_plan(first)?.map(|plan| plan.geometry); + let mut existing = None; + for entry in firmware.queues.render.entries.iter().flatten() { + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + if pair.matches(self.owner, &self.context)? + && pair.pool().fits(geometry) + && !pair.pool().vacant() + { + existing = Some(pair.pool().clone()); + break; + } + } + (resources, existing) + }; + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + let pool = match existing { + Some(pool) => pool, + None => self.context.render_pool( + &alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?, + }; + let pin = self.pin_render_pool(&pool, &resources, lease)?; + Ok((pool, pin)) + } + + /// Pins `pool`'s identity. When every M5 identity is taken, an identity + /// whose final free-list release firmware consumed is reclaimed and its + /// physical storage kept for `pool`. ENOSPC when none is reclaimable: the + /// caller waits for render capacity. + fn pin_render_pool( + &self, + pool: &Arc, + resources: &BuildResources, + lease: &Lease, + ) -> Result> { + let mut reclaimed_pages = None; + loop { + if !lease.is_current() { + return Err(EAGAIN); + } + match pool.pin_id() { + Ok(pin) => { + if let Some(reclaimed) = reclaimed_pages.take() { + pool.cache_reclaimed(reclaimed); + } + return Ok(pin); + } + Err(error) if error == ENOSPC && resources.uat.is_m5() => { + // Repeat the release observation after a failed claim: a + // concurrent pin may have taken the reclaimed identity. + let reclaimed = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + firmware.queues.render.pools.observe(&firmware.init)?; + resources + .ids + .reclaim_one(|id| firmware.queues.render.pools.released(u32::from(id), 0)) + }; + // Dropped outside the device lock: its mappings may take + // memory-management locks. + drop(reclaimed_pages.take()); + if reclaimed.is_some() { + reclaimed_pages = reclaimed; + continue; + } + self.shared.queue_events(); + return Err(error); + } + Err(error) => return Err(error), + } + } + } + + /// Arms an M5 pool's backing for the pair build. A pool without storage + /// takes the physical storage of a released identity first, without + /// evicting a metadata-only owner. The caller holds the pool's pin. + fn prepare_pool_backing( + &self, + pool: &Arc, + resources: &BuildResources, + lease: &Lease, + client: &crate::mmu::Vm, + ) -> Result { + if resources.uat.is_m5() && pool.needs_storage() { + let reclaimed = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + firmware.queues.render.pools.observe(&firmware.init)?; + resources + .ids + .reclaim_backing(|id| firmware.queues.render.pools.released(u32::from(id), 0)) + }; + if let Some(reclaimed) = reclaimed { + pool.cache_reclaimed(reclaimed); + } + } + let dev = self.shared.drm()?; + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + pool.ensure_backing(&alloc, &resources.global, client) + } + fn build_render_pair( &self, slot: u8, @@ -87,6 +252,12 @@ impl super::super::Backend { ) -> Result { let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; + // M5 pool identities are scarce: reserve one before any pair moves. + let _pool_claim = if self.context.is_m5() { + Some(self.claim_render_pool(first, &lease)?) + } else { + None + }; let (resources, build) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -130,7 +301,12 @@ impl super::super::Backend { ) } else { let pair = entry.pair.as_ref().ok_or(EBUSY)?; - if pair.matches(self.owner, &self.context)? { + // An M5 pool that no longer fits the command's geometry + // is replaced by a same-owner rebind below. + let plan = resources.pool_plan(first)?; + if pair.matches(self.owner, &self.context)? + && pair.pool().fits(plan.map(|p| p.geometry)) + { if !pair.pool().vacant() { return Ok(()); } @@ -140,9 +316,17 @@ impl super::super::Backend { dev: &dev, uat: &resources.uat, }; + if resources.uat.is_m5() { + return self.prepare_pool_backing( + &pool, + &resources, + &lease, + self.context.vm(), + ); + } return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); } - if !pair.reusable()? { + if !pair.bindable(self.owner, &self.context)? { return Err(EBUSY); } // The release of the pair's scheduler is armed or still being @@ -152,7 +336,14 @@ impl super::super::Backend { if firmware.queues.teardown.releasing(pair.context()) { return Err(EBUSY); } - let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); + // On M5 a different logical context, even of the same VM, + // gets its own graph; only a pool rebind of the same + // context keeps the graph and its queue records. + let foreign = if resources.uat.is_m5() { + !Arc::ptr_eq(pair.context(), &self.context) + } else { + !Arc::ptr_eq(pair.context().status(), self.context.status()) + }; if foreign { firmware .queues @@ -247,7 +438,7 @@ impl super::super::Backend { ctx: &BuildContext<'_>, reservation: Reservation, pm_generation: u64, - metrics: MetricsLease, + mut metrics: MetricsLease, ) -> Result { let BuildContext { lease, @@ -257,9 +448,16 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result> { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + resources.bind_metrics(&self.context, &mut metrics)?; + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } Pair::new( alloc, &resources.objects, @@ -277,6 +475,7 @@ impl super::super::Backend { resources.config.deflake, resources.config.descriptor_flags, self.shared.variant, + self.shared.m5_board, ) })(); let (mut built, error) = match built { @@ -328,7 +527,7 @@ impl super::super::Backend { mut parked: crate::g17::queue::render::parked::ParkedPair, buffer: (u8, u64), pm_generation: u64, - metrics: MetricsLease, + mut metrics: MetricsLease, ) -> Result { let BuildContext { lease, @@ -338,9 +537,17 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + parked.clear_for_vm(self.context.vm().id())?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + resources.bind_metrics(&self.context, &mut metrics)?; + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } let prepared = parked.prepare( alloc, &resources.objects, @@ -444,7 +651,7 @@ impl super::super::Backend { mut pair: KBox, buffer: Option<(u8, u64)>, pm_generation: Option, - metrics: Option, + mut metrics: Option, ) -> Result { let BuildContext { lease, @@ -454,9 +661,18 @@ impl super::super::Backend { first, } = *ctx; let prepared = (|| { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + if let Some(metrics) = metrics.as_mut() { + resources.bind_metrics(&self.context, metrics)?; + } + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } pair.prepare_bind( alloc, &resources.objects, @@ -499,11 +715,15 @@ impl super::super::Backend { { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; + // A rebind of the same context (a render that needs a bigger pool) keeps its + // live scheduler state; only another context's is displaced. + if !pair.matches(self.owner, &self.context)? { + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + } pair.bind_owner( prepared.as_mut().ok_or(EIO)?, alloc, @@ -593,8 +813,8 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; - // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. - if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + // An owner held to one pair has no sibling to warm. + if self.pairs == 1 { return Ok(()); } let sibling = { @@ -625,7 +845,9 @@ impl super::super::Backend { packet: &Arc, deferred: &mut super::super::DeferredBatch, sleep: bool, + outer_wait: &mut Option<(Arc, u64)>, ) -> Result { + *outer_wait = None; let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { @@ -648,6 +870,15 @@ impl super::super::Backend { firmware.init.stamp_addresses(qids[1])?, ], }; + // M5: the pair's two slots on the shared work ring are claimed before preparing + // and lent to its publication. Sampled first: a miss waits for the next return. + if let Ok(domain) = firmware.init.outer_available() { + let observed = domain.observe(); + if !self.claim_outer(firmware, crate::g17::job::Engine::Render)? { + *outer_wait = Some((domain, observed)); + return Err(EBUSY); + } + } let pair = firmware .queues .render @@ -737,6 +968,12 @@ impl super::super::Backend { if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { return Err(ECANCELED); } + let mut held = self.outer_render.lock(); + let _scope = (*held) + .as_mut() + .filter(|claim| claim.remaining() != 0) + .map(|claim| firmware.init.outer_scope(claim)) + .transpose()?; firmware.publish_render(self.owner, slot, packet, deferred) })(); let result = (|| { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 499e3858fcca1..aa78a7ae63c66 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -10,6 +10,29 @@ use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; use kernel::prelude::*; impl super::RenderState { + /// (an executed group of the owner retired in `generation` or later, + /// a published group of the owner is still live and not parked). + pub(in crate::g17) fn owner_render_state(&self, owner: u64, generation: u64) -> (bool, bool) { + let mut executed = false; + let mut live = false; + for pair in self.entries.iter().flatten().filter_map(|entry| entry.pair.as_ref()) { + if pair.owner() != Some(owner) { + continue; + } + executed |= pair.executed_since(generation); + live |= pair.live_unparked(); + } + (executed, live) + } + /// A parked render of the owner precedes a command waiting through `through`. + pub(in crate::g17) fn owner_parked_through(&self, owner: u64, through: Option) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.owner() == Some(owner) && pair.replay_through(through)) + } + pub(in crate::g17) fn replay_polling(&self) -> bool { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 970035547dfee..985e823a7fd1a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -83,7 +83,7 @@ impl Backend { if !sleep { return Err(FALLBACK); } - match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + match pool.acquire_render(self.owner, self.pairs.into(), Request::of(packet), deadline_ns, || { // Atomics only: the callback runs under the pool mutex. if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { return Err(ECANCELED); @@ -258,11 +258,23 @@ impl Backend { observed: &mut Option, sleep: bool, packet: &Packet, + ) -> Result { + self.wait_render_retirement(clock, observed, sleep, packet, super::super::WORK_RING_WAIT_NS) + } + + /// [`Self::wait_render_busy`] bounded at `bound_ns`. + fn wait_render_retirement( + &self, + clock: &mut WaitClock, + observed: &mut Option, + sleep: bool, + packet: &Packet, + bound_ns: u64, ) -> Result { if !sleep { return Err(FALLBACK); } - let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; + let deadline = self.wait_deadline(clock, bound_ns)?; // Sample, then retry once: only a refusal after the sample waits. let Some(sample) = observed.take() else { *observed = Some(self.shared.renders.observe()); @@ -374,7 +386,7 @@ impl Backend { Some(lease) => Some(lease), None if known != 0 => self .admission - .try_acquire_render(self.owner, Request::of(packet)) + .try_acquire_render(self.owner, self.pairs.into(), Request::of(packet)) .ok() .flatten(), None => None, @@ -404,6 +416,8 @@ impl Backend { let mut recovery_since = None; let mut seen = None; let mut states = None; + let mut outer_wait = None; + let mut channel = WaitClock::default(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -471,7 +485,16 @@ impl Backend { } result => result?, } - match self.prepare_render(slot, packet, deferred, sleep) { + match self.prepare_render(slot, packet, deferred, sleep, &mut outer_wait) { + Err(error) + if !packet.is_published() && error == crate::g17::context::RENDER_CHANNEL_PENDING => + { + // Both of the context's render channels serve other pairs: one moves here + // once its publications retire. Like a compute queue, it frees only as this + // context's earlier work retires, so wait as long. + self.wait_render_retirement(&mut channel, &mut seen, sleep, packet, + crate::g17::preparation::QUEUE_WAIT_NS)?; + } Err(error) if !packet.is_published() && error == ENOSPC => { self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; @@ -498,6 +521,15 @@ impl Backend { self.shared .preparations .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; + } else if let Some((domain, observed)) = outer_wait.take() { + // No unclaimed room on the shared work ring: consumption or a + // returned claim frees it. + if !sleep { + return Err(FALLBACK); + } + let deadline = self.wait_deadline(&mut busy, super::super::WORK_RING_WAIT_NS)?; + self.shared.queue_events(); + self.wait_domain(&domain, observed, deadline, packet)?; } else if error == EBUSY { self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; } else if !sleep { @@ -513,7 +545,7 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + let admission = match self.admission.try_acquire_render(self.owner, self.pairs.into(), Request::of(packet)) { Ok(Some(lease)) => lease, _ => return false, }; @@ -544,6 +576,7 @@ impl Backend { packet.abandon_publication(); return false; } + self.release_outer(crate::g17::job::Engine::Render); let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -568,9 +601,22 @@ impl Registry { .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } - if dependencies.render_count != 0 - && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } - if pass.fragment_barriers) + let staged = matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.stage_prefixes); + if staged { + let tiling = self.render_prefix_at(owner, packet, packet.order.tiling_wait_through[0])?; + for dependency in tiling.as_slice() { + *dependencies + .render_tiling + .get_mut(dependencies.render_tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_tiling_count += 1; + } + } + if staged + || (dependencies.render_count != 0 + && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.fragment_barriers)) { let tiling = self.tiling_prefix(owner, packet)?; for dependency in tiling.as_slice() { @@ -585,9 +631,20 @@ impl Registry { dependencies.compute = queue.dependency( owner, packet.completion.status(), - packet.order.wait_through[1], + packet.order.tiling_wait_through[1], state, )?; + if packet.order.tiling_wait_through[1] != packet.order.wait_through[1] { + let whole = queue.dependency( + owner, + packet.completion.status(), + packet.order.wait_through[1], + state, + )?; + if whole != dependencies.compute { + dependencies.fragment_compute = whole; + } + } } Ok(dependencies) } @@ -686,6 +743,7 @@ impl crate::g17::Firmware { owner, slot, footprint, + packet.order.wait_through[0], &mut dependencies.hazards, )?; } @@ -700,7 +758,14 @@ impl crate::g17::Firmware { } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); - let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + // An M5 PB token is bound to the hardware context and PB state it names. + let buffer_context = pair.m5_pb_state_va().map(|pb_va| (packet.context.id(), pb_va)); + let (buffer_id, buffer_token) = match buffer_context { + Some((context, pb_va)) => { + self.queues.buffers.acquire_owner_bound(buffer_id, context, pb_va)? + } + None => self.queues.buffers.acquire_owner(buffer_id)?, + }; let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { Ok(preparation) => preparation, Err(error) => { @@ -711,6 +776,7 @@ impl crate::g17::Firmware { let lease = PublicationLease { free_list_generation: preparation.generation, buffer_token, + buffer_context: buffer_context.map(|(context, _)| context), }; let result = self.with_render(usize::from(slot), |pair, host| { pair.publish( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index fffa6d28dd289..5b8379078b151 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -16,7 +16,21 @@ impl RenderState { self.pools.observe(init) } + /// Nothing of `context` still runs on its render pairs, so a kill of it may be published. + /// A pair whose commands all ran unstamped (`Pair::executed_for_kill`) counts as idle: only + /// that kill releases it. pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + self.idle_for(context, false) + } + + /// As [`Self::teardown_idle`], for releasing `context`'s scheduler: a pair whose commands ran + /// unstamped counts only once a kill of the context has been published, since until then + /// the firmware still links that work. + pub(in crate::g17::runtime) fn release_idle(&self, context: &Arc) -> Result { + self.idle_for(context, true) + } + + fn idle_for(&self, context: &Arc, release: bool) -> Result { // A pair that changed owner keeps running this scheduler's queues until // the new owner's first command leaves it, whichever context owns it now. let scheduler = Some(context.scheduler_va()); @@ -38,7 +52,9 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { + if !pair.idle_for_context(context)? + && !((!release || context.kill_published()) && pair.executed_for_kill(context)) + { return Ok(false); } } @@ -77,7 +93,13 @@ impl RenderState { } let mut index = 0; while index < self.retired.len() { - if self.retired[index].buffer_id().is_some() { + // An M5 pool replaced by a same-context rebind stays until + // firmware acknowledged its release. + let pool = self.retired[index].pool(); + if self.retired[index].buffer_id().is_some() + || (self.retired[index].wait_for_pool_release() + && !self.pools.released(u32::from(pool.id()), pool.control_va())) + { index += 1; continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 88252103b676b..cbaf5c6e64251 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -7,7 +7,7 @@ use crate::g17::{ buffer::BufferIds, channel::Rings, context::Context, - fw::channels::{ControlRecord, SchedulerStateRelease}, + fw::channels::ControlRecord, initdata::InitData, queue::render::RetiredBinding, recovery::Memory as _, @@ -111,7 +111,7 @@ struct Service<'a> { impl Host for Service<'_> { fn render_idle(&mut self, context: &Arc) -> Result { - self.render.teardown_idle(context) + self.render.release_idle(context) } fn compute_idle(&self, context: &Arc) -> bool { self.compute @@ -126,13 +126,10 @@ impl Host for Service<'_> { fn control_backpressured(&self) -> bool { self.render.control_backpressured() } - fn publish(&mut self, record: &SchedulerStateRelease) -> Result { + fn publish_release(&mut self, page_va: u64, id: [u8; 4]) -> Result { Rings::publish_control( self.init, - &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( - record.page_va, - record.id, - )), + &ControlRecord::SchedulerStateRelease(self.init.scheduler_release(page_va, id)), ) } fn notify(&mut self) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index 75df8ad0480ae..3b9edc279786c 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -78,6 +78,11 @@ impl VmStatus { self.recovery_guilty.load(Ordering::Acquire) } + /// The client's VM handle is gone (`release_mirror`): the client submits nothing more. + pub(crate) fn client_closed(&self) -> bool { + self.client_closed.load(Ordering::Acquire) + } + /// Record permanent admission failure and report it to the client. #[track_caller] pub(crate) fn record(&self, error: Error) { diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index fa2a72a77f060..39208dc7b3f34 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -4,7 +4,7 @@ //! private-memory release and control-ring consumption are separate ownership witnesses. //! The device mutex serializes this state with the Registry's displaced graph ownership. -use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; +use super::{context::Context, freelist::control_consumed}; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -82,7 +82,9 @@ pub(crate) trait Host { fn compute_idle(&self, context: &Arc) -> bool; fn flists_released(&self, context: &Arc) -> bool; fn control_backpressured(&self) -> bool; - fn publish(&mut self, record: &SchedulerStateRelease) -> Result; + /// Publishes the release of the scheduler page at `page_va`, identity bytes in record + /// order, in the family's record. + fn publish_release(&mut self, page_va: u64, id: [u8; 4]) -> Result; fn notify(&mut self) -> Result; fn detach(&mut self, context: &Arc) -> Result; fn report(&mut self, context: &Arc, failure: Failure); @@ -215,11 +217,7 @@ impl Pending { } let result = (|| { let id = entry.context.release_identity()?; - let record = SchedulerStateRelease::new( - entry.context.scheduler_va(), - [id[3], id[0], id[1], id[2]], - ); - host.publish(&record) + host.publish_release(entry.context.scheduler_va(), [id[3], id[0], id[1], id[2]]) })(); let next = match result { Ok(next) => next, diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs new file mode 100644 index 0000000000000..0f5c21a77488a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU power budget governor for M5 (J714). +//! +//! The firmware limits the GPU to the host power budget word in Region C within about a second +//! (the calibrated maximum is written there at initialization). Left at that maximum under +//! sustained load, the SoC hotspot temperature computed by the SMC (about 6 C above the hottest +//! GPU die sensor while the GPU dominates) spends time above roughly 97 C, and the firmware's +//! emergency path then pins the GPU to its base performance state for 20-40 s, repeatedly. This +//! governor reads the hotspot temperature through the thermal framework once per second while the +//! GPU is busy and, once it reaches the engage temperature, lowers the budget with a PI controller +//! (plus a feed-forward on the temperature's rise) so it settles at the target, below that region. +//! +//! The controller works below the GPU's actual draw (the calibrated maximum is far above it). It +//! keeps the integral it converged to across releases, so after a release or a firmware clamp it +//! resumes from that budget; after a long period without control the memory decays to the +//! cold-start budget, never to the ceiling. It holds while the firmware's emergency clamp is +//! active and never releases then. The default target keeps the hotspot below the level at which +//! the firmware clamps even with a heat-soaked chassis. As a safety net, each emergency clamp the +//! governor sees lowers its effective target by 1 C (at most 6 C, not below 84 C), and every 10 +//! minutes without one raises it again by 1 C. It publishes the budget like the effort input, only when it changes, and +//! restores the calibrated maximum on release, when the GPU goes idle, when no sensor can be read +//! and at shutdown. Below the engage temperature nothing is written. + +use core::ffi::{c_char, c_int, c_uint, c_void, CStr}; +use core::sync::atomic::{AtomicI32, Ordering}; +use kernel::{bindings, prelude::*}; + +// fans.c: the SMC hwmon cell's fan requests and thermal zone reads, which have stubs where +// their drivers are not built. +extern "C" { + fn asahi_zone_temp(name: *const c_char, temp: *mut c_int) -> c_int; + fn asahi_fans_request(dev: *mut c_void, permille: c_uint, min_step_rpm: c_uint) -> c_int; +} + +/// SoC hotspot temperature computed by the SMC (registered as a thermal zone by the SMC driver): +/// the input of the firmware's emergency clamp. +const J714_HOTSPOT_ZONE: &CStr = c"smc-TCMb"; +/// The hottest of the 42 SMC GPU die sensors of J714 near the limit, and their offset to the +/// hotspot while the GPU dominates: the fallback input when the hotspot cannot be read. +const J714_GPU_DIE_ZONES: [&CStr; 8] = [ + c"smc-Tg08", c"smc-Tg12", c"smc-Tg0O", c"smc-Tg0X", c"smc-Tg29", c"smc-Tg2P", c"smc-Tg1U", + c"smc-Tg43", +]; +const GPU_DIE_TO_HOTSPOT_MC: i32 = 6000; +/// Where the SMC driver discovers the temperatures and the device tree names the hotspot key +/// (TCMb) as the SoC die zone's only key (J775D), that zone reads the hotspot. +const SOC_DIE_ZONE: &CStr = c"macsmc_soc_die"; + +/// Region C word holding the host GPU power budget (mW). +pub(crate) const REGION_C_POWER_BUDGET: usize = 0x88; +/// Shared power-policy word holding the GPU's current power draw (mW), written by the firmware. +pub(crate) const POLICY_GPU_POWER: usize = 0xad4; +/// Shared power-policy words of the firmware's primary performance limiter: the mask of active +/// limit reasons and the limit. The emergency reason with the limit at its floor is the clamp. +pub(crate) const POLICY_LIMITER_MASK: usize = 0x44; +pub(crate) const POLICY_LIMITER_LIMIT: usize = 0x48; +const LIMITER_EMERGENCY: u32 = 0x4; +const LIMITER_FLOOR: u32 = 100; +/// Control period while the GPU is busy. +pub(crate) const PERIOD_MS: u32 = 1000; +/// Restore the maximum after this long without a submission. +const IDLE_RESTORE_NS: u64 = 2_000_000_000; +/// Engage when the hotspot reaches this; release below it minus the hysteresis once the +/// controller has returned to its ceiling. +const ENGAGE_BELOW_TARGET_MC: i32 = 4000; +const RELEASE_HYSTERESIS_MC: i32 = 3000; +/// Lowest budget the governor sets (mW). +const FLOOR_MW: i64 = 10_000; +/// Lowest ceiling of the controller (mW): the GPU's draw at engage, but at least this. +const CEILING_MIN_MW: i64 = 40_000; +/// Proportional gain (mW per C) and integral gain (mW per C per second). Power follows the +/// budget within a second; the die settles over 30-60 s at ~1.2-1.5 C per W. +const KP_MW_PER_C: i64 = 1500; +const KI_MW_PER_C_S: i64 = 50; +/// Feed-forward on the hotspot's rise (mW per C/s), applied only while it rises, to curb the +/// overshoot when the GPU restarts from cool. +const KF_MW_PER_C_PER_S: i64 = 2000; +/// Without a converged integral, start at this fraction of the ceiling: the budget that holds +/// the target is about half the GPU's unconstrained draw. +const INITIAL_NUM: i64 = 3; +const INITIAL_DEN: i64 = 5; +/// The converged integral is kept for this long after the controller was last active, then +/// decays linearly to the cold-start integral over the decay time. +const MEMORY_HOLD_NS: u64 = 60_000_000_000; +const MEMORY_DECAY_NS: u64 = 300_000_000_000; +/// Effective target reduction per emergency clamp seen while governing, its limit, the lowest +/// effective target it leads to, and the time without a clamp after which it is reduced by one +/// step again. +const CLAMP_TARGET_STEP_MC: i32 = 1000; +const CLAMP_TARGET_MAX_MC: i32 = 6000; +const CLAMP_TARGET_FLOOR_MC: i32 = 84_000; +const CLAMP_TARGET_RECOVER_NS: u64 = 600_000_000_000; +/// Budget changes smaller than this are not published. +const STEP_MW: i64 = 250; + +/// CPU ceiling: one step lowers every cpufreq policy's maximum by this share of its hardware +/// maximum, down to the floor share. +const CPU_STEP_PERCENT: u32 = 4; +const CPU_FLOOR_PERCENT: u32 = 60; +/// Minimum time between CPU ceiling steps; the hotspot and the GPU draw follow within it. +const CPU_DWELL_NS: u64 = 2_000_000_000; +/// The GPU uses its budget when it draws at least this share of it, and leaves headroom below +/// the lower share (it then waits for the CPU). +const GPU_BOUND_PERCENT: i64 = 90; +const GPU_SLACK_PERCENT: i64 = 70; +/// The CPU's own heat: the hotspot this far above the target while the GPU budget is within the +/// slack of its floor. +const CPU_HEAT_MARGIN_MC: i32 = 2000; +const CPU_HEAT_FLOOR_SLACK_MW: i64 = 1000; + +fn enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 1 != 0 +} +fn cpu_balance_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 4 != 0 +} +fn fan_boost_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 8 != 0 +} +/// The proportional fan curve is on unless bit 4 (16) of m5_thermal_governor opts out. +fn fan_curve_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 16 == 0 +} +pub(super) fn requested() -> bool { + enabled() +} +fn target_mc() -> i32 { + let parameter = crate::module_parameters::m5_thermal_target_mc.value(); + i32::try_from(parameter).unwrap_or(90_000).clamp(70_000, 104_000) +} + +/// Whether the firmware's primary limiter words describe the emergency clamp. +pub(crate) fn is_emergency_clamp(mask: u32, limit: u32) -> bool { + mask & LIMITER_EMERGENCY != 0 && limit <= LIMITER_FLOOR +} + +/// Pins the sensor provider and serializes reads with its probe and removal. +/// Its thermal zones are managed by the hwmon device registered during probe; +/// removal releases that device's resources while holding the provider lock. +struct SensorProvider(*mut bindings::device); + +impl SensorProvider { + fn acquire() -> Option { + // SAFETY: the bus and name have static lifetimes. The bus iterator + // returns an owned device reference on success. + let device = unsafe { + bindings::bus_find_device( + &bindings::platform_bus_type, + core::ptr::null_mut(), + c"macsmc-hwmon".as_ptr().cast(), + Some(bindings::device_match_name), + ) + }; + if device.is_null() { + return None; + } + // SAFETY: the owned reference keeps the device and mutex alive. No + // GPU driver lock is held during acquisition or sensor reads. + unsafe { bindings::mutex_lock(&raw mut (*device).mutex) }; + let provider = Self(device); + // SAFETY: the provider's lock protects binding and unbinding. + if unsafe { (*device).driver.is_null() } { + return None; + } + Some(provider) + } +} + +impl Drop for SensorProvider { + fn drop(&mut self) { + // SAFETY: acquire obtained this reference and locked this mutex on + // the current thread; the guard never leaves the sensor-read call. + unsafe { + bindings::mutex_unlock(&raw mut (*self.0).mutex); + bindings::put_device(self.0); + } + } +} + +/// The provider guard prevents its managed zones from disappearing during +/// lookup and read. Do not retain a zone pointer after dropping the guard. +fn zone_temperature_mc(_provider: &SensorProvider, name: &CStr) -> Option { + let mut temp: c_int = 0; + // SAFETY: name is a NUL-terminated string that outlives the call, and temp is a valid + // destination; the zone is looked up and read under the provider guard. + (unsafe { asahi_zone_temp(name.as_ptr(), &mut temp) } == 0) + .then_some(temp) + .filter(|temp| (0..=125_000).contains(temp)) +} + +/// SoC hotspot temperature in milli-C (the hottest GPU die sensor plus its offset when the hotspot +/// cannot be read), or None when no sensor can be read. Called without driver locks held: zone +/// lookups and sensor reads may sleep. +pub(crate) fn hotspot_temperature_mc() -> Option { + let provider = SensorProvider::acquire()?; + zone_temperature_mc(&provider, J714_HOTSPOT_ZONE) + .or_else(|| zone_temperature_mc(&provider, SOC_DIE_ZONE)) + .or_else(|| { + J714_GPU_DIE_ZONES.iter().filter_map(|name| zone_temperature_mc(&provider, name)).max() + .map(|temp| temp + GPU_DIE_TO_HOTSPOT_MC) + }) +} + +/// Keeps the fans boosted at least this long once boosted, so a short release or idle period +/// (a loading screen) does not cycle them. +const FAN_HOLD_NS: u64 = 30_000_000_000; + +/// Proportional fan curve on the hotspot while the GPU is busy: on J714 the SMC's automatic mode +/// keeps the fans near their minimum under sustained GPU load on Linux. From the board's curve +/// start ([`super::fw::m5::initdata::Board::fan_curve_start_mc`]) to the governor target the fans +/// ramp from the minimum to two thirds of the range, and on to FAN_CURVE_TOP_PERMILLE at +/// FAN_CURVE_TOP_MC. +static FAN_CURVE_START_MC: AtomicI32 = + AtomicI32::new(super::fw::m5::initdata::J714.fan_curve_start_mc); +const FAN_CURVE_TOP_MC: i32 = 98_000; +const FAN_CURVE_TARGET_PERMILLE: i32 = 660; +const FAN_CURVE_TOP_PERMILLE: i32 = 800; +/// Below this the curve releases the fans to automatic mode (after FAN_HOLD_NS). +const FAN_CURVE_RELEASE_MC: i32 = 82_000; +/// A new target is written only when it moves at least this far. +const FAN_CURVE_STEP_RPM: u32 = 150; + +/// Selects the board's thermal settings at probe; J714's until then. +pub(crate) fn select_board(board: &super::fw::m5::initdata::Board) { + FAN_CURVE_START_MC.store(board.fan_curve_start_mc, Ordering::Relaxed); +} + +/// The permille of the fan range the curve asks for at hotspot `temp_mc`. +fn fan_curve_permille(temp_mc: i32, target_mc: i32) -> i32 { + let start_mc = FAN_CURVE_START_MC.load(Ordering::Relaxed); + let target_mc = target_mc.max(start_mc + 1000); + if temp_mc <= start_mc { + 0 + } else if temp_mc <= target_mc { + (temp_mc - start_mc) * FAN_CURVE_TARGET_PERMILLE / (target_mc - start_mc) + } else { + let top = FAN_CURVE_TOP_MC.max(target_mc + 1000); + (FAN_CURVE_TARGET_PERMILLE + (temp_mc.min(top) - target_mc) + * (FAN_CURVE_TOP_PERMILLE - FAN_CURVE_TARGET_PERMILLE) / (top - target_mc)) + .min(FAN_CURVE_TOP_PERMILLE) + } +} + +/// Sets every fan the SMC hwmon cell lets a kernel policy set to `permille` of its +/// minimum-to-maximum range, or hands them back to the SMC for 0, under the provider lock (which +/// serializes with its unbinding). A fan whose target is within `min_step_rpm` of the new one +/// keeps it. Returns the first fan's target rpm, 0 once handed back. +fn request_fans(permille: u32, min_step_rpm: u32) -> Option { + let provider = SensorProvider::acquire()?; + // SAFETY: the provider lock keeps the device bound, as the request requires. + let rpm = unsafe { asahi_fans_request(provider.0.cast(), permille, min_step_rpm) }; + u32::try_from(rpm).ok() +} + +/// Fan boost: while the governor limits the GPU, every fan runs at its hardware maximum instead of +/// the SMC's automatic curve, which settles near two thirds of the maximum at the governor's +/// target. +/// Released (automatic mode) when the governor releases or the GPU goes idle, after a hold. +pub(crate) struct FanBoost { + boosted: bool, + since_ns: u64, + /// The fan target the curve last wrote (rpm), 0 while the fans are automatic. + curve_rpm: u32, +} + +impl FanBoost { + const fn new() -> Self { + Self { boosted: false, since_ns: 0, curve_rpm: 0 } + } + + /// Puts every fan in manual mode at `permille` of its minimum-to-maximum range, or back in + /// automatic mode for `None`. Returns the first fan's target rpm, 0 for automatic. + fn set_curve(&mut self, permille: Option) -> Option { + request_fans(permille.map_or(0, |permille| permille.clamp(1, 1000) as u32), 0) + } + + /// One governor step of the proportional curve: `busy` while the governor runs a busy GPU. + fn apply_curve(&mut self, busy: bool, temp_mc: Option, target_mc: i32, now_ns: u64) { + let permille = temp_mc.filter(|_| busy).map(|t| fan_curve_permille(t, target_mc)).unwrap_or(0); + if permille > 0 { + self.since_ns = now_ns; + // The target rpm moves with the range only when it moved enough. + if let Some(rpm) = self.set_curve_if_moved(permille) { + self.curve_rpm = rpm; + } + } else if self.curve_rpm != 0 + && (!busy || temp_mc.is_none_or(|t| t < FAN_CURVE_RELEASE_MC)) + && now_ns.saturating_sub(self.since_ns) >= FAN_HOLD_NS + && self.set_curve(None).is_some() + { + self.curve_rpm = 0; + } + } + + fn set_curve_if_moved(&mut self, permille: i32) -> Option { + // A fan already on the curve keeps its target while the new one is within a step. + let step = if self.curve_rpm != 0 { FAN_CURVE_STEP_RPM } else { 0 }; + request_fans(permille.clamp(1, 1000) as u32, step) + } + + fn set(&mut self, boost: bool) -> bool { + request_fans(if boost { 1000 } else { 0 }, 0).is_some() + } + + /// Applies one governor step's request. Called from the governor work without driver locks: + /// SMC accesses sleep. + fn apply(&mut self, want: bool, temp_mc: Option, now_ns: u64) { + if !fan_boost_enabled() { + if fan_curve_enabled() { + self.apply_curve(want, temp_mc, target_mc(), now_ns); + } else if self.curve_rpm != 0 && self.set_curve(None).is_some() { + self.curve_rpm = 0; + } + } + let want = want && fan_boost_enabled(); + if want { + self.since_ns = now_ns; + } + if want && !self.boosted { + self.boosted = self.set(true); + } else if !want && self.boosted + && (!fan_boost_enabled() || now_ns.saturating_sub(self.since_ns) >= FAN_HOLD_NS) + { + if self.set(false) { + self.boosted = false; + } + } + } + + pub(crate) fn release(&mut self) { + if self.boosted && self.set(false) { + self.boosted = false; + } + if self.curve_rpm != 0 && self.set_curve(None).is_some() { + self.curve_rpm = 0; + } + } + + pub(crate) fn boosted(&self) -> bool { + self.boosted || self.curve_rpm != 0 + } +} + +impl Drop for FanBoost { + fn drop(&mut self) { + self.release(); + } +} + +/// The host's thermal actuators besides the GPU budget, applied outside the device lock. +pub(crate) struct Actuators { + pub(crate) cpu: CpuCeiling, + pub(crate) fans: FanBoost, +} + +impl Actuators { + pub(crate) const fn new() -> Self { + Self { cpu: CpuCeiling::new(), fans: FanBoost::new() } + } + + pub(crate) fn apply(&mut self, advice: CpuAdvice, fans: bool, temp_mc: Option, now_ns: u64) { + self.cpu.apply(advice, now_ns); + self.fans.apply(fans, temp_mc, now_ns); + } + + pub(crate) fn release(&mut self) { + self.cpu.release(); + self.fans.release(); + } +} + +/// What the CPU ceiling should do after a governor step. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum CpuAdvice { + /// Keep the current ceiling. + Hold, + /// The GPU draws its whole thermally limited budget: give it the CPU's share of the heat. + Lower, + /// The GPU is not limited by its budget (or waits for the CPU): give the CPU a step back. + Raise, + /// Not governing: drop the ceiling. + Release, +} + +/// One cpufreq policy's maximum-frequency request. +struct CpuRequest { + policy: *mut bindings::cpufreq_policy, + request: KBox, + max_khz: u32, +} + +// SAFETY: the policy reference and the request are only used from the governor work, under the +// ceiling's lock. +unsafe impl Send for CpuRequest {} + +/// Shares the SoC's heat between the CPU and the GPU while the governor limits the GPU: when the +/// GPU is budget-bound it lowers the maximum +/// frequency of every CPU cluster one step at a time, and gives the steps back once the GPU no +/// longer uses its budget (it then waits for the CPU) or is no longer limited. The requests are +/// ordinary frequency QoS maxima, combined with any other limit on the policies. +pub(crate) struct CpuCeiling { + requests: KVec, + level: u32, + last_change_ns: u64, +} + +impl CpuCeiling { + pub(crate) const fn new() -> Self { + Self { requests: KVec::new(), level: 0, last_change_ns: 0 } + } + + fn max_level() -> u32 { + (100 - CPU_FLOOR_PERCENT) / CPU_STEP_PERCENT + } + + fn limit_khz(max_khz: u32, level: u32) -> i32 { + let share = 100 - level.min(Self::max_level()) * CPU_STEP_PERCENT; + (u64::from(max_khz) * u64::from(share) / 100).min(i32::MAX as u64) as i32 + } + + /// Takes a request on every policy (each policy once, through its first CPU). + fn acquire(&mut self) -> bool { + if !self.requests.is_empty() { + return true; + } + // SAFETY: nr_cpu_ids is set once during early boot. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: cpufreq_cpu_get accepts any CPU number and returns a counted reference or + // null. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + // SAFETY: the reference keeps the policy alive. + let (first, max_khz) = unsafe { ((*policy).cpu, (*policy).cpuinfo.max_freq) }; + let request = if first == cpu && max_khz != 0 { + // The zeroed (default) request is the unused state freq_qos_add_request expects. + KBox::new(bindings::freq_qos_request::default(), GFP_KERNEL).ok() + } else { + None + }; + let Some(mut request) = request.filter(|_| self.requests.reserve(1, GFP_KERNEL).is_ok()) + else { + // SAFETY: drops the reference taken above. + unsafe { bindings::cpufreq_cpu_put(policy) }; + continue; + }; + // SAFETY: the policy outlives the request (the reference is held until removal) and + // the request is in its unused state, at a stable address in its box. + let added = unsafe { + bindings::freq_qos_add_request( + &raw mut (*policy).constraints, + &mut *request, + bindings::freq_qos_req_type_FREQ_QOS_MAX, + Self::limit_khz(max_khz, 0), + ) + }; + if added < 0 { + // SAFETY: drops the reference taken above. + unsafe { bindings::cpufreq_cpu_put(policy) }; + continue; + } + // Capacity was reserved above, so this cannot fail. + let _ = self.requests.push(CpuRequest { policy, request, max_khz }, GFP_KERNEL); + } + !self.requests.is_empty() + } + + /// Removes every request and the policy references. + pub(crate) fn release(&mut self) { + for mut entry in self.requests.drain_all() { + // SAFETY: the request was added in acquire and the policy reference is still held. + unsafe { + bindings::freq_qos_remove_request(&mut *entry.request); + bindings::cpufreq_cpu_put(entry.policy); + } + } + self.level = 0; + } + + fn publish(&mut self) { + for entry in self.requests.iter_mut() { + let limit = Self::limit_khz(entry.max_khz, self.level); + // SAFETY: the request was added in acquire and stays valid until release. + unsafe { bindings::freq_qos_update_request(&mut *entry.request, limit) }; + } + } + + /// Applies one governor step's advice. Called from the governor work without driver locks: + /// frequency QoS updates may sleep. + pub(crate) fn apply(&mut self, advice: CpuAdvice, now_ns: u64) { + if !cpu_balance_enabled() || advice == CpuAdvice::Release { + if !self.requests.is_empty() { + self.release(); + } + return; + } + if now_ns.saturating_sub(self.last_change_ns) < CPU_DWELL_NS { + return; + } + match advice { + CpuAdvice::Lower if self.level < Self::max_level() => { + if !self.acquire() { + return; + } + self.level += 1; + } + CpuAdvice::Raise if self.level > 0 => { + self.level -= 1; + if self.level == 0 { + self.release(); + self.last_change_ns = now_ns; + return; + } + } + _ => return, + } + self.last_change_ns = now_ns; + self.publish(); + } +} + +impl Drop for CpuCeiling { + fn drop(&mut self) { + self.release(); + } +} + +pub(crate) struct Governor { + maximum_mw: u32, + budget_mw: u32, + integral_mw: i64, + /// Upper limit of the controller while active: the GPU's draw at engage. + ceiling_mw: i64, + /// Integral the controller last converged to (output inside its range), kept across releases. + converged_mw: i64, + active: bool, + last_tick_ns: u64, + last_activity_ns: u64, + /// Last tick at which the controller was active, for the memory decay. + last_active_ns: u64, + last_temp_mc: Option, + /// Effective target reduction learnt from emergency clamps, and when it last changed. + target_offset_mc: i32, + target_offset_ns: u64, + was_clamped: bool, +} + +impl Governor { + pub(crate) const fn new() -> Self { + Self { maximum_mw: 0, budget_mw: 0, integral_mw: 0, ceiling_mw: 0, converged_mw: 0, active: false, + last_tick_ns: 0, last_activity_ns: 0, last_active_ns: 0, last_temp_mc: None, target_offset_mc: 0, + target_offset_ns: 0, was_clamped: false } + } + + /// The firmware accepted a submission; returns whether the control loop should run. + pub(crate) fn note_activity(&mut self, now_ns: u64) -> bool { + self.last_activity_ns = now_ns; + requested() || self.budget_mw != self.maximum_mw + } + + /// Learn the calibrated maximum from the budget word before the first change. + pub(crate) fn set_maximum(&mut self, current_mw: u32) { + if self.maximum_mw == 0 && i64::from(current_mw) >= FLOOR_MW { + self.maximum_mw = current_mw; + self.budget_mw = current_mw; + } + } + + /// The budget to publish while stopping, if it differs from the maximum. + pub(crate) fn restore(&mut self) -> Option { + self.active = false; + self.integral_mw = 0; + self.converged_mw = 0; + self.target_offset_mc = 0; + if self.maximum_mw == 0 || self.budget_mw == self.maximum_mw { + return None; + } + self.budget_mw = self.maximum_mw; + Some(self.budget_mw) + } + + /// Integral to engage with: the converged one while it is fresh, decaying to the cold-start + /// integral (a fraction of the ceiling) after a long period without control. + fn resume_integral(&self, now_ns: u64) -> i64 { + let initial = self.ceiling_mw * INITIAL_NUM / INITIAL_DEN; + if self.converged_mw == 0 { + return initial; + } + let kept = self.converged_mw.min(self.ceiling_mw); + let cool_ns = now_ns.saturating_sub(self.last_active_ns); + if cool_ns <= MEMORY_HOLD_NS { + return kept; + } + let decay_ns = (cool_ns - MEMORY_HOLD_NS).min(MEMORY_DECAY_NS); + kept + (initial - kept) * (decay_ns / 1_000_000) as i64 / (MEMORY_DECAY_NS / 1_000_000) as i64 + } + + /// Learn from the firmware's emergency clamp: lower the effective target by a step on each + /// clamp seen while governing, raise it again after a long period without one. + fn adapt_target(&mut self, clamped: bool, now_ns: u64) { + if clamped && !self.was_clamped && self.active { + let limit = (target_mc() - CLAMP_TARGET_FLOOR_MC).clamp(0, CLAMP_TARGET_MAX_MC); + self.target_offset_mc = (self.target_offset_mc + CLAMP_TARGET_STEP_MC).min(limit); + self.target_offset_ns = now_ns; + } else if !clamped && self.target_offset_mc > 0 + && now_ns.saturating_sub(self.target_offset_ns) >= CLAMP_TARGET_RECOVER_NS + { + self.target_offset_mc -= CLAMP_TARGET_STEP_MC; + self.target_offset_ns = now_ns; + } + self.was_clamped = clamped; + } + + /// How the CPU ceiling should follow the last step, from the GPU's draw (mW) against the + /// budget it was given. While governing, a GPU that draws its whole budget below its ceiling + /// is limited by heat the CPU shares; one that leaves headroom waits for the CPU. When the + /// hotspot stays above the target with the GPU budget at its floor, the heat is the CPU's + /// and only the CPU can keep the firmware from clamping the GPU. + pub(crate) fn cpu_advice(&self, power_mw: u32, now_ns: u64) -> CpuAdvice { + if self.maximum_mw == 0 || now_ns.saturating_sub(self.last_activity_ns) >= IDLE_RESTORE_NS { + return CpuAdvice::Release; + } + if !self.active { + // Released by the controller: hand the steps back one at a time, so the CPU's heat + // returns gradually and does not re-engage the governor at once. + return CpuAdvice::Raise; + } + let budget = i64::from(self.budget_mw); + let power = i64::from(power_mw); + let target = target_mc() - self.target_offset_mc; + let temp = self.last_temp_mc.unwrap_or(target); + if temp >= target + CPU_HEAT_MARGIN_MC + && budget <= FLOOR_MW.min(self.ceiling_mw) + CPU_HEAT_FLOOR_SLACK_MW + { + CpuAdvice::Lower + } else if budget < self.ceiling_mw && power * 100 >= budget * GPU_BOUND_PERCENT { + CpuAdvice::Lower + } else if budget >= self.ceiling_mw + || (power * 100 < budget * GPU_SLACK_PERCENT && temp < target) + { + // A GPU waiting for the CPU gets it back only below the target: above it, the + // controller is still taking heat out. + CpuAdvice::Raise + } else { + CpuAdvice::Hold + } + } + + /// Whether the fans should be boosted: while the controller governs a busy GPU. + pub(crate) fn fans_wanted(&self, now_ns: u64) -> bool { + self.maximum_mw != 0 && self.active + && now_ns.saturating_sub(self.last_activity_ns) < IDLE_RESTORE_NS + } + + /// One control step with the hotspot temperature, the GPU's current draw (mW) and whether the + /// firmware's emergency clamp is active. Returns the budget to publish (if it changed) and + /// whether to run again. + pub(crate) fn tick(&mut self, temp_mc: Option, power_mw: u32, clamped: bool, now_ns: u64) + -> (Option, bool) + { + let maximum = i64::from(self.maximum_mw); + let idle = now_ns.saturating_sub(self.last_activity_ns) >= IDLE_RESTORE_NS; + let dt_ns = now_ns.saturating_sub(self.last_tick_ns).min(2 * IDLE_RESTORE_NS); + self.last_tick_ns = now_ns; + let previous_mc = core::mem::replace(&mut self.last_temp_mc, temp_mc); + if enabled() { + self.adapt_target(clamped, now_ns); + } + let target = target_mc() - self.target_offset_mc; + let desired = match temp_mc { + _ if maximum == 0 => return (None, false), + None => { + self.active = false; + maximum + } + _ if idle || !enabled() => { + self.active = false; + maximum + } + Some(temp) => { + let error_mc = i64::from(target - temp); + let proportional = KP_MW_PER_C * error_mc / 1000; + let rise_mc_per_s = match previous_mc { + Some(previous) if temp > previous && dt_ns >= 1_000_000 => + i64::from(temp - previous) * 1000 / (dt_ns / 1_000_000) as i64, + _ => 0, + }; + let feed_forward = KF_MW_PER_C_PER_S * rise_mc_per_s / 1000; + if !self.active && temp >= target - ENGAGE_BELOW_TARGET_MC { + self.active = true; + self.ceiling_mw = i64::from(power_mw).max(CEILING_MIN_MW).clamp(FLOOR_MW, maximum); + self.integral_mw = self.resume_integral(now_ns); + } + if self.active && clamped { + // The firmware's emergency clamp: hold the budget and the integral until it + // ends, so the GPU resumes at the converged budget. + self.last_active_ns = now_ns; + i64::from(self.budget_mw) + } else if self.active { + self.last_active_ns = now_ns; + let floor = FLOOR_MW.min(self.ceiling_mw); + let candidate = self.integral_mw + KI_MW_PER_C_S * error_mc * (dt_ns / 1_000_000) as i64 / 1_000_000; + let output = proportional + candidate - feed_forward; + // Anti-windup: integrate only while the output is inside its range; at a + // limit, back-calculate the integral so the output sits on the limit. + let output = if output > self.ceiling_mw { + self.integral_mw = self.ceiling_mw - proportional + feed_forward; + self.ceiling_mw + } else if output < floor { + self.integral_mw = floor - proportional + feed_forward; + floor + } else { + self.integral_mw = candidate; + self.converged_mw = candidate; + output + }; + if output >= self.ceiling_mw + && temp < target - ENGAGE_BELOW_TARGET_MC - RELEASE_HYSTERESIS_MC + { + // Released: back to the calibrated maximum. + self.active = false; + maximum + } else { + output + } + } else { + maximum + } + } + }; + let desired = if desired >= maximum { maximum } else { desired - desired % STEP_MW }; + let publish = (desired != i64::from(self.budget_mw)).then(|| { + self.budget_mw = desired as u32; + self.budget_mw + }); + let again = (!idle && requested()) || self.budget_mw != self.maximum_mw; + (publish, again) + } +} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index ce003e6b18196..3da5e255408b5 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -157,6 +157,47 @@ pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator bool { + use core::sync::atomic::Ordering; + let now = crate::g17::now_ns(); + if PROGRESS.swap(word, Ordering::Relaxed) != word { + LAST_PROGRESS_NS.store(now, Ordering::Relaxed); + } + match LAST_PROGRESS_NS.load(Ordering::Relaxed) { + 0 => false, + last => now.saturating_sub(last) < PROGRESS_INTERVAL_NS, + } +} + +/// Work whose context has an older node in use ahead of its own is not late: the firmware +/// lists a context's nodes by their words and flushes from the head, so this work's stamps wait +/// behind the older node until it drains. A published job behind its context's older node is +/// renewed while the firmware's progress word moves, with two limits: +/// +/// - Not once its VM has failed work (`failed`). The older node's holder may be that failed +/// work, held until the firmware drains it, and a failed blocker excuses nothing ([`decide`]). +/// - At most [`SHARED_RENEWALS`] times, counted with the job's other deferrals (`renewals`). +/// The older node's own job runs under the same allowances and started earlier, so a healthy +/// older node drains or fails within that budget; past it the job times out as any other. +/// +/// A firmware that stops progressing for the interval expires the job as before, and an older +/// node of another VM still goes through [`attribute`]. +pub(crate) fn behind_own_head(published: bool, older: Option, progressing: bool, + failed: bool, renewals: u8) -> bool { + published && older.is_some() && progressing && !failed && renewals < SHARED_RENEWALS +} + #[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; @@ -284,6 +325,19 @@ mod tests { assert!(!renew_shared(true, true, &mut renewals)); } #[test] + fn work_behind_its_own_context_is_renewed_only_while_healthy_and_bounded() { + let older = Some(7); + for renewals in 0..SHARED_RENEWALS { + assert!(behind_own_head(true, older, true, false, renewals)); + } + assert!(!behind_own_head(true, older, true, false, SHARED_RENEWALS)); + assert!(!behind_own_head(true, older, true, false, u8::MAX)); + assert!(!behind_own_head(true, older, true, true, 0)); + assert!(!behind_own_head(true, None, true, false, 0)); + assert!(!behind_own_head(true, older, false, false, 0)); + assert!(!behind_own_head(false, older, true, false, 0)); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; diff --git a/drivers/gpu/drm/asahi/g18/boot/mod.rs b/drivers/gpu/drm/asahi/g18/boot/mod.rs index 7cb051ec680f6..28717fe4a261f 100644 --- a/drivers/gpu/drm/asahi/g18/boot/mod.rs +++ b/drivers/gpu/drm/asahi/g18/boot/mod.rs @@ -171,6 +171,8 @@ pub(crate) const fn stage_runs_firmware(stage: u32) -> bool { /// The firmware initializes once per boot; the module is pinned from then on, /// so this survives unbind/rebind and refuses a second attempt. static G18_FIRMWARE_ATTEMPTED: AtomicBool = AtomicBool::new(false); +/// Set once the module is pinned for the retained session. +static G18_MODULE_RETAINED: AtomicBool = AtomicBool::new(false); /// Bit per PSD request kind written by this module instance: bit 0 sgx, /// bit 1 gfx-asc, bit 2 gfx1-asc (power-on), bit 3 restore. Each kind is /// written at most once per module load; a re-bind does not repeat it. @@ -2023,13 +2025,10 @@ fn prepare_firmware_session(pdev: &platform::Device>, probe: T8152Probe // Point of no return: from here every path retains the session, and the // module stays loaded so RTKit callbacks keep valid code until reboot. + // No ASC CPU has started yet: once module removal has begun, refuse with + // nothing started and the session freed. + crate::retain::retain_module(&G18_MODULE_RETAINED)?; G18_FIRMWARE_ATTEMPTED.store(true, Ordering::Release); - // SAFETY: THIS_MODULE is this module; the reference is never dropped. - unsafe { - bindings::__module_get( - ::THIS_MODULE.as_ptr(), - ) - }; Ok(Retained(Some(session))) } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 551e801ba2a29..60e4827957427 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -236,7 +236,7 @@ pub(crate) struct AsahiObjConfig { /// Create a new kernel-owned GEM object. pub(crate) fn new_kernel_object(dev: &AsahiDrmDevice, size: usize) -> Result { - new_kernel_object_mapped(dev, size, false) + new_kernel_object_mapped(dev, size, false, true) } /// Create a new kernel-owned GEM object whose kernel mapping is write-combined. @@ -244,7 +244,17 @@ pub(crate) fn new_kernel_object(dev: &AsahiDrmDevice, size: usize) -> Result Result { - new_kernel_object_mapped(dev, size, true) + new_kernel_object_mapped(dev, size, true, true) +} + +/// Create a new kernel-owned write-combined GEM object that is not counted as kernel memory: +/// its owner holds a user charge for it ([`memacct::Charge::user`]), and counting both would +/// take its size twice from the device limit. +pub(crate) fn new_kernel_object_wc_user_charged( + dev: &AsahiDrmDevice, + size: usize, +) -> Result { + new_kernel_object_mapped(dev, size, true, false) } /// Validate before rounding: wrapping a user size can create a much smaller @@ -259,8 +269,18 @@ pub(crate) fn checked_object_size(size: usize) -> Result { .ok_or(EOVERFLOW) } -fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { +fn new_kernel_object_mapped( + dev: &AsahiDrmDevice, + size: usize, + map_wc: bool, + kernel_charge: bool, +) -> Result { let size = checked_object_size(size)?; + let charge = if kernel_charge { + memacct::Charge::kernel(&memacct::LEDGER, size as u64) + } else { + memacct::Charge::none() + }; let gem = shmem::Object::::new( dev, size, @@ -274,7 +294,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, - charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), + charge, }, )?; diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 438d071de971c..869dbe5b39891 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -19,6 +19,7 @@ mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; +pub(crate) mod t6050; #[cfg(CONFIG_DRM_ASAHI_G18)] pub(crate) mod t8152; diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index d531e20c7e9f9..337adb07ddf84 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -33,13 +33,14 @@ pub(crate) const CONFIG: Config = Config { crash_buffers: CrashBuffers::FirmwareData, // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), windows: ®ISTER_WINDOWS, reserved_slots: &[], - perf: &PERF, + perf: Some(&PERF), opps: OPPS, // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The // rest is the policy the M4 Pro has run with; none of it was tested otherwise. @@ -49,10 +50,12 @@ pub(crate) const CONFIG: Config = Config { page_tlbi: true, handoff_lock: false, handoff_slot: 0xffff_ffff, + handoff_unk2: 0, native_gpu_permissions: true, }, - region_c: ®ION_C, + region_c: Some(®ION_C), chip: None, + m5: None, adopt_root_entries: false, fabric_bridge: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on @@ -69,7 +72,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, @@ -80,7 +84,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, @@ -91,7 +96,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 8be466f7842ab..0c949ff11caee 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -24,13 +24,14 @@ pub(crate) const CONFIG: Config = Config { secondary_root_offset: None, crash_buffers: CrashBuffers::FirmwareData, // Unverified placement; use the M4 Pro layout. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_SLOTS, - perf: &PERF, + perf: Some(&PERF), opps: OPPS, uat_quirks: UatQuirks { reserved_tables: true, @@ -38,10 +39,12 @@ pub(crate) const CONFIG: Config = Config { page_tlbi: true, handoff_lock: false, handoff_slot: 0xffff_ffff, + handoff_unk2: 0, native_gpu_permissions: true, }, - region_c: ®ION_C, + region_c: Some(®ION_C), chip: Some(&CHIP), + m5: None, adopt_root_entries: false, fabric_bridge: false, recovery_slot_keys: None, @@ -53,7 +56,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25F84 for T6041 (normalized loaded image)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x5983a, 0x5991d], + entropy: &[[0x5983a, 0x5991d]], + boot_filled: &[], text_sha256: [ 0x6d, 0xa4, 0xd4, 0x42, 0x35, 0x35, 0x14, 0x44, 0xf6, 0xec, 0x00, 0x33, 0xf1, 0x9c, 0xbf, 0x98, 0xd3, 0x43, 0x91, 0x28, 0x83, 0xe4, 0xe2, 0x94, 0xc1, 0x8b, 0xb8, 0xb8, diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs new file mode 100644 index 0000000000000..7856adc317e03 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 Pro, M5 Max and M5 Ultra GPU firmware and address-space configuration. +//! +//! The M5 firmware builds its HW data, performance tables and region C in its own graph +//! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. +use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::regs::G17Id; + +// The coprocessors request a crash log buffer at these addresses (primary first) when they start and hand +// its contents back when they crash; outside the canonical range of dynamic objects. They are the +// ADT sgx gfx-data-shared-ro-base and gfx1-data-shared-ro-base (each 0x8000), so they differ per board. +const J714_CRASH_BUFFERS: [FixedBuffer; 2] = [ + FixedBuffer { va: 0x100_025b_4000, size: 0x8000 }, + FixedBuffer { va: 0x100_0267_0000, size: 0x8000 }, +]; + +pub(crate) const CONFIG: Config = Config { + chip_id: 0x6050, + identity: Identity { + family: 0xb, + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Unknown until the board has bare-metal page tables. + secondary_root_offset: None, + crash_buffers: CrashBuffers::Fixed(J714_CRASH_BUFFERS), + profile: FirmwareProfile::M5NoBareMetalTables, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J714), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // No loaded image is admitted: the board has no bare-metal page tables yet. + firmware: crate::g17::fw::variant::Firmware::Identified(&[]), + // Two clusters of ten cores. The buffers are sized for every core slot of the board, so the + // board's cluster and slot counts admit the GPU, whichever cores are fused off. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::TWO_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// M5 Max: ID variant 4, GPU variant C, HAL generation 300 like the M5 Pro. +pub(crate) const CONFIG_MAX: Config = Config { + chip_id: 0x6050, + identity: Identity { + family: 0xb, + variant: 4, + num_dies: 1, + gpu_variant: GpuVariant::C, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Bare metal, as the M5 Ultra: both roles publish their kernel roots, the secondary's + // 0x40000 into `pagetables`, and each asks for its crash log inside `fw-data`. + secondary_root_offset: Some(0x40000), + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + // The M5 Ultra's UAT policy, which this board's firmware expects too. + uat_quirks: CONFIG_ULTRA.uat_quirks, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J716C), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // As on the M5 Ultra, the firmware in `fw-text` must be this build. The boot loader loads + // the G17C build of both roles back to back: the primary's TEXT 0x50000, then the + // secondary's 0x54000. The digest is that of the firmware's TEXT with the boot-filled ranges + // zeroed. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428), G17C", + images: &[ + crate::fwid::FirmwareBuild { + name: "26A428 for T6050 G17C, primary then secondary", + text_size: 0xa4000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0x9d918, 0x9d9fb]], + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0x9d1d8, 0x740), (0x9d918, 0x231), + ], + text_sha256: [ + 0x30, 0x4d, 0x85, 0x44, 0x66, 0x29, 0xbc, 0xdc, 0xa5, 0x95, 0x80, 0xb5, 0xff, + 0x00, 0x6d, 0xd8, 0x9f, 0x2c, 0xd8, 0xb4, 0xca, 0x60, 0x59, 0x9a, 0x0c, 0xb5, + 0x5c, 0x35, 0xec, 0xb0, 0x98, 0xf4, + ], + }, + ], + variant: &crate::g17::fw::variant::V26A428_M5, + }]), + // Four clusters of ten cores. The buffers are sized for every core slot of the board, so the + // board's cluster and slot counts admit the GPU, whichever cores are fused off. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::FOUR_BY_TEN, + deflake: crate::g17::fw::render::Deflake::FOUR_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// M5 Ultra (J775D, two dies, G17D) on bare metal: both roles publish their kernel roots, the +/// secondary's 0x40000 into `pagetables`, and each asks for its crash log inside `fw-data`. The +/// UAT tables are no-map reservations outside /memory; the rest of the UAT policy is the one +/// this board has run with: every table change written back, page-by-page TLB invalidation, and +/// the handoff values its firmware expects (current slot 0xffff, the byte after the flush records +/// 1, the lock never taken). +pub(crate) const CONFIG_ULTRA: Config = Config { + chip_id: 0x6050, + // Family 0xb (G17), variant 6 (G17D), revision 0x11. + identity: Identity { + family: 0xb, + variant: 6, + num_dies: 2, + gpu_variant: GpuVariant::D, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + secondary_root_offset: Some(0x40000), + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff, + handoff_unk2: 1, + native_gpu_permissions: false, + }, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J775D), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // On bare metal nothing vouches for the firmware the boot loader left in `fw-text`: it + // must be this build, or the GPU is refused before anything is written to it. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428)", + images: &[crate::fwid::FirmwareBuild { + // The boot loader loads both roles' TEXT back to back into `fw-text`: the primary's + // 0x50000, then the secondary's 0x58000, each with its own GKTS/ECAP entropy + // records. The digest is that of the firmware's TEXT with the boot-filled ranges + // zeroed, so a loaded copy normalizes to it on every boot. + name: "26A428 for T6050, primary then secondary", + text_size: 0xa8000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0xa1918, 0xa19fb]], + // Per role: two load fields of the image header, then the two sections the boot + // loader fills with the boot arguments (the entropy included). + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0xa11d8, 0x740), (0xa1918, 0x231), + ], + text_sha256: [ + 0xc3, 0x53, 0x54, 0x40, 0x76, 0x02, 0x48, 0x44, 0x04, 0xaa, 0x6f, 0xc9, 0x46, + 0xcd, 0xd2, 0x4c, 0xb8, 0x2c, 0x64, 0xa8, 0xbb, 0xd1, 0x07, 0xa4, 0x94, 0x79, + 0xa0, 0x8c, 0xe9, 0xcc, 0x82, 0x51, + ], + }], + variant: &crate::g17::fw::variant::V26A428_M5, + }]), + // Eight clusters of ten cores. M5 renders lay out their tiler context store themselves + // (`fw::m5::render_registers`), so the deflake layout is the M5 Pro's. As on the M5 Pro, the + // buffers are sized for all 80 core slots and the board admits the GPU by clusters and slots: + // parts with fused-off cores (64 enabled) take the same layout. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::EIGHT_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +// Each compute operand bank holds 6 KiB per core slot of the board. +const _: () = { + use crate::g17::fw::m5::initdata::{J714, J716C, J775D}; + assert!(CONFIG.core_layout.operand_state.bank_bytes() == J714.gpu_cores * 0x1800); + assert!(CONFIG_MAX.core_layout.operand_state.bank_bytes() == J716C.gpu_cores * 0x1800); + assert!(CONFIG_ULTRA.core_layout.operand_state.bank_bytes() == J775D.gpu_cores * 0x1800); +}; + +/// Every T6050 board binds the same compatible; the ID registers select the configuration. +pub(crate) fn config(id: &G17Id) -> Option<&'static Config> { + [&CONFIG, &CONFIG_MAX, &CONFIG_ULTRA].into_iter().find(|cfg| id.family == cfg.identity.family + && id.variant == cfg.identity.variant && id.num_dies == cfg.identity.num_dies) +} diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 02381c2aae344..23abb32e91ec8 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -39,19 +39,21 @@ pub(crate) const CONFIG: Config = Config { ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_REGISTER_SLOTS, - perf: &PERF, + perf: Some(&PERF), opps: OPPS, uat_quirks: crate::mmu::UatQuirks::DEFAULT, - region_c: &crate::g17::fw::initdata::REGION_C_26A428, + region_c: Some(&crate::g17::fw::initdata::REGION_C_26A428), chip: None, + m5: None, adopt_root_entries: true, fabric_bridge: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), @@ -316,7 +318,7 @@ pub(crate) mod resources { ]; pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; - pub(crate) const RESERVED_RANGES: usize = 48; + pub(crate) const RESERVED_RANGES: usize = 50; /// Compact allocations search these shared client arenas in this order. pub(crate) fn compact_ranges() -> [Range; 4] { @@ -363,6 +365,13 @@ pub(crate) mod resources { } add(AUXILIARY); add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); + // The dynamic arenas the kernel places a client's queue aliases in (kick and + // descriptor rings, render generation state, client-private compute and render + // backing) stay reserved from creation too, as the fixed windows above are: an alias + // created later is then never mapped over a user bind, and the bind check's view of the + // reserved ranges cannot change between parsing a bind and committing it. + add(super::dynamic::LOWER); + add(super::dynamic::CLIENT_LOWER); Some(ranges) } } @@ -419,10 +428,12 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - // One physical pair per logical owner leaves room for every execution - // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH, - // as many as a pair has command records (the render memory's RECORDS). - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; + // The QID budget reserves one physical pair per logical owner, which + // leaves room for every execution context in the shared 128-QID + // namespace. A second pair comes only from surplus, on G17D + // (g17::pairs). Commands queue at RENDER_DEPTH per pair, as many as a pair + // has command records (the render memory's RECORDS). + pub(crate) const RENDER_PAIRS_MAX: usize = 2; pub(crate) const RENDER_DEPTH: u8 = 32; // The firmware stores an ioctl's compute completion stamps only once all of the ioctl's // kicks finished, so finished commands hold their place until the ioctl's longest one ends. diff --git a/drivers/gpu/drm/asahi/m5/live_boot.rs b/drivers/gpu/drm/asahi/m5/live_boot.rs index 89bf0b08c7124..8f3c9f54325eb 100644 --- a/drivers/gpu/drm/asahi/m5/live_boot.rs +++ b/drivers/gpu/drm/asahi/m5/live_boot.rs @@ -279,6 +279,9 @@ fn new_uat(dev: &AsahiDrmDevice) -> Result<(Arc, [mmu::KernelMapping; Ok((uat, views)) } +/// Set once the module is pinned for an M5 session, which is retained until reboot. +static M5_MODULE_RETAINED: AtomicBool = AtomicBool::new(false); + pub(crate) struct M5Runtime { session: Option>, application: bool, @@ -321,8 +324,10 @@ impl M5Runtime { // even when firmware rejects the graph. Failed probe unwinding must // never release UAT or transports beneath a running firmware CPU. // Keep callback code resident even if a failed stop retains owners. - // This explicit RAM-boot experiment is released by reboot only. - unsafe { kernel::bindings::__module_get((*kernel::module::this_module::()).as_ptr()); } + // This explicit RAM-boot experiment is released by reboot only. No + // CPU has started yet: once module removal has begun, refuse with + // nothing started and the session freed. + crate::retain::retain_module(&M5_MODULE_RETAINED)?; Ok(Self { session: Some(session), application: false, faulted: false, application_generation: 0, application_compute_generation: 0 }) } diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index 12fc90b23d8eb..a61be841bff11 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -279,7 +279,8 @@ unsafe impl Send for Charge {} unsafe impl Sync for Charge {} impl Charge { - /// An object that is not counted (a dma-buf import). + /// An object that is not counted: a dma-buf import, or a kernel object whose owner holds a + /// user charge for it. pub(crate) fn none() -> Self { Self { kind: Kind::None, @@ -351,6 +352,33 @@ impl Charge { } } + /// Moves a user object's charge to `to`, the client now using it: its committed bytes, + /// within `to`'s limit, and its held bytes. The device totals and the process attribution + /// stay as they are. A refusal fails with `ENOMEM` and leaves the charge where it was. + /// Other kinds are attributed to no client and are left alone. + pub(crate) fn transfer(&mut self, to: &Arc) -> Result { + let bytes = self.bytes; + let held = *self.held.get_mut(); + let Kind::User { commit, .. } = &mut self.kind else { + return Ok(()); + }; + if Arc::ptr_eq(&commit.client, to) { + return Ok(()); + } + let ledger = commit.ledger; + if !try_add_within(&to.committed, bytes, ledger.client_limit.load(Ordering::Relaxed), 0) { + ledger.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + if held { + to.resident.fetch_add(bytes, Ordering::AcqRel); + sub(&commit.client.resident, bytes); + } + sub(&commit.client.committed, bytes); + commit.client = to.clone(); + Ok(()) + } + /// Records that the GPU now holds (or no longer holds) the object's backing. /// /// Idempotent: only a change of state is counted. @@ -490,6 +518,36 @@ mod tests { Ok(()) } + static MOVE: Ledger = Ledger::new(); + + #[test] + fn a_transfer_moves_the_charge_once() -> Result { + MOVE.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); + let a = Client::new()?; + let b = Client::new()?; + let mut charge = Charge::user(MOVE.admit(&a, 40 * MIB)?, None); + charge.set_held(true); + let other = MOVE.admit(&b, 30 * MIB)?; + // b is within 60 MiB only without the 40 MiB: refused, and nothing moved. + assert_eq!(charge.transfer(&b), Err(ENOMEM)); + assert_eq!((a.committed(), a.resident(), b.committed()), (40 * MIB, 40 * MIB, 30 * MIB)); + drop(other); + charge.transfer(&b)?; + assert_eq!((a.committed(), a.resident()), (0, 0)); + assert_eq!((b.committed(), b.resident()), (40 * MIB, 40 * MIB)); + // The device counts the bytes once, before and after the move. + assert_eq!(MOVE.user_committed.load(Ordering::Relaxed), 40 * MIB); + assert_eq!(MOVE.user_resident.load(Ordering::Relaxed), 40 * MIB); + assert_eq!(MOVE.kernel_resident.load(Ordering::Relaxed), 0); + // Moving to the current client changes nothing; dropping uncharges the new client. + charge.transfer(&b)?; + drop(charge); + assert_eq!((b.committed(), b.resident()), (0, 0)); + assert_eq!(MOVE.user_committed.load(Ordering::Relaxed), 0); + assert_eq!(MOVE.user_resident.load(Ordering::Relaxed), 0); + Ok(()) + } + #[test] fn imports_are_not_counted() { let charge = Charge::none(); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4cc9bf4223faf..f2c64c459d76c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -112,16 +112,20 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod layout; + mod lifetime; #[cfg(CONFIG_DRM_ASAHI_M5)] mod m5_context; mod ppl; mod residency; mod shared; +mod sparse_tile; +mod table; #[cfg(CONFIG_DRM_ASAHI_G18)] mod t8152; pub(crate) use bind::PreparedUserBindBatch; -pub(crate) use lifetime::VmJobGuard; +pub(crate) use lifetime::{RenderPoolMappings, VmJobGuard}; #[cfg(CONFIG_DRM_ASAHI_G18)] pub(crate) use t8152::G18ExecutionContext; /// The module-1134 name of the job guard G18 holds while a job uses a VM. @@ -221,6 +225,9 @@ pub(crate) struct UatQuirks { pub(crate) handoff_lock: bool, /// Handoff "current slot" written at initialization. pub(crate) handoff_slot: u32, + /// Handoff byte after the flush records written at initialization. It is 1 on T6050, where + /// it also turns the legacy flush protocol off. + pub(crate) handoff_unk2: u8, /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. pub(crate) native_gpu_permissions: bool, @@ -234,6 +241,7 @@ impl UatQuirks { page_tlbi: false, handoff_lock: true, handoff_slot: 0, + handoff_unk2: 0, native_gpu_permissions: false, }; } @@ -392,6 +400,16 @@ impl Drop for MappingMutation { } } +/// Where client-private aliases are placed: [`layout::CLIENT_PRIVATE_RANGE`] on M5, +/// outside every Mesa window; the A18 Pro's arena otherwise. +pub(crate) fn client_private_range(m5: bool) -> Range { + if m5 { + layout::CLIENT_PRIVATE_RANGE + } else { + crate::hw::t8140::dynamic::CLIENT_LOWER + } +} + /// Inner data for a Vm instance. struct VmInner { dev: driver::AsahiDevRef, @@ -405,6 +423,17 @@ struct VmInner { uat_inner: Arc, binding: Arc>, id: u64, + /// Per-VM sparse tiles (DRM_ASAHI_BIND_SPARSE_TILE): [read-only zero, + /// writable sink], allocated on first use. + sparse_tiles: [Option>; 2], +} + +/// GPUVA filled from a kernel sparse tile instead of its GEM's first page. +pub(crate) const GPUVA_SPARSE_TILE: u32 = kernel::bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS; + +/// Whether a GPUVA carries [`GPUVA_SPARSE_TILE`]. +fn is_sparse_tile(flags: GpuVaFlags) -> bool { + u32::from(flags) & GPUVA_SPARSE_TILE != 0 } /// Exec-locked access to the [`VmInner`] of a GPUVM. @@ -614,6 +643,11 @@ impl gpuvm::DriverGpuVm for VmInner { if ctx.dev.is_none() { return Err(ENODEV); } + // A live leaf in the range belongs to another mapping, such as a kernel alias in a + // driver window: refuse before writing anything, so the rollback below never clears + // that mapping's leaves. + self.page_table + .ensure_unmapped(map_addr..map_addr + map_length as u64)?; // T8152 (module 1134): GPUVM cannot see driver-owned KernelMappings, and G18 places // driver aliases in user VMs outside the VM_CREATE kernel range. Preflight the full // object range in the authoritative page table (EBUSY on any populated leaf) and @@ -622,6 +656,19 @@ impl gpuvm::DriverGpuVm for VmInner { self.page_table .prepare_map(map_addr..map_addr.checked_add(map_length as u64).ok_or(EOVERFLOW)?)?; } + if is_sparse_tile(op.flags()) { + // Kernel tile fill: the GEM only anchors the GPUVA. + let result = if one_page { + self.map_sparse_tile(iova..iova + left as u64, ctx.prot) + } else { + Err(EINVAL) + }; + if let Err(error) = result { + self.rollback_failed_map(map_addr, map_length as u64); + return Err(error); + } + left = 0; + } // Bind preparation retained the table's backing before taking the VM reservation. // Never acquire the object reservation from inside the GPUVM map callback. let segments = op.vm_bo().data().segments.lock(); @@ -803,6 +850,23 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// Fill `range` from the kernel sparse tile for `prot`: the global zero + /// tile for read-only fills, this VM's own sink for writable ones (never + /// shared, so discarded writes cannot be read by another VM). The caller + /// holds the mapping mutation guard. + fn map_sparse_tile(&mut self, range: Range, prot: Prot) -> Result { + let kind = usize::from(prot != PROT_GPU_SHARED_RO); + let tile = match &self.sparse_tiles[kind] { + Some(tile) => tile.clone(), + None => { + let tile = sparse_tile::SparseTile::new()?; + self.sparse_tiles[kind] = Some(tile.clone()); + tile + } + }; + self.page_table.map_tiled(range, &tile, prot) + } + /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { mod_dev_dbg!( @@ -954,6 +1018,21 @@ impl VmInner { self.tlbi_context_mask(iova, size, contexts); } + /// Invalidate a VA range in every TTBAT context that names this Vm's root in either half: a + /// kernel upper mapping reaches the firmware contexts and every execution context, a kernel + /// lower one the contexts that still name the kernel lower root. + fn tlbi_contexts_naming_root_in_either_half(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + + let contexts = self + .uat_inner + .lock() + .contexts_naming_root_in_either_half(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { // Complete new leaves even if no context currently names this root: // a later context can publish it without repeating the map operation. @@ -988,6 +1067,12 @@ impl VmInner { self.page_table.prepare_map(node.start()..end)?; let uat_inner = self.uat_inner.clone(); let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; + // Refuse a range holding another mapping's leaves before writing anything: the + // discard below clears the whole range. + if let Err(error) = self.page_table.ensure_unmapped(node.start()..end) { + drop(handoff); + return Err(error); + } let result = self.map_node_prepared(node, prot); let discard = if result.is_err() { // Rollback still changes live leaves and must remain under the PPL lease. @@ -1099,12 +1184,17 @@ pub(crate) struct Vm { #[cfg_attr(not(any(CONFIG_DRM_ASAHI_M5, CONFIG_DRM_ASAHI_G18)), allow(dead_code))] kernel_range: Range, mapping_epoch: Option>, + /// Created by an M5 GPU's UAT ([`Uat::is_m5`]). + m5: bool, bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, shared_bindings: Option>, status: Option>, + /// The memory accounting of the client that created this VM, which kernel memory sized + /// by that client's requests is charged to. `None` for kernel VMs. + memory: Option>, gpuvm: ARef>, dummy_obj: ARef, binding: Arc>, @@ -1121,6 +1211,14 @@ pub(crate) struct ExecutionContext { vm: Vm, } +/// One published command's claim on its context's hardware identity, held +/// until the command retires exactly. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ContextPin { + pub(crate) id: u8, + pub(crate) generation: u64, +} + struct ExecutionRoot { released: AtomicBool, inner: Arc, @@ -1137,6 +1235,17 @@ impl ExecutionContext { pub(crate) fn generation(&self) -> u8 { self.root.generation } + /// Claim the context's hardware identity for one published command until + /// it retires exactly. + pub(crate) fn pin(&self) -> Result { + Ok(ContextPin { + id: self.root.id, + generation: self.root.generation.into(), + }) + } + + pub(crate) fn unpin(&self, _pin: ContextPin) {} + pub(crate) fn vm(&self) -> &Vm { &self.vm } @@ -1459,6 +1568,23 @@ impl KernelMapping { self.0.start()..(self.0.start() + self.0.mapped_size as u64) } + /// Whether the UAT tracks which TTBAT contexts name each root (published roots), so that + /// [`Self::publish_to_contexts`] reaches every context that can walk this mapping. Under the + /// handoff the caller keeps its own invalidation. + pub(crate) fn tracks_contexts(&self) -> bool { + self.0.uat_inner.firmware != UatFirmware::Handoff + } + + /// Publishes this mapping's page-table updates to every TTBAT context whose lower or upper + /// half names the owner's root. Call once the mapping set is complete, before work naming it + /// becomes visible. + pub(crate) fn publish_to_contexts(&self) { + self.0 + .owner + .lock_inner() + .tlbi_contexts_naming_root_in_either_half(self.iova(), self.size()); + } + /// Resolves a byte in this retained mapping without creating page tables. pub(crate) fn translate_offset(&self, offset: usize) -> Result { if offset >= self.size() { @@ -1541,9 +1667,10 @@ impl KernelMapping { } let _mutation = owner.mapping_mutation(); mutated = true; + // The prefix keeps the mapping's own access and changes only its cache attribute. owner.page_table.reprot_pages( self.iova()..self.iova() + prefix as u64, - PROT_GPU_FW_PRIV_RW, + self.0.prot.as_cached(), )?; mem::sync(); mem::tlbi_all(); @@ -1937,6 +2064,19 @@ impl UatShared { }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } + + /// Returns the mask of TTBAT contexts naming the root `ttb` in their lower or upper half. + fn contexts_naming_root_in_either_half(&self, ttb: u64) -> u64 { + self.ttbs() + .iter() + .enumerate() + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + slot.ttb0.load(Ordering::Acquire) == expected + || slot.ttb1.load(Ordering::Acquire) == expected + }) + .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) + } } /// Returns the TTBAT value naming root `ttb` in context `ctx`. @@ -2099,6 +2239,8 @@ pub(crate) struct Uat { kernel_vm: Vm, kernel_lower_vm: Vm, + /// The GPU behind this UAT runs the M5 firmware's protocol ([`Uat::is_m5`]). + m5: bool, } impl Handoff { @@ -2137,7 +2279,7 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32, unk2: u8) -> Result { #[cfg(CONFIG_DRM_ASAHI_G18)] if firmware.is_no_peer() && (self.magic_fw.load(Ordering::Acquire) != 0 @@ -2148,6 +2290,7 @@ impl Handoff { } self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(slot, Ordering::Relaxed); + self.unk2.store(unk2, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -2346,6 +2489,7 @@ impl Vm { uat_inner, binding: binding.clone(), id, + sparse_tiles: [None, None], }, )?; let gpuvm = ARef::from(&*unique_gpuvm); @@ -2358,12 +2502,14 @@ impl Vm { kernel_range, gpuvm, mapping_epoch, + m5: false, bind_sequence, lifetime: None, residency: None, context_bindings: None, shared_bindings: None, status: None, + memory: None, dummy_obj: dummy_obj.gem.clone(), binding, }) @@ -2458,6 +2604,17 @@ impl Vm { } /// Status shared by every queue using this VM, if enabled by its GPU. + /// Charges kernel memory sized by this VM's client to `client` ([`Self::memory_client`]). + pub(crate) fn with_memory_client(mut self, client: Arc) -> Self { + self.memory = Some(client); + self + } + + /// The memory accounting of the client that created this VM, if it is a client VM. + pub(crate) fn memory_client(&self) -> Option<&Arc> { + self.memory.as_ref() + } + pub(crate) fn status(&self) -> Option<&Arc> { self.status.as_ref() } @@ -2960,6 +3117,13 @@ impl Vm { let transaction_uat = inner.uat_inner.clone(); let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); + // As for kernel mappings: refuse another mapping's leaves before the discard below + // could clear them. + if let Err(error) = inner.page_table.ensure_unmapped(iova..end) { + drop(mutation); + drop(handoff); + return Err(error); + } let ret = inner .page_table .map_pages(iova..end, phys as PhysicalAddr, prot, false); @@ -3422,6 +3586,7 @@ impl Uat { geometry, kernel_vm, kernel_lower_vm, + m5: false, inner, slots: slotalloc::SlotAllocator::new( UAT_USER_CTX as u32, @@ -3497,6 +3662,16 @@ impl Uat { &self.kernel_lower_vm } + /// Links the kernel lower root's tables for each range, with no mappings. A published-roots + /// firmware samples that root's top-level entries only when it starts. + pub(crate) fn prepare_lower_tables(&self, ranges: &[Range]) -> Result { + let mut inner = lock_vm(&self.kernel_lower_vm.gpuvm, None, false)?; + for range in ranges { + inner.page_table.alloc_pages(range.clone())?; + } + Ok(()) + } + /// Installed descriptor and bind contexts rooted in this user VM. A graph /// constructor checks this before changing a retained pool's mappings. pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { @@ -3704,8 +3879,16 @@ impl Uat { None => return Err(ENOSPC), }; inner.execution_contexts |= 1u64 << id; - inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); - (id, inner.context_generations[id]) + // On M5 a slot's reuse generation is never zero, which M5's context token reserves + // for none: it wraps from 255 to 1. Slots reused past 255 times (reclaimed M5 queues) + // would otherwise hand out zero. + let generation = &mut inner.context_generations[id]; + *generation = if self.m5 { + generation.checked_add(1).unwrap_or(1) + } else { + generation.wrapping_add(1) + }; + (id, *generation) }; let low = tagged_root(vm.ttb(), id); let high = tagged_root(self.kernel_vm.ttb(), id); @@ -3744,7 +3927,7 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { self.inner.check_m4()?; - let vm = Vm::new( + let mut vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, @@ -3752,10 +3935,51 @@ impl Uat { None, id, )?; + vm.m5 = self.m5; self.inner.check_m4()?; Ok(vm) } + /// The GPU behind this UAT runs the M5 firmware's protocol. + pub(crate) fn is_m5(&self) -> bool { + self.m5 + } + + /// Marks a published-roots UAT as an M5 GPU's ([`Uat::is_m5`]). + pub(crate) fn with_m5(mut self) -> Self { + self.m5 = true; + self + } + + /// Keep dynamic objects outside the M5 boot graph, its unmapped sentinel, + /// and the firmware register aperture. Client mappings have their own VA. + pub(crate) fn kernel_dynamic_range(&self) -> Range { + let mut range = self.geometry.kernel_range(); + range.start += if self.is_m5() { + 0x2_0000_0000 + } else { + crate::hw::t8140::dynamic::KERNEL_OFFSET + }; + range + } + + /// Canonical lower objects are aliased into every owning client VM. + /// Sparse VA reservation has no physical-memory cost. + pub(crate) fn kernel_lower_dynamic_range(&self) -> Range { + if self.is_m5() { + layout::CANONICAL_RANGE + } else { + crate::hw::t8140::dynamic::LOWER + } + } + + /// The MMU's arenas an M5 client VM keeps from userspace binds, even while they + /// hold nothing: the canonical arena its queues alias and the client-private one. + /// The client VM also reserves the compact window (`BlockGeometry::M5_COMPACT`). + pub(crate) fn m5_client_reserved_ranges(&self) -> [Range; 2] { + [layout::CANONICAL_RANGE, layout::CLIENT_PRIVATE_RANGE] + } + /// Returns the address space geometry of this UAT. pub(crate) fn geometry(&self) -> UatGeometry { self.geometry @@ -4025,6 +4249,7 @@ impl Uat { geometry, kernel_vm, kernel_lower_vm, + m5: false, inner, slots: slotalloc::SlotAllocator::new( UAT_USER_CTX as u32, @@ -4042,7 +4267,7 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; + inner.handoff().init(firmware, quirks.handoff_slot, quirks.handoff_unk2)?; if quirks.clean_tables { pgtable::clean_poc( core::ptr::from_ref(inner.handoff()).cast::(), diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index acce5677b9f92..a792cdd1bec08 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -15,6 +15,7 @@ pub(crate) struct PreparedUserMap { size: u64, offset: u64, single_page: bool, + sparse_tile: bool, context: context::PreparedBinding, _residency: Option, _sg_lease: kernel::drm::gem::shmem::SGLease, @@ -59,6 +60,9 @@ impl PreparedUserBindBatch { impl Vm { /// Prepares host state before taking the GPUVM exec lock or changing any leaf. + /// With `sparse_tile` (DRM_ASAHI_BIND_SPARSE_TILE) the range is filled from a kernel + /// sparse tile and `gem` only anchors it; it requires `single_page`. + #[allow(clippy::too_many_arguments)] pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -67,7 +71,11 @@ impl Vm { offset: u64, prot: Prot, single_page: bool, + sparse_tile: bool, ) -> Result { + if sparse_tile && !single_page { + return Err(EINVAL); + } gem::validate_vm_binding(gem, self)?; let residency = self.enter_residency_metadata()?; let context = self.prepare_context_binding(addr, size, offset)?; @@ -103,6 +111,7 @@ impl Vm { size, offset, single_page, + sparse_tile, context, }) } @@ -138,7 +147,12 @@ impl Vm { let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { - let (flags, repeat) = if map.single_page { + let (flags, repeat) = if map.sparse_tile { + ( + GpuVaFlags::from(GpuVaFlag::REPEAT) | GpuVaFlags::from(super::GPUVA_SPARSE_TILE), + UAT_PGSZ as u32, + ) + } else if map.single_page { (GpuVaFlag::REPEAT.into(), UAT_PGSZ as u32) } else { (GpuVaFlags::empty(), 0) diff --git a/drivers/gpu/drm/asahi/mmu/layout.rs b/drivers/gpu/drm/asahi/mmu/layout.rs new file mode 100644 index 0000000000000..6bd1a17df97a8 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/layout.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Fixed GPU VA arenas of the M5 lower address space. + +use core::ops::Range; + +/// Canonical queue backing has its own sparse arena. It must not overlap +/// the client-private arena or retain the A18 Pro's sub-2-GiB limit. +pub(super) const CANONICAL_RANGE: Range = 0x80_0000_0000..0x90_0000_0000; +/// Client-private aliases of M5 queues, placed per client VM. Like the +/// canonical arena it lies above Mesa's M5 windows (driver reservation, data +/// heap 0x40.., its read-only shadow 0x60..0x80, which Mesa binds for every +/// allocation, and the USC window 0x100..), so no user mapping can need it. +pub(super) const CLIENT_PRIVATE_RANGE: Range = 0x90_0000_0000..0xa0_0000_0000; diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 66d507c2d7694..9da7820dd92f6 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -28,6 +28,10 @@ struct PoolMappings { mappings: KVec, } +/// A graph's pool mappings moved out of a VM slot (see +/// Vm::take_render_pool_mappings). Dropping it unmaps them. +pub(crate) struct RenderPoolMappings(PoolMappings); + impl DriverMappings { fn ranges(&self) -> impl Iterator> + '_ { Iterator::chain( @@ -170,6 +174,51 @@ impl Vm { Ok(()) } + /// Move a drained graph's pool mappings for `slot` out of this VM + /// (so another graph of the same VM can install there); `put` returns them + /// if that install fails. Only the exact owner identity is taken. + pub(crate) fn take_render_pool_mappings(&self, slot: u8, id: u64) -> Option { + if id == 0 { + return None; + } + let lifetime = self.lifetime.as_ref()?; + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver)? + .render + .get_mut(usize::from(slot))? + .take_if(|pool| pool.id == id) + .map(RenderPoolMappings) + } + pub(crate) fn put_render_pool_mappings(&self, slot: u8, mappings: RenderPoolMappings) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if pool.is_some() { + return Err(EBUSY); + } + *pool = Some(mappings.0); + Ok(()) + } + + /// The graph identity whose pool mappings occupy `slot` in this + /// VM (0 when free). A second graph cannot install there until it leaves. + pub(crate) fn render_pool_owner(&self, slot: u8) -> u64 { + self.lifetime + .as_ref() + .and_then(|lifetime| { + lifetime + .driver + .lock() + .as_ref()? + .render + .get(usize::from(slot))? + .as_ref() + .map(|pool| pool.id) + }) + .unwrap_or(0) + } + pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { if id == 0 { return None; @@ -652,6 +701,31 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } + // Healthy last-job release need not close mapping/job admission when + // there is no accepted cleanup to drain. Keep every recovery, close, + // in-progress commit and failed-cleanup path on the existing drain. + // M5 only: other G17 GPUs keep the drain on every last release. + if self.vm.m5 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed + && state.commits == 0 + && state.close_ranges.is_none() + && state.unmaps.is_empty() + && state.objects.is_empty() + && self.vm.status.as_ref().is_some_and(|status| { + status.get() == 0 && status.reported_error() == 0 + }) + { + drop(state); + // The C deferred list is independent of the queues above. Its + // GEM puts may sleep/reenter; run after unlocking, while this + // job guard still retains the VM and its residency lease. + self.vm.bo_deferred_cleanup(); + return; + } state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 1df852184b31d..a83068a5c9e9f 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -616,6 +616,14 @@ impl Vm { inner.page_table.discard_partial_map(map.addr..end)?; inner.tlbi_contexts_naming_root(map.addr, map.range as usize); let prot = map.inner.prot.ok_or(EINVAL)?; + if super::is_sparse_tile(map.flags) { + // A tile fill has no GEM pages to restore: reinstall it. + let _change = inner.mapping_mutation(); + inner.map_sparse_tile(map.addr..end, prot)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + inner.set_invalidated(map.addr, map.range, false)?; + return Ok(()); + } let repeat = map.flags.contains(GpuVaFlag::REPEAT); let mut skip = map .offset diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index fec7a6b61ad8f..a5ea381434d44 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -108,11 +108,12 @@ impl Vm { pub(crate) fn render_footprint( &self, attachments: &Attachments, + whole: bool, ) -> Result>> { if attachments.as_slice().is_empty() { return Ok(None); } - let mut footprint = Footprint::new(self.id); + let mut footprint = Footprint::new(self.id, whole); let bindings = self.shared_bindings.as_ref().and_then(|shared| { (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) }); diff --git a/drivers/gpu/drm/asahi/mmu/sparse_tile.rs b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs new file mode 100644 index 0000000000000..dc4a5459eff10 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Kernel-owned scratch for sparse fills (DRM_ASAHI_BIND_SPARSE_TILE). + +use kernel::{prelude::*, sync::Arc}; + +/// A zeroed, physically contiguous kernel allocation. The page tables repeat +/// its first page across a filled range, as a SINGLE_PAGE bind repeats its +/// GEM's first page. +pub(super) struct SparseTile { + page: *mut kernel::bindings::page, + physical: u64, +} + +// SAFETY: The pages are exclusively owned until Drop; only their physical +// address is shared, and it is immutable. +unsafe impl Send for SparseTile {} +// SAFETY: See Send. No interior mutability. +unsafe impl Sync for SparseTile {} + +impl SparseTile { + /// One UAT page, the only part the page tables read: the order of the kernel pages that + /// make up a UAT page, naturally aligned to it. + const ORDER: u32 = (crate::pgtable::UAT_PGSZ / kernel::page::PAGE_SIZE).trailing_zeros(); + + pub(super) fn new() -> Result> { + // SAFETY: A plain zeroed higher-order page allocation. + let page = unsafe { kernel::bindings::alloc_pages( + kernel::bindings::GFP_KERNEL | kernel::bindings::__GFP_ZERO, Self::ORDER) }; + if page.is_null() { return Err(ENOMEM); } + // SAFETY: page is a live allocation returned above. + let physical = unsafe { kernel::bindings::page_to_phys(page) } as u64; + let tile = Arc::new(Self { page, physical }, GFP_KERNEL); + if tile.is_err() { + // SAFETY: The allocation above was not published anywhere. + unsafe { kernel::bindings::__free_pages(page, Self::ORDER) }; + } + Ok(tile?) + } + + pub(super) fn physical(&self) -> u64 { self.physical } +} + +impl Drop for SparseTile { + fn drop(&mut self) { + // SAFETY: The VM that maps the tile owns it and drops it after its + // page tables. + unsafe { kernel::bindings::__free_pages(self.page, Self::ORDER) }; + } +} diff --git a/drivers/gpu/drm/asahi/mmu/table.rs b/drivers/gpu/drm/asahi/mmu/table.rs new file mode 100644 index 0000000000000..53d6c30273b5e --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/table.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Page-table operations the M5 paths use beyond the common ones. + +use super::{sparse_tile::SparseTile, Prot, UatPageTable}; +use core::ops::Range; +use kernel::{addr::PhysicalAddr, prelude::*, sync::Arc}; + +impl UatPageTable { + pub(super) fn visit_runs( + &self, + range: Range, + visit: impl FnMut(u64, u64, u8) -> Result, + ) -> Result { + let _ = (range, visit); + Err(EOPNOTSUPP) + } + + /// Fill `range` from a sparse tile: its first (zeroed) page repeats across + /// the range, as a SINGLE_PAGE bind repeats its GEM's first page. + pub(super) fn map_tiled(&mut self, range: Range, tile: &Arc, prot: Prot) -> Result { + self.map_pages(range, tile.physical() as PhysicalAddr, prot, true) + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 0f915607cb1f4..f1980e6c4f4f9 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -289,7 +289,7 @@ const HIGH_BITS_PXN: u16 = 1 << 1; const HIGH_BITS_UXN: u16 = 1 << 2; const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct Prot { memattr: u8, ap: u8, @@ -417,6 +417,10 @@ impl Prot { pub(crate) const fn as_uncached(&self) -> Self { self.memattr(MEMATTR_UNCACHED) } + + pub(crate) const fn as_cached(&self) -> Self { + self.memattr(MEMATTR_CACHED) + } } impl Default for Prot { @@ -669,6 +673,7 @@ impl UatPageTable { mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; + let firmware_tree = reserved.as_deref(); let entry = |p: *mut u8| -> Result> { let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is @@ -706,7 +711,21 @@ impl UatPageTable { } } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + let child = upte_val & oas_mask & (!UAT_PGMSK as u64); + // Below a root the driver did not allocate, the descriptors were + // written by the firmware or the bootloader. Follow one only into a + // reserved table page or a table this driver allocated, never into + // other RAM, whose PTEs this walk would then write. + if let (Some(firmware_tree), Some(tables)) = + (firmware_tree, driver_tables.as_ref()) + { + if !firmware_tree.contains(child) && !tables.contains(&child) { + pr_err!("UATPageTable::with_pages: table PTE {:#x} at iova {:#x} level {} names a page outside the reserved and driver tables\n", + upte_val, iova, level + 1); + return Err(EIO); + } + } + Ok(Some(child)) } else if upte_val == 0 || (!alloc && !free) { mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); Ok(None) @@ -986,11 +1005,15 @@ impl UatPageTable { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); if ptev != 0 { + // Never replace a live leaf: it belongs to another mapping. The callers + // check their whole range first (`ensure_unmapped`), so this is reached + // only by a collision they missed. pr_err!( "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", iova + (idx * UAT_PGSZ) as u64, ptev ); + return Err(EBUSY); } pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); if !one_page { @@ -1001,6 +1024,28 @@ impl UatPageTable { }) } + /// Refuses with `EBUSY`, writing nothing, when any leaf of `iova_range` is live. + /// + /// Every PTE store writes either zero or a complete entry, so a nonzero leaf belongs to + /// another mapping. A map operation checks its whole range with this before its first + /// store: its failure path clears the whole range it asked for, which would otherwise + /// also clear the other mapping's leaves. + pub(crate) fn ensure_unmapped(&mut self, iova_range: Range) -> Result { + self.walk(iova_range, false, false, false, |iova, ptes| { + match ptes.iter().position(|pte| pte.load(Ordering::Relaxed) != 0) { + Some(idx) => { + pr_err!( + "UATPageTable::ensure_unmapped: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptes[idx].load(Ordering::Relaxed) + ); + Err(EBUSY) + } + None => Ok(()), + } + }) + } + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { mod_pr_debug!( "UATPageTable::reprot_pages: {:#x?} {:?}\n", diff --git a/drivers/gpu/drm/asahi/queue_share.rs b/drivers/gpu/drm/asahi/queue_share.rs new file mode 100644 index 0000000000000..b7a8ff42affff --- /dev/null +++ b/drivers/gpu/drm/asahi/queue_share.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Each process's share of the GPU's independent queues. +//! +//! A process can open the device as often as it likes, so a queue limit per open file limits +//! nothing. Queues are therefore counted, device-wide, against the process that created them, +//! identified by its mm as [`crate::memacct`] attributes memory, for as long as they exist. +//! +//! A process may always hold up to half of the device's queues. It may take more while at least +//! [`reserve`] queues stay free for the other processes (the compositor and anything started +//! later), or while no other process holds a queue, so a process alone can still use every one. +//! A process below half its share is never refused here, so the reserve stays open to them. + +use kernel::{ + mm::Mm, + prelude::*, + sync::aref::ARef, // +}; + +kernel::sync::global_lock! { + // SAFETY: Module initialization initializes this lock before registering adapters. + unsafe(uninit) static PROCESSES: Mutex> = KVec::new(); +} + +/// Initialize the registry before a platform adapter can probe. +/// +/// # Safety +/// +/// Called exactly once per module instance, before any queue is created. +pub(crate) unsafe fn init() { + // SAFETY: The module initializer establishes the single-call ordering above. + unsafe { PROCESSES.init() }; +} + +/// The queues one process holds. +pub(crate) struct Process { + key: usize, + /// Keeps the `mm_struct`, and with it `key`, from being reused while queues count against it. + _mm: Option>, + held: u32, +} + +/// Queues kept free for the other processes once one process holds half of the device's `max`. +fn reserve(max: u32) -> u32 { + (max / 8).max(1) +} + +/// Whether a process holding `held` of the `total` queues counted device-wide may take another +/// of the device's `max`. The device itself refuses a queue it has no room for. +fn may_take(held: u32, total: u32, max: u32) -> bool { + held < max / 2 || held == total || max.saturating_sub(total.saturating_add(1)) >= reserve(max) +} + +/// Counts a new queue against the process `key`; EBUSY when [`may_take`] refuses it. +fn take(processes: &mut KVec, key: usize, mm: Option>, max: u32) -> Result { + let total = processes.iter().map(|process| process.held).sum::(); + let index = processes.iter().position(|process| process.key == key); + let held = index.map_or(0, |index| processes[index].held); + if !may_take(held, total, max) { + return Err(EBUSY); + } + match index { + Some(index) => processes[index].held += 1, + None => processes.push(Process { key, _mm: mm, held: 1 }, GFP_KERNEL)?, + } + Ok(()) +} + +/// Gives back one queue of `key`; returns the process once it holds none, to drop unlocked. +fn give_back(processes: &mut KVec, key: usize) -> Option { + let index = processes.iter().position(|process| process.key == key)?; + processes[index].held -= 1; + (processes[index].held == 0).then(|| processes.swap_remove(index)) +} + +/// One queue counted against the process that created it; dropping it gives the queue back. +pub(crate) struct Share { + key: usize, +} + +impl Share { + /// Counts a new queue against the current process, of the device's `max` queues. + pub(crate) fn take_current(max: u32) -> Result { + let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); + let key = mm.as_ref().map_or(0, |mm| mm.as_raw() as usize); + take(&mut PROCESSES.lock(), key, mm, max)?; + Ok(Self { key }) + } +} + +impl Drop for Share { + fn drop(&mut self) { + let gone = give_back(&mut PROCESSES.lock(), self.key); + // Drop the process, and with it the mm reference, after the lock. + drop(gone); + } +} + +#[kunit_tests(asahi_queue_share)] +mod tests { + use super::*; + + #[test] + fn a_process_alone_may_use_every_queue() { + assert!(may_take(57, 57, 58)); + assert!(may_take(41, 41, 42)); + } + + #[test] + fn a_large_holder_leaves_the_reserve_to_others() { + // 58 queues keep 7 free for the others; 42 keep 5. + assert_eq!(reserve(58), 7); + assert_eq!(reserve(42), 5); + // With a compositor holding 3, one process gets to 48 of 58, not 29. + assert!(may_take(47, 50, 58)); + assert!(!may_take(48, 51, 58)); + // Below half its share, a process may take queues from the reserve. + assert!(may_take(3, 51, 58)); + assert!(may_take(28, 57, 58)); + } + + #[test] + fn a_lone_process_with_two_files_reaches_the_full_budget() -> Result { + let mut processes = KVec::new(); + // Another process held queues and gave them all back: it no longer counts. + take(&mut processes, 2, None, 58)?; + assert!(give_back(&mut processes, 2).is_some()); + assert!(processes.is_empty()); + // Both files of process 1 count against one key, which may take all 58. + for _ in 0..58 { + take(&mut processes, 1, None, 58)?; + } + assert_eq!(processes.len(), 1); + assert_eq!(processes[0].held, 58); + Ok(()) + } + + #[test] + fn reopening_does_not_reset_the_count() -> Result { + let mut processes = KVec::new(); + // One process (key 1) beside a compositor (key 2) holding 3 queues of 58. + for _ in 0..3 { + take(&mut processes, 2, None, 58)?; + } + // Every file of process 1 counts against the same key. + for _ in 0..48 { + take(&mut processes, 1, None, 58)?; + } + assert_eq!(take(&mut processes, 1, None, 58), Err(EBUSY)); + // The compositor still gets one, and so does a new process. + take(&mut processes, 2, None, 58)?; + take(&mut processes, 3, None, 58)?; + // Giving every queue back removes the process. + for _ in 0..47 { + assert!(give_back(&mut processes, 1).is_none()); + } + assert!(give_back(&mut processes, 1).is_some()); + assert!(processes.iter().all(|process| process.key != 1)); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 976c789930068..60e485f29f06b 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -625,6 +625,55 @@ impl G17Resources { Ok(true) } + /// Publishes through the M5 host command port, including the kick that + /// makes an idle queue runnable. Core activity is not a prerequisite. + pub(crate) fn submit_m5_work( + &self, + submission: &crate::g17::fw::m5::queue::HardwareSubmission, + ) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + for &(offset, value) in submission.writes() { + sgx.try_write64(value, offset)?; + } + Ok(()) + } + + /// The M5 fault window, read for recovery evidence: the fault status, the MMU fault status + /// and address, and three further fault words. The selector is observed, never changed. + pub(crate) fn m5_fault_registers(&self) -> Result<[u64; 6]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Err(EAGAIN); + } + Ok([ + sgx.try_read64(0xd800)?, + sgx.try_read64(0xd8c0)?, + sgx.try_read64(0xd8c8)?, + sgx.try_read64(0xc078)?, + sgx.try_read64(0xc008)?, + sgx.try_read64(0xc028)?, + ]) + } + + /// Sample only the two owned queue configuration words through the + /// host tag/data port. The caller holds the device mutex and power hold. + pub(crate) fn m5_queue_configurations(&self, qids: [u8; 2]) -> Result<[u64; 2]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Err(EAGAIN); + } + let mut values = [0; 2]; + for (value, qid) in values.iter_mut().zip(qids) { + let tag = crate::g17::fw::m5::queue::HardwareSubmission::configuration_tag(qid)?; + sgx.try_write64(tag, 0x4090)?; + *value = sgx.try_read64(0x4098)?; + } + Ok(values) + } + /// Registers a queue KSM slot through the firmware mailbox. pub(crate) fn register_ksm_queue(&self, qid: u8, descriptor: u64) -> Result { if qid > 127 { @@ -821,6 +870,22 @@ impl G17Resources { /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. + /// Prepares the GPU for the M5 firmware before its initialization: sets bit 0 of the + /// firmware start control, which reads 0x30000 or 0x30001 until then, and enables dynamic + /// core power gating. + pub(crate) fn prepare_m5_firmware(&self) -> Result { + { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let value = sgx.try_read32(0xe90000)?; + if value != 0x30000 && value != 0x30001 { + return Err(EIO); + } + sgx.try_write32(value | 1, 0xe90000)?; + } + self.enable_g17_gating() + } + pub(crate) fn enable_g17_gating(&self) -> Result { let sgx = self.sgx.try_access().ok_or(ENODEV)?; let sgx = (&*sgx).as_view(); diff --git a/drivers/gpu/drm/asahi/retain.c b/drivers/gpu/drm/asahi/retain.c new file mode 100644 index 0000000000000..66b1548a7bc3c --- /dev/null +++ b/drivers/gpu/drm/asahi/retain.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// +// THIS_MODULE for the once-per-boot module pin (retain.rs): a firmware +// session that is never freed keeps callbacks into this module. + +#include +#include + +bool asahi_module_try_get(void); +void asahi_module_put(void); + +/* + * Pin this module until reboot, unless its removal has begun: a deferred or + * asynchronous probe can run after an ordinary rmmod has passed its reference + * check, which __module_get() would not stop. + */ +bool asahi_module_try_get(void) +{ + return try_module_get(THIS_MODULE); +} + +/* Drop a reference asahi_module_try_get() took. */ +void asahi_module_put(void) +{ + module_put(THIS_MODULE); +} diff --git a/drivers/gpu/drm/asahi/g15/retain.rs b/drivers/gpu/drm/asahi/retain.rs similarity index 77% rename from drivers/gpu/drm/asahi/g15/retain.rs rename to drivers/gpu/drm/asahi/retain.rs index 30fd79d510869..9df6f417be2fe 100644 --- a/drivers/gpu/drm/asahi/g15/retain.rs +++ b/drivers/gpu/drm/asahi/retain.rs @@ -2,15 +2,23 @@ //! The once-per-boot module pin taken before the firmware is first touched. //! -//! A started G15 firmware session cannot be stopped, so this module must stay loaded until -//! reboot from the first firmware-visible write on. `__module_get` alone does not ensure that: -//! a deferred or asynchronous probe can run while an ordinary rmmod is already past its -//! reference check, and the module is then unloaded under a running firmware. -//! `try_module_get` fails once removal has begun, so the pin is taken with it, before the -//! session is marked retained and before anything firmware-visible happens. +//! A started firmware session that is never freed (every backend's: G15, G16, G18, the M5 +//! RAM-boot path, and G17 on M5) keeps RTKit and mailbox callbacks into this module, so the +//! module must stay loaded until reboot from the first firmware-visible write on. +//! `__module_get` alone does not ensure that: a deferred or asynchronous probe can run while +//! an ordinary rmmod is already past its reference check, and the module is then unloaded +//! under a running firmware. `try_module_get` fails once removal has begun, so the pin is +//! taken with it, before the session is marked retained and before anything +//! firmware-visible happens. use core::sync::atomic::{AtomicBool, Ordering}; +#[cfg(not(test))] +extern "C" { + fn asahi_module_try_get() -> bool; + fn asahi_module_put(); +} + /// The module is being removed: nothing was pinned or marked, and nothing may be started. #[derive(Debug, PartialEq, Eq)] pub(crate) struct Removing; @@ -38,6 +46,21 @@ pub(crate) fn retain( Ok(()) } +/// Pins this module until reboot, once for the session `retained` tracks +/// (`try_module_get(THIS_MODULE)`, see [`retain`]). ENODEV, with nothing pinned or marked, once +/// the module's removal has begun. +#[cfg(not(test))] +pub(crate) fn retain_module(retained: &AtomicBool) -> kernel::error::Result { + retain( + retained, + // SAFETY: Takes a reference to this module unless its removal has begun. + || unsafe { asahi_module_try_get() }, + // SAFETY: Drops only the reference this call took; the winner's is never put. + || unsafe { asahi_module_put() }, + ) + .map_err(|_| kernel::error::code::ENODEV) +} + #[cfg(test)] mod tests { use super::*; diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 0e146c0fa68fc..997716eadf3aa 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -87,6 +87,8 @@ struct macsmc_hwmon_fan { bool manual; /* The device tree allows a manual speed from userspace. */ bool manual_control; + /* Manual for a kernel thermal policy (macsmc_hwmon_fans_request()). */ + bool kernel; }; struct macsmc_hwmon_sensors { @@ -454,6 +456,7 @@ static int macsmc_hwmon_fan_auto(struct macsmc_hwmon *hwmon, ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_AUTO); fan->manual = false; + fan->kernel = false; hwmon->fan.manual--; if (hwmon->fan.unlocked && !hwmon->fan.manual) { @@ -575,6 +578,8 @@ static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long va if (!ret) ret = macsmc_hwmon_fan_manual(hwmon, fan); if (!ret) { + /* Userspace takes the fan over from a kernel policy. */ + fan->kernel = false; ret = macsmc_hwmon_write_key(hwmon->smc, &fan->set, clamp(val, min, max)); if (ret >= 0) return 0; @@ -584,6 +589,102 @@ static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long va return ret; } +/** + * macsmc_hwmon_fans_request() - set the fans for a kernel thermal policy + * @dev: the bound macsmc-hwmon device; the caller holds its device lock + * @permille: each fan's target as a share of its minimum-to-maximum range; + * 0 hands the fans this interface set back to the SMC + * @min_step_rpm: a fan already set by this interface whose target is closer + * than this to the new one keeps its target + * + * The fans go through the same manual mode as a userspace request, so the + * mode key has one owner: fans userspace holds are left alone, and every fan + * returns to the SMC at suspend, shutdown and unbind. Only fans whose device + * tree node names their mode, target, minimum and maximum keys take part. + * + * Return: the first such fan's target in rpm (0 once handed back), -ENODEV + * without such a fan, -EBUSY while stopped, or the first error. + */ +int macsmc_hwmon_fans_request(struct device *dev, unsigned int permille, + unsigned int min_step_rpm) +{ + struct macsmc_hwmon *hwmon = dev_get_drvdata(dev); + long first = -ENODEV; + int ret = 0; + u32 i; + + if (!hwmon) + return -ENODEV; + permille = min(permille, 1000U); + + guard(mutex)(&hwmon->fan.lock); + + if (permille && hwmon->fan.stopped) + return -EBUSY; + + for (i = 0; i < hwmon->fan.count; i++) { + struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[i]; + long min, max, now, target; + int err; + + if (!fan->mode.macsmc_key || + (fan->attrs & (HWMON_F_TARGET | HWMON_F_MIN | HWMON_F_MAX)) != + (HWMON_F_TARGET | HWMON_F_MIN | HWMON_F_MAX)) + continue; + if (fan->manual && !fan->kernel) + continue; + + if (!permille) { + if (first < 0) + first = 0; + err = macsmc_hwmon_fan_auto(hwmon, fan); + if (err && !ret) + ret = err; + continue; + } + + err = macsmc_hwmon_read_key(hwmon->smc, &fan->min, 1, &min); + if (!err) + err = macsmc_hwmon_read_key(hwmon->smc, &fan->max, 1, &max); + if (!err && (min <= 0 || max < min)) + err = -EIO; + if (err) { + dev_warn_ratelimited(hwmon->dev, "%s: no fan range for a kernel request (%d)\n", + fan->label, err); + if (!ret) + ret = err; + continue; + } + target = min + (max - min) * permille / 1000; + if (first < 0) + first = target; + + if (fan->kernel && + !macsmc_hwmon_read_key(hwmon->smc, &fan->set, 1, &now) && + abs(now - target) < (long)min_step_rpm) + continue; + + err = macsmc_hwmon_fan_manual(hwmon, fan); + if (!err) { + fan->kernel = true; + err = macsmc_hwmon_write_key(hwmon->smc, &fan->set, target); + if (err > 0) + err = 0; + } + if (err) { + /* -EOPNOTSUPP: the fan is in system mode and this SMC has no unlock key. */ + dev_warn_ratelimited(hwmon->dev, "%s: kernel fan request refused (%d)\n", + fan->label, err); + macsmc_hwmon_fan_auto(hwmon, fan); + if (!ret) + ret = err; + } + } + + return ret ? ret : first; +} +EXPORT_SYMBOL_GPL(macsmc_hwmon_fans_request); + static int macsmc_hwmon_write_fan(struct device *dev, u32 attr, int channel, long val) { @@ -889,6 +990,9 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) if (!smc->key_count || smc->key_count > U16_MAX) return -EINVAL; + ret = of_property_count_strings(hwmon->dev->of_node, "apple,soc-die-keys"); + hwmon->die_listed = ret > 0 ? ret : 0; + hwmon->temp.sensors = devm_kcalloc(hwmon->dev, smc->key_count, sizeof(*sensor), GFP_KERNEL); if (!hwmon->temp.sensors) @@ -922,6 +1026,12 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) /* Key identity only: do not invent locations or calibration. */ snprintf(sensor->label, sizeof(sensor->label), "%c%c%c%c", key >> 24, (key >> 16) & 0xff, (key >> 8) & 0xff, key & 0xff); + /* A listed SoC die key keeps its discovered channel and joins the die zone. */ + if (of_property_match_string(hwmon->dev->of_node, "apple,soc-die-keys", + sensor->label) >= 0) { + sensor->die = true; + hwmon->die_count++; + } } return 0; @@ -1105,7 +1215,11 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { n_temperature++; } - n_temperature += n_die_keys; + /* With discovery, listed die keys mark discovered channels instead (see there). */ + if (!of_property_read_bool(hwmon_node, "apple,discover-temperatures")) + n_temperature += n_die_keys; + else + n_die_keys = 0; if (n_temperature) { hwmon->temp.sensors = devm_kcalloc(hwmon->dev, n_temperature, diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d5b60c897a4c2..23d72a3b270bc 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -190,6 +191,26 @@ #define DART_T8110_ENABLE_STREAMS 0xc00 #define DART_T8110_DISABLE_STREAMS 0xc20 +/* + * Revision 2.2 and later moved the error mask; 2.4 also moved the error + * status, the error interrupt enable and the per-stream error enables. + */ +#define DART_T8110_V2_ERROR_MASK 0x4060 +#define DART_T8110_ERR_IRQ_STATUS 0x704 +#define DART_T8110_ERR_IRQ_ENABLE 0x700 +#define DART_T8110_ERR_SID_ENABLE(i) (0x720 + 4 * (i)) +#define DART_T8110_V24_ERR_IRQ_STATUS 0x4204 +#define DART_T8110_V24_ERR_IRQ_ENABLE 0x4200 +#define DART_T8110_V24_ERR_SID_ENABLE(i) (0x4220 + 4 * (i)) + +/* PARAMS1 bit 29: client partitions (the 0x800.. registers) exist. */ +#define DART_T8110_PARAMS1_CLIENT_PARTITIONS BIT(29) +#define DART_T8110_CLIENT_PARTITION_BASE 0x800 +#define DART_T8110_CLIENT_PARTITION_END 0x1000 +/* PARAMS2 bit 4: a translation context cache that a full flush leaves alone. */ +#define DART_T8110_PARAMS2_CTC BIT(4) +#define DART_T8110_CTC_FLUSH 0x600 + #define DART_T8110_TCR 0x1000 #define DART_T8110_TCR_REMAP GENMASK(11, 8) #define DART_T8110_TCR_REMAP_ENABLE BIT(7) @@ -262,6 +283,21 @@ struct apple_dart_fw_root { u64 *accepted; }; +/* + * The subtrees behind the slots firmware holds in a four-level locked root + * (fw_mirror), recorded when the root is mapped: per slot, the root entry and + * a mapping of its mid-level table, and per mid entry the entry and a mapping + * of its leaf table. Leaf entries are read live. A table entry that no longer + * matches what was recorded resolves nothing. Only lookups use it: Linux + * still never publishes into those slots. + */ +struct apple_dart_fw_tree { + u64 *entry; + u64 **mid; + u64 **mid_entry; + u64 ***leaf; +}; + /* * Private structure associated with each DART device. * @@ -302,6 +338,15 @@ struct apple_dart { u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; + /* + * fw_mirror's four-level locked roots hold firmware's own subtrees, which + * are recorded and reserved (apple_dart_snapshot_fw_tree()). Set only for + * apple,t6050-dart, where it was tested (the DCP's log and crash log). Before + * widening it, check on T6041/T8140/T8142 whether any locked stream is + * four-level and what its foreign slots hold: the reservation moves DMA + * placement for every client of that stream. + */ + u32 fw_tree : 1; u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; @@ -311,6 +356,16 @@ struct apple_dart { struct apple_tunable *tunables; u16 version; + /* + * Set up from the device tree for a DART nothing else initialised: + * tunables applied at every reset, streams kept in static bypass, every + * error reported, and the context cache flushed after a full TLB flush. + */ + DECLARE_BITMAP(static_bypass, DART_MAX_STREAMS); + bool full_errors; + bool ctc_flush; + bool client_partitions; + /* J713 internal display DARTs, borrowed from the running DCP. */ struct apple_dart_dcp *dcp; struct apple_dart_dcp *frontend[2]; @@ -365,6 +420,8 @@ struct apple_dart { u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Leaf tables behind the root slots firmware held when we attached. */ struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* Firmware's subtrees in a four-level locked root (TTBR 0 only). */ + struct apple_dart_fw_tree *locked_fw_tree[DART_MAX_STREAMS]; /* * First DVA of the window of the domain using each locked stream. The * window can be the client's own (apple,dma-range on the client node) @@ -832,15 +889,22 @@ static bool apple_dart_fw_handoff_enabled(const struct apple_dart *dart) handoff == 1; } +/* A 16K table page, aligned and inside the output address space. */ +static bool apple_dart_fw_table_page(const struct apple_dart *dart, + phys_addr_t phys, phys_addr_t *end) +{ + return dart->pgsize == SZ_16K && IS_ALIGNED(phys, dart->pgsize) && + !check_add_overflow(phys, (phys_addr_t)dart->pgsize - 1, end) && + dart->oas < 64 && *end < BIT_ULL(dart->oas); +} + static bool apple_dart_fw_table_ram(const struct apple_dart *dart, phys_addr_t phys) { unsigned long first, last, pfn; phys_addr_t end; - if (dart->pgsize != SZ_16K || !IS_ALIGNED(phys, dart->pgsize) || - check_add_overflow(phys, (phys_addr_t)dart->pgsize - 1, &end) || - dart->oas >= 64 || end >= BIT_ULL(dart->oas)) + if (!apple_dart_fw_table_page(dart, phys, &end)) return false; first = PHYS_PFN(phys); last = PHYS_PFN(end); @@ -850,14 +914,41 @@ static bool apple_dart_fw_table_ram(const struct apple_dart *dart, return true; } -static bool apple_dart_fw_table_decode(const struct apple_dart *dart, - u64 pte, phys_addr_t *phys) +/* A valid table entry with no bits but the next table's address. */ +static bool apple_dart_fw_table_pte(const struct apple_dart *dart, u64 pte, + phys_addr_t *phys) { if (!(pte & APPLE_DART_PTE_VALID) || (pte & ~(GENMASK_ULL(37, 10) | APPLE_DART_PTE_VALID))) return false; *phys = apple_dart_pte_to_paddr(dart, pte); - return apple_dart_fw_table_ram(dart, *phys); + return true; +} + +static bool apple_dart_fw_table_decode(const struct apple_dart *dart, + u64 pte, phys_addr_t *phys) +{ + return apple_dart_fw_table_pte(dart, pte, phys) && + apple_dart_fw_table_ram(dart, *phys); +} + +/* + * A table page of firmware's four-level subtrees, which is mapped write-back: + * reserved RAM as for apple_dart_fw_table_ram(), or memory no resource + * claims. T6050's boot loader places these tables outside the memory node, + * so they are not RAM to Linux. An entry naming Linux's own RAM or a + * device's registers is refused, and its DVAs resolve nothing. + */ +static bool apple_dart_fw_tree_table(const struct apple_dart *dart, + phys_addr_t phys) +{ + phys_addr_t end; + + if (apple_dart_fw_table_ram(dart, phys)) + return true; + return apple_dart_fw_table_page(dart, phys, &end) && + region_intersects(phys, dart->pgsize, IORESOURCE_MEM, + IORES_DESC_NONE) == REGION_DISJOINT; } static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, @@ -1084,6 +1175,118 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) return fw; } +static void apple_dart_free_fw_tree(struct apple_dart *dart, + struct apple_dart_fw_tree *tree) +{ + size_t i, j, entries = dart->pgsize / sizeof(u64); + + if (!tree) + return; + /* Any of the four arrays may be missing after a failed allocation. */ + for (i = 0; i < entries; i++) { + if (tree->leaf && tree->leaf[i]) { + for (j = 0; j < entries; j++) + if (tree->leaf[i][j]) + memunmap(tree->leaf[i][j]); + kfree(tree->leaf[i]); + } + if (tree->mid_entry) + kfree(tree->mid_entry[i]); + if (tree->mid && tree->mid[i]) + memunmap(tree->mid[i]); + } + kfree(tree->leaf); + kfree(tree->mid_entry); + kfree(tree->mid); + kfree(tree->entry); + kfree(tree); +} + +/* + * Record firmware's subtrees in the four-level root @live of @sid, so that + * firmware-placed buffers (an RTKit co-processor's shared segments, which + * it asks the AP to map at its own DVA) resolve through iova_to_phys. A + * table that cannot be mapped, or that apple_dart_fw_tree_table() refuses, + * is left out; its DVAs resolve nothing. A slot can map up to one leaf table + * per mid-level entry (2048 with 16K pages); firmware holds a few slots. + */ +static struct apple_dart_fw_tree * +apple_dart_snapshot_fw_tree(struct apple_dart *dart, int sid, const u64 *live) +{ + size_t i, j, entries = dart->pgsize / sizeof(u64); + struct apple_dart_fw_tree *tree; + + if (!dart->fw_tree || !dart->fw_mirror || dart->fw_handoff || + dart->hw->fmt != APPLE_DART2 || + !dart->hw->tcr_4level || + !(apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level)) + return NULL; + + tree = kzalloc(sizeof(*tree), GFP_KERNEL); + if (!tree) + return NULL; + tree->entry = kcalloc(entries, sizeof(*tree->entry), GFP_KERNEL); + tree->mid = kcalloc(entries, sizeof(*tree->mid), GFP_KERNEL); + tree->mid_entry = kcalloc(entries, sizeof(*tree->mid_entry), GFP_KERNEL); + tree->leaf = kcalloc(entries, sizeof(*tree->leaf), GFP_KERNEL); + if (!tree->entry || !tree->mid || !tree->mid_entry || !tree->leaf) { + apple_dart_free_fw_tree(dart, tree); + return NULL; + } + + for (i = 0; i < entries; i++) { + u64 pte = READ_ONCE(live[i]); + phys_addr_t phys; + + if (!apple_dart_fw_table_pte(dart, pte, &phys) || + !apple_dart_fw_tree_table(dart, phys)) + continue; + tree->mid_entry[i] = kcalloc(entries, sizeof(u64), GFP_KERNEL); + tree->leaf[i] = kcalloc(entries, sizeof(u64 *), GFP_KERNEL); + tree->mid[i] = memremap(phys, dart->pgsize, MEMREMAP_WB); + if (!tree->mid_entry[i] || !tree->leaf[i] || !tree->mid[i]) + continue; + tree->entry[i] = pte; + for (j = 0; j < entries; j++) { + u64 mid = READ_ONCE(tree->mid[i][j]); + + if (!apple_dart_fw_table_pte(dart, mid, &phys) || + !apple_dart_fw_tree_table(dart, phys)) + continue; + tree->leaf[i][j] = memremap(phys, dart->pgsize, MEMREMAP_WB); + if (tree->leaf[i][j]) + tree->mid_entry[i][j] = mid; + } + } + return tree; +} + +/* What firmware's recorded four-level subtree translates @addr to, or 0. */ +static phys_addr_t apple_dart_fw_tree_phys(const struct apple_dart *dart, + const struct apple_dart_fw_tree *tree, + const u64 *live, u64 addr, int *prot_out) +{ + u32 pgshift = ilog2(dart->pgsize); + u32 bits = ilog2(dart->pgsize / sizeof(u64)); + u64 mask = BIT_ULL(bits) - 1; + u64 slot = addr >> (pgshift + 2 * bits); + u64 mid = (addr >> (pgshift + bits)) & mask; + u64 page = (addr >> pgshift) & mask; + phys_addr_t phys; + int prot; + + if (!tree || !live || slot > mask || !tree->entry[slot] || + READ_ONCE(live[slot]) != tree->entry[slot] || + !tree->mid_entry[slot][mid] || + READ_ONCE(tree->mid[slot][mid]) != tree->mid_entry[slot][mid] || + !apple_dart_fw_leaf_decode(READ_ONCE(tree->leaf[slot][mid][page]), + addr, &phys, &prot)) + return 0; + if (prot_out) + *prot_out = prot; + return phys; +} + static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 lo, u64 hi); @@ -1094,6 +1297,7 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + struct apple_dart_fw_tree *tree = NULL; struct apple_dart_fw_root *fw; unsigned long flags; u32 ttbr; @@ -1140,10 +1344,14 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) devm_memunmap(dart->dev, l1_tbl); return PTR_ERR(fw); } + if (!idx) + tree = apple_dart_snapshot_fw_tree(dart, sid, l1_tbl); spin_lock_irqsave(&dart->lock, flags); dart->locked_owned[sid][idx] = owned; dart->locked_ttbr[sid][idx] = l1_tbl; dart->locked_fw[sid][idx] = fw; + if (!idx) + dart->locked_fw_tree[sid] = tree; spin_unlock_irqrestore(&dart->lock, flags); } @@ -1165,6 +1373,7 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; + struct apple_dart_fw_tree *tree = NULL; struct apple_dart_fw_root *fw; unsigned long flags; bool first = false; @@ -1189,6 +1398,10 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx fw = dart->locked_fw[sid][idx]; dart->locked_fw[sid][idx] = NULL; dart->locked_ttbr[sid][idx] = NULL; + if (!idx) { + tree = dart->locked_fw_tree[sid]; + dart->locked_fw_tree[sid] = NULL; + } spin_unlock_irqrestore(&dart->lock, flags); /* Once per stream: the error is never cleared. */ if (first) @@ -1196,6 +1409,7 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx "stream %d: TLB invalidate failed retiring its root (%d); keeping the page tables of its domains until reboot\n", sid, err); apple_dart_free_fw_root(dart, fw); + apple_dart_free_fw_tree(dart, tree); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); dart->locked_owned[sid][idx] = NULL; @@ -1377,6 +1591,10 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, if (ret) break; } + /* A V2 flush leaves the context cache alone: flush all of it after. */ + if (!ret && dart->ctc_flush && dart->version >= 0x0202 && + command == DART_T8110_TLB_CMD_OP_FLUSH_SID) + apple_dart_writel(dart, 0, DART_T8110_CTC_FLUSH); spin_unlock_irqrestore(&dart->lock, flags); if (ret) dev_err_ratelimited(dart->dev, @@ -1750,6 +1968,59 @@ apple_dart_t8110_hw_enable_remap(struct apple_dart *dart, int sid, int remap_to) * acknowledge the errors latched so far, unlock a DART that locks on errors, * unmask the error interrupt and flush the TLBs. */ +/* + * Unmask every error and enable its interrupt for each stream, with the + * registers of this revision: the T8110 reset value leaves them masked on + * 2.2 and later, where the error mask is no longer at ERROR_MASK. + */ +static void apple_dart_t8110_enable_errors(struct apple_dart *dart) +{ + bool v24 = dart->version >= 0x0204; + unsigned int i; + + apple_dart_writel(dart, 0, dart->version >= 0x0202 ? + DART_T8110_V2_ERROR_MASK : DART_T8110_ERROR_MASK); + apple_dart_readl(dart, dart->version >= 0x0202 ? + DART_T8110_V2_ERROR_MASK : DART_T8110_ERROR_MASK); + apple_dart_writel(dart, U32_MAX, v24 ? DART_T8110_V24_ERR_IRQ_STATUS : + DART_T8110_ERR_IRQ_STATUS); + apple_dart_writel(dart, U32_MAX, v24 ? DART_T8110_V24_ERR_IRQ_ENABLE : + DART_T8110_ERR_IRQ_ENABLE); + for (i = 0; i < DIV_ROUND_UP(dart->num_streams, 32); i++) { + u32 sids = dart->num_streams >= 32 * (i + 1) ? U32_MAX : + GENMASK(dart->num_streams - 32 * i - 1, 0); + + apple_dart_writel(dart, sids, v24 ? DART_T8110_V24_ERR_SID_ENABLE(i) : + DART_T8110_ERR_SID_ENABLE(i)); + } + /* No lock on error: the driver acknowledges and carries on. */ + apple_dart_writel(dart, 0, DART_T8110_DIAG_LOCK); +} + +/* + * Apply the DART's own tunables. Client partition registers exist only where + * PARAMS1 says so; a write to one that does not exist is skipped. + */ +static void apple_dart_apply_own_tunables(struct apple_dart *dart) +{ + size_t i; + + for (i = 0; i < dart->tunables->sz; ++i) { + u32 offset = dart->tunables->values[i].offset; + u32 old_value, value; + + if (offset >= DART_T8110_CLIENT_PARTITION_BASE && + offset < DART_T8110_CLIENT_PARTITION_END && + !dart->client_partitions) + continue; + old_value = apple_dart_readl(dart, offset); + value = old_value & ~dart->tunables->values[i].mask; + value |= dart->tunables->values[i].value; + if (value != old_value) + apple_dart_writel(dart, value, offset); + } +} + static int apple_dart_hw_finish_reset(struct apple_dart *dart) { struct apple_dart_stream_map stream_map = { .dart = dart }; @@ -1795,6 +2066,9 @@ static int apple_dart_hw_finish_reset(struct apple_dart *dart) if (ret) dev_err(dart->dev, "failed to flush the TLBs after reset: %d\n", ret); + if (dart->full_errors) + apple_dart_t8110_enable_errors(dart); + return ret; } @@ -2012,10 +2286,18 @@ static int apple_dart_hw_reset(struct apple_dart *dart) bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); + /* Before the first stream is configured. */ + if (!dart->tunneled && dart->tunables) + apple_dart_apply_own_tunables(dart); + apple_dart_hw_disable_dma(&stream_map); apple_dart_hw_clear_all_ttbrs(&stream_map); apple_dart_hw_apply_remaps(dart); + /* Streams that bypass translation for good, outside any domain. */ + for_each_set_bit(i, dart->static_bypass, dart->num_streams) + apple_dart_writel(dart, DART_T8110_TCR_BYPASS_DART, DART_TCR(dart, i)); + /* enable all streams globally since TCR is used to control isolation */ for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) apple_dart_writel(dart, U32_MAX, @@ -2257,11 +2539,18 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, return 0; } +/* + * What firmware's own retained translations in @domain map @address to, or 0. + * With @prot_out, also the protection every stream grants (IOMMU_READ and + * IOMMU_WRITE as the leaves say). + */ static phys_addr_t -apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +apple_dart_inherited_phys_prot(struct apple_dart_domain *domain, dma_addr_t address, + int *prot_out) { struct apple_dart_atomic_stream_map *map; phys_addr_t result = 0; + int granted = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; int i, sid; for_each_stream_map(i, domain, map) { @@ -2271,31 +2560,90 @@ apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) size_t page = (address / dart->pgsize) % entries; size_t slot = address / dart->pgsize / entries; - if (!dart->locked || (!dart->fw_handoff && !dart->fw_mirror) || slot >= entries) + if (!dart->locked || (!dart->fw_handoff && !dart->fw_mirror)) return 0; spin_lock_irqsave(&dart->lock, flags); for (sid = 0; sid < dart->num_streams; sid++) { + const struct apple_dart_fw_tree *tree; struct apple_dart_fw_root *fw; - phys_addr_t phys; + phys_addr_t phys = 0; int prot; if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; + tree = dart->locked_fw_tree[sid]; fw = dart->locked_fw[sid][0]; - if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || - !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), - address, &phys, &prot) || - (result && result != phys)) { + prot = 0; + if (tree) + phys = apple_dart_fw_tree_phys(dart, tree, + dart->locked_ttbr[sid][0], + address, &prot); + else if (slot >= entries || + !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || + !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), + address, &phys, &prot)) + phys = 0; + if (!phys || (result && result != phys)) { spin_unlock_irqrestore(&dart->lock, flags); return 0; } result = phys; + granted &= prot; } spin_unlock_irqrestore(&dart->lock, flags); } + if (prot_out) + *prot_out = result ? granted : 0; return result; } +static phys_addr_t +apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +{ + return apple_dart_inherited_phys_prot(domain, address, NULL); +} + +/* + * Resolves a DVA a co-processor's firmware names in a request of its own, for + * a host copy to or from that buffer. Only firmware's retained mappings count: + * a DVA Linux mapped itself (a framebuffer, an imported or DMA_TO_DEVICE + * buffer) is refused, so the host never writes what the device may only read. + * @write requires the leaf to grant writes. -EACCES when refused, -ENXIO when + * unmapped. + */ +int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, + phys_addr_t *phys) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart_domain *dart_domain; + struct io_pgtable_ops *ops; + struct io_pgtable_cfg *cfg; + dma_addr_t address; + phys_addr_t result; + int prot; + + if (!domain || domain->owner != &apple_dart_iommu_ops) + return -ENODEV; + dart_domain = to_dart_domain(domain); + ops = dart_domain->pgtbl_ops; + if (dart_domain->dcp || !ops) + return -ENODEV; + address = (iova + dart_domain->dma_offset) & dart_domain->mask; + if (ops->iova_to_phys(ops, address)) + return -EACCES; + cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; + if (cfg->apple_dart_cfg.n_levels != 3 && cfg->apple_dart_cfg.n_levels != 4) + return -ENXIO; + result = apple_dart_inherited_phys_prot(dart_domain, address, &prot); + if (!result) + return -ENXIO; + if (!(prot & IOMMU_READ) || (write && !(prot & IOMMU_WRITE))) + return -EACCES; + *phys = result; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_fw_remote); + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -2315,7 +2663,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (phys) return phys; cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; - if (cfg->apple_dart_cfg.n_levels != 3) + if (cfg->apple_dart_cfg.n_levels != 3 && cfg->apple_dart_cfg.n_levels != 4) return 0; /* Existing firmware translations need no replacement software mapping. */ return apple_dart_inherited_phys(dart_domain, address); @@ -3550,7 +3898,12 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, struct apple_dart *dart = stream->dart; size_t entries = dart->pgsize / sizeof(u64); - if (!dart->fw_handoff || entries > DART_MAX_ROOT_ENTRIES) + /* + * Only the handoff and fw_tree reserve anything: leave the other + * fw_mirror DARTs (T6041, T8140, T8142) untouched, TCR reads included. + */ + if ((!dart->fw_handoff && !dart->fw_tree) || + entries > DART_MAX_ROOT_ENTRIES) continue; for_each_set_bit(sid, stream->sidmap, dart->num_streams) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) { @@ -3574,6 +3927,13 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, __set_bit(slot, slots); spin_unlock_irqrestore(&dart->lock, flags); + /* + * A three-level fw_mirror slot may repeat firmware's + * pages (apple_dart_check_fw_map()); a four-level one + * is firmware's alone, so keep the allocator out of it. + */ + if (!dart->fw_handoff && !(four && dart->fw_tree)) + continue; if (!four && check_add_overflow(window, ((u64)idx << (ilog2(dart->pgsize) + 2 * ilog2(entries))), &window)) @@ -3855,6 +4215,17 @@ static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) readl(dart->regs + DART_T8110_EXCEPTION(0)); } + /* With every error enabled, 2.4 latches the rest in its own status. */ + if (dart->full_errors && dart->version >= 0x0204) { + u32 status = readl(dart->regs + DART_T8110_V24_ERR_IRQ_STATUS); + + if (status) { + dev_err_ratelimited(dart->dev, "error status: 0x%08x\n", status); + writel(status, dart->regs + DART_T8110_V24_ERR_IRQ_STATUS); + ret = IRQ_HANDLED; + } + } + if (dart->version >= 0x0203) { block_error = readl(dart->regs + DART_T8110_BLOCK_ERROR); if (apple_dart_t8110_v2_block_pending(dart->version, block_error)) { @@ -4127,6 +4498,56 @@ static bool apple_dart_has_live_rtkit_client(struct device_node *np) return false; } +/* + * The setup a T8110 DART needs when nothing initialised it before Linux: + * apple,tunable (applied at every reset), apple,bypass-sids (streams kept in + * static bypass), apple,full-error-reporting and apple,ctc-flush. Log the + * identification registers of such a DART. + */ +static int apple_dart_parse_own_setup(struct apple_dart *dart, struct resource *res, + const u32 *params) +{ + struct device *dev = dart->dev; + struct device_node *np = dev->of_node; + int count, i; + u32 sid; + + dart->full_errors = of_property_read_bool(np, "apple,full-error-reporting"); + dart->ctc_flush = of_property_read_bool(np, "apple,ctc-flush") && + (params[1] & DART_T8110_PARAMS2_CTC); + dart->client_partitions = params[0] & DART_T8110_PARAMS1_CLIENT_PARTITIONS; + + if (of_property_present(np, "apple,tunable")) { + dart->tunables = devm_apple_tunable_parse(dev, np, "apple,tunable", res); + if (IS_ERR(dart->tunables)) + return dev_err_probe(dev, PTR_ERR(dart->tunables), + "failed to parse DART tunables\n"); + } + + count = of_property_count_u32_elems(np, "apple,bypass-sids"); + for (i = 0; i < count; i++) { + if (of_property_read_u32_index(np, "apple,bypass-sids", i, &sid) || + sid >= dart->num_streams) + return dev_err_probe(dev, -EINVAL, "invalid bypass stream\n"); + __set_bit(sid, dart->static_bypass); + } + + if (dart->tunables || count > 0 || dart->full_errors || + of_property_read_bool(np, "apple,ctc-flush")) + dev_info(dev, + "params %08x %08x %08x %08x, version %04x, %u streams, AS %u -> %u, protect %08x/%08x, diag lock %08x; %zu tunables%s, %d bypass streams, CTC flush %s\n", + params[0], params[1], params[2], params[3], dart->version, + dart->num_streams, dart->ias, dart->oas, + apple_dart_readl(dart, DART_T8110_PROTECT), + apple_dart_readl(dart, DART_T8110_PROTECT_LOCK), + apple_dart_readl(dart, DART_T8110_DIAG_LOCK), + dart->tunables ? dart->tunables->sz : 0, + dart->tunables && !dart->client_partitions ? + " (no client partitions)" : "", + max(count, 0), dart->ctc_flush ? "on" : "off"); + return 0; +} + static int apple_dart_probe(struct platform_device *pdev) { int ret, sid; @@ -4433,6 +4854,12 @@ static int apple_dart_probe(struct platform_device *pdev) } params_done: + if (!tunneled && dart->hw->type == DART_T8110) { + ret = apple_dart_parse_own_setup(dart, res, dart_params); + if (ret) + goto err_clk_disable; + } + dart->dma_min = 0; dart->dma_max = DMA_BIT_MASK(dart->ias); dart->dma_offset = 0; @@ -4536,9 +4963,14 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = apple_dart_is_locked(dart); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); - /* Preserve our retained-root path when no reserved-table handoff exists. */ + /* + * Preserve our retained-root path when no reserved-table handoff exists. + * T6050's boot loader leaves every display DART locked on four-level + * roots of its own (dart-dcpextN SID 23, dart-dispextN SIDs 0 and 16). + */ dart->fw_mirror = dart->locked && !dart->fw_handoff && (of_device_is_compatible(dev->of_node, "apple,t6041-dart") || + of_device_is_compatible(dev->of_node, "apple,t6050-dart") || of_device_is_compatible(dev->of_node, "apple,t8140-dart") || of_device_is_compatible(dev->of_node, "apple,t8142-dart")); /* @@ -4573,6 +5005,8 @@ static int apple_dart_probe(struct platform_device *pdev) of_node_put(iommu); of_node_put(mtp); } + dart->fw_tree = dart->fw_mirror && + of_device_is_compatible(dev->of_node, "apple,t6050-dart"); if (apple_t8132_live_mtp() && of_machine_is_compatible("apple,t8132") && res->start == 0x394800000ULL && dart->hw->type == DART_T8110 && !dart->locked && !tunneled && dart->supports_bypass && diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 546e5cdbb72ad..927ddaa00e0df 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple SMC (System Management Controller) MFD driver * @@ -59,9 +59,54 @@ static const struct mfd_cell apple_smc_devs[] = { MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; -/* The second GPIO bank ('gp' keys) only exists on some SMCs. */ -static const struct mfd_cell apple_smc_low_gpio_dev = - MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); +/* This profile exposes notifications and the explicitly described outputs. */ +static const struct mfd_cell apple_smc_t6050_devs[] = { + MFD_CELL_NAME("macsmc-input"), + MFD_CELL_NAME("macsmc-power"), + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), +}; + +/* + * Cells registered only where the device tree describes them: the MFD core would otherwise + * create an unbound device and warn that it found no node on every SMC without one. The pcIO + * outputs and the second GPIO bank ('gp' keys) only exist on some SMCs. + */ +static const struct mfd_cell apple_smc_described_devs[] = { + MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"), + MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"), +}; + +/* + * T6050's reboot/power-off and RTC, for a kernel that owns the machine (J775D, booted bare + * metal). Registered only where the device tree describes them, so a T6050 device tree without + * them registers what it did before. + */ +static const struct mfd_cell apple_smc_t6050_described_devs[] = { + MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), + MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), +}; + +static int apple_smc_add_described_devices(struct apple_smc *smc, const struct mfd_cell *cells, + size_t count) +{ + struct device_node *np; + size_t i; + int ret; + + for (i = 0; i < count; i++) { + np = of_get_compatible_child(smc->dev->of_node, cells[i].of_compatible); + if (!np) + continue; + of_node_put(np); + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, &cells[i], 1, + NULL, 0, NULL); + if (ret) + return dev_err_probe(smc->dev, ret, "Failed to register %s\n", + cells[i].name); + } + + return 0; +} /* T8152 currently exposes read-only telemetry; firmware owns fan control. */ static const struct mfd_cell apple_smc_telemetry_devs[] = { @@ -250,8 +295,8 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ u8 key_info[6]; int ret; + /* Keep the shared reply buffer stable until its metadata is copied. */ guard(mutex)(&smc->mutex); - ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { apple_smc_read_sram(smc, key_info, sizeof(key_info)); @@ -483,9 +528,9 @@ static void apple_smc_disable_notifications(void *data) static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np; struct apple_smc *smc; __be32 count = 0; + bool t6050; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -559,32 +604,30 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; - ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, - apple_smc_devs, ARRAY_SIZE(apple_smc_devs), - NULL, 0, NULL); + t6050 = of_device_is_compatible(dev->of_node, "apple,t6050-smc"); + if (t6050) + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_t6050_devs, + ARRAY_SIZE(apple_smc_t6050_devs), NULL, 0, NULL); + else + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_devs, ARRAY_SIZE(apple_smc_devs), + NULL, 0, NULL); if (ret) return dev_err_probe(smc->dev, ret, "Failed to register sub-devices"); - /* - * Register the second GPIO bank only where the device tree describes - * it: the MFD core would otherwise create an unbound device and warn - * that it found no node on every SMC without one. - */ - np = of_get_compatible_child(dev->of_node, "apple,smc-low-gpio"); - if (np) { - of_node_put(np); - ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, - &apple_smc_low_gpio_dev, 1, NULL, 0, NULL); - if (ret) - return dev_err_probe(smc->dev, ret, - "Failed to register the second GPIO bank"); - } + ret = apple_smc_add_described_devices(smc, apple_smc_described_devs, + ARRAY_SIZE(apple_smc_described_devs)); + if (ret || !t6050) + return ret; - return 0; + return apple_smc_add_described_devices(smc, apple_smc_t6050_described_devs, + ARRAY_SIZE(apple_smc_t6050_described_devs)); } static const struct of_device_id apple_smc_of_match[] = { { .compatible = "apple,t8152-smc" }, + { .compatible = "apple,t6050-smc" }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 65107c7de3800..f45d296eebef4 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple Silicon Display Crossbar multiplexer driver * @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -90,7 +93,10 @@ enum { MUX_DPPHY = 0, MUX_DPIN0 = 1, MUX_DPIN1 = 2, MUX_MAX = 3 }; static const char *apple_dpxbar_names[MUX_MAX] = { "dpphy", "dpin0", "dpin1" }; +enum { T8142_MUX_MAX = 5 }; + struct apple_dpxbar_hw { + unsigned int n_mux; unsigned int n_ufp; u32 tunable; const struct mux_control_ops *ops; @@ -102,18 +108,24 @@ struct apple_dpxbar_hw { bool no_tunable; /* power domain held only while an output is selected */ bool runtime_pm; + /* T8142 layout: a connection status still busy after its poll is only logged */ + bool status_log_only; }; struct apple_dpxbar { struct device *dev; const struct apple_dpxbar_hw *hw; void __iomem *regs; - int selected_dispext[MUX_MAX]; + int selected_dispext[T8142_MUX_MAX]; /* T6030 programs all its outputs itself, see apple_dpxbar_t6030_link() */ bool t6030_dpin; /* outputs holding a runtime PM reference (runtime_pm crossbars) */ unsigned long rpm_held; bool tunnel_link_up[MUX_MAX]; + u8 pclk[T8142_MUX_MAX]; + bool uhbr[T8142_MUX_MAX]; + bool enabled[T8142_MUX_MAX]; + bool faulted; spinlock_t lock; }; @@ -880,6 +892,227 @@ int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state) } EXPORT_SYMBOL_GPL(apple_dpxbar_tunnel_select_source); +/* + * T8142 (also used by T6050) has four pixel clocks and five outputs. + * Keep selection separate from clock activation: the PHY/DP IN owner supplies + * its actual clock and coding before link_up(), after configuring that clock. + * Only the four two-port external display sources are described here; AUX + * sources use a different selector-to-clock-bit mapping. + */ +static const u8 t8142_dest_shift[T8142_MUX_MAX] = { 8, 0, 4, 12, 16 }; +static const u8 t8142_select_shift[T8142_MUX_MAX] = { 8, 0, 4, 24, 0 }; +static const u8 t8142_select_shift2[T8142_MUX_MAX] = { 20, 12, 16, 28, 4 }; + +/* + * Wait up to 1 ms for @mask to clear in status word @reg. Where the hardware + * only reports these bits (T6050), one still set is logged and the change + * goes on; elsewhere the crossbar is taken as faulted. + */ +static int t8142_settle(struct apple_dpxbar *xbar, unsigned int reg, u32 mask, + unsigned int index, const char *what) +{ + u32 value; + int ret; + + ret = readl_poll_timeout_atomic(xbar->regs + reg, value, !(value & mask), 1, 1000); + if (!ret) + return 0; + if (xbar->hw->status_log_only) { + dev_warn_ratelimited(xbar->dev, + "output %u %s: status +%#x = %#x, %#x still set; going on\n", + index, what, reg, value, mask); + return 0; + } + xbar->faulted = true; + return ret; +} + +static int t8142_link_down(struct apple_dpxbar *xbar, unsigned int index) +{ + u32 src, dst, clock; + int ret; + + if (!xbar->enabled[index]) + return 0; + src = BIT(xbar->selected_dispext[index]); + dst = BIT(t8142_dest_shift[index]); + clock = xbar->pclk[index] - 1; + + dpxbar_clear32(xbar, 0x3c, src); + dpxbar_clear32(xbar, 0x00, src); + dpxbar_clear32(xbar, 0x0c + 4 * clock, src); + dpxbar_clear32(xbar, 0x24 + 4 * clock, dst); + udelay(1); + ret = t8142_settle(xbar, 0x800, src, index, "link down"); + if (!ret) + ret = t8142_settle(xbar, 0x808 + 4 * clock, src, index, "link down"); + if (!ret) + ret = t8142_settle(xbar, 0x81c + 4 * clock, dst, index, "link down"); + if (ret) + return ret; + dpxbar_clear32(xbar, 0x08, src); + dpxbar_clear32(xbar, 0x20, 7 << (3 * xbar->selected_dispext[index])); + dpxbar_clear32(xbar, 0x38, 7 << t8142_dest_shift[index]); + dpxbar_set32(xbar, 0x04, src); + dpxbar_set32(xbar, 0x1c, src); + dpxbar_set32(xbar, 0x34, dst); + xbar->enabled[index] = false; + return 0; +} + +static int t8142_link_up(struct apple_dpxbar *xbar, unsigned int index) +{ + int state = xbar->selected_dispext[index]; + u32 clock, select, src, dst; + int ret; + + if (state < 0) + return -ENODEV; + if (!xbar->pclk[index]) + return -EINVAL; + if (xbar->enabled[index]) + return 0; + + /* The coding bit is shared by every output using this pixel clock. */ + for (unsigned int i = 0; i < T8142_MUX_MAX; i++) + if (i != index && xbar->enabled[i] && + xbar->pclk[i] == xbar->pclk[index] && + xbar->uhbr[i] != xbar->uhbr[index]) + return -EBUSY; + + clock = xbar->pclk[index] - 1; + select = 2 * clock + 1; + src = BIT(state); + dst = BIT(t8142_dest_shift[index]); + dpxbar_clear32(xbar, 0x04, src); + dpxbar_clear32(xbar, 0x1c, src); + dpxbar_clear32(xbar, 0x34, dst); + udelay(1); + ret = t8142_settle(xbar, 0x804, src, index, "link up"); + if (!ret) + ret = t8142_settle(xbar, 0x818, src, index, "link up"); + if (!ret) + ret = t8142_settle(xbar, 0x82c, dst, index, "link up"); + if (ret) + return ret; + dpxbar_set32(xbar, 0x08, src); + dpxbar_mask32(xbar, 0x20, 7 << (3 * state), select << (3 * state)); + dpxbar_mask32(xbar, 0x38, 7 << t8142_dest_shift[index], + select << t8142_dest_shift[index]); + dpxbar_mask32(xbar, 0x48, BIT(clock), xbar->uhbr[index] ? BIT(clock) : 0); + dpxbar_set32(xbar, 0x00, src); + dpxbar_set32(xbar, 0x0c + 4 * clock, src); + dpxbar_set32(xbar, 0x24 + 4 * clock, dst); + dpxbar_set32(xbar, 0x4c, dst); + dpxbar_set32(xbar, 0x3c, src); + xbar->enabled[index] = true; + return 0; +} + +static int apple_dpxbar_set_t8142(struct mux_control *mux, int state) +{ + struct apple_dpxbar *xbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + u32 mask, value; + int ret = 0; + + if (state != MUX_IDLE_DISCONNECT && + (state < 0 || state >= 8 || (index == MUX_DPPHY && (state & 1)))) + return -EINVAL; + + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->faulted) { + ret = -EIO; + goto out; + } + if (state >= 0) { + for (unsigned int i = 0; i < T8142_MUX_MAX; i++) { + if (i != index && xbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -EBUSY; + } + } + } + + if (xbar->selected_dispext[index] == state) + goto out; + if (xbar->selected_dispext[index] >= 0) { + ret = t8142_link_down(xbar, index); + if (ret) + goto out; + } + + /* No implicit clock choice or activation when selecting a new source. */ + xbar->pclk[index] = 0; + xbar->selected_dispext[index] = state; + if (state >= 0) { + mask = (0xfU << t8142_select_shift[index]) | + (0xfU << t8142_select_shift2[index]); + value = ((u32)state << t8142_select_shift[index]) | + ((u32)state << t8142_select_shift2[index]); + dpxbar_mask32(xbar, index == 4 ? 0x40 : 0x44, mask, value); + } +out: + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} + +static const struct mux_control_ops apple_dpxbar_t8142_ops = { + .set = apple_dpxbar_set_t8142, +}; + +/* Caller holds the mux selection throughout configuration and activation. */ +int apple_dpxbar_link_configure(struct mux_control *mux, unsigned int pclk, bool uhbr) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + int ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t8142_ops) + return -EOPNOTSUPP; + if (pclk < 1 || pclk > 4) + return -EINVAL; + xbar = mux_chip_priv(mux->chip); + index = mux_control_get_index(mux); + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->faulted) + ret = -EIO; + else if (xbar->selected_dispext[index] < 0) + ret = -ENODEV; + else if (xbar->enabled[index]) + ret = -EBUSY; + else { + xbar->pclk[index] = pclk; + xbar->uhbr[index] = uhbr; + } + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_configure); + +static int apple_dpxbar_t8142_link(struct mux_control *mux, bool up) +{ + struct apple_dpxbar *xbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->faulted) + ret = -EIO; + else if (xbar->selected_dispext[index] < 0) + ret = -ENODEV; + else if (up) + ret = t8142_link_up(xbar, index); + else + ret = t8142_link_down(xbar, index); + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} + + static int apple_dpxbar_t602x_link_up(struct mux_control *mux) { struct apple_dpxbar *xbar; @@ -961,6 +1194,9 @@ static int apple_dpxbar_t6030_link_set(struct mux_control *mux, bool up) int apple_dpxbar_link_up(struct mux_control *mux) { + if (mux && mux->chip->ops == &apple_dpxbar_t8142_ops) + return apple_dpxbar_t8142_link(mux, true); + if (!mux) return -EINVAL; if (mux->chip->ops == &apple_dpxbar_t602x_ops) { @@ -1009,6 +1245,9 @@ static int apple_dpxbar_t602x_link_down(struct mux_control *mux) int apple_dpxbar_link_down(struct mux_control *mux) { + if (mux && mux->chip->ops == &apple_dpxbar_t8142_ops) + return apple_dpxbar_t8142_link(mux, false); + if (!mux) return -EINVAL; if (mux->chip->ops == &apple_dpxbar_t602x_ops) { @@ -1022,6 +1261,39 @@ int apple_dpxbar_link_down(struct mux_control *mux) } EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); +static void apple_dpxbar_domain_release(void *data) +{ + struct device *pd = data; + + pm_runtime_put_sync_suspend(pd); + dev_pm_domain_detach(pd, true); +} + +static int apple_dpxbar_domains(struct device *dev) +{ + static const char * const names[] = { "usb-aon", "common" }; + int ret; + + /* Older handoff-only trees have no Linux-managed domains. */ + if (!of_property_present(dev->of_node, "power-domain-names")) + return 0; + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct device *pd = dev_pm_domain_attach_by_name(dev, names[i]); + + if (IS_ERR_OR_NULL(pd)) + return pd ? PTR_ERR(pd) : -ENODEV; + ret = pm_runtime_resume_and_get(pd); + if (ret < 0) { + dev_pm_domain_detach(pd, true); + return ret; + } + ret = devm_add_action_or_reset(dev, apple_dpxbar_domain_release, pd); + if (ret) + return ret; + } + return 0; +} + static int apple_dpxbar_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1031,7 +1303,16 @@ static int apple_dpxbar_probe(struct platform_device *pdev) int ret; hw = of_device_get_match_data(dev); - mux_chip = devm_mux_chip_alloc(dev, MUX_MAX, sizeof(*dpxbar)); + if (hw->ops == &apple_dpxbar_t8142_ops) { + struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + if (!res || resource_size(res) < 0x830) + return -EINVAL; + ret = apple_dpxbar_domains(dev); + if (ret) + return dev_err_probe(dev, ret, "cannot power crossbar controls\n"); + } + mux_chip = devm_mux_chip_alloc(dev, hw->n_mux, sizeof(*dpxbar)); if (IS_ERR(mux_chip)) return PTR_ERR(mux_chip); @@ -1052,14 +1333,14 @@ static int apple_dpxbar_probe(struct platform_device *pdev) return ret; } - if (!hw->no_tunable && + if (hw->ops == &apple_dpxbar_ops && !hw->no_tunable && !of_device_is_compatible(dev->of_node, "apple,t6020-display-crossbar")) { readl(dpxbar->regs + UNK_TUNABLE); writel(hw->tunable, dpxbar->regs + UNK_TUNABLE); readl(dpxbar->regs + UNK_TUNABLE); } - for (unsigned int i = 0; i < MUX_MAX; ++i) { + for (unsigned int i = 0; i < hw->n_mux; ++i) { mux_chip->mux[i].states = hw->n_ufp; mux_chip->mux[i].idle_state = MUX_IDLE_DISCONNECT; dpxbar->selected_dispext[i] = -1; @@ -1124,18 +1405,21 @@ static int apple_dpxbar_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_dpxbar_pm_ops, NULL, apple_dpxbar_resume); static const struct apple_dpxbar_hw apple_dpxbar_hw_t8103 = { + .n_mux = MUX_MAX, .n_ufp = 2, .tunable = 0, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t8112 = { + .n_mux = MUX_MAX, .n_ufp = 4, .tunable = 4278196325, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { + .n_mux = MUX_MAX, .n_ufp = 9, .tunable = 5, .ops = &apple_dpxbar_ops, @@ -1143,6 +1427,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, }; @@ -1155,6 +1440,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { * The ADT lists nine UFP endpoints (dispext0-3, scodec0 and the DP IN ports). */ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6041 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, .t6030_flow = true, @@ -1170,6 +1456,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6041 = { * output is selected; probe touches no register. */ static const struct apple_dpxbar_hw apple_dpxbar_hw_t8132 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, .t6030_flow = true, @@ -1177,7 +1464,23 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t8132 = { .runtime_pm = true, }; +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8142 = { + .n_mux = T8142_MUX_MAX, + .n_ufp = 8, + .ops = &apple_dpxbar_t8142_ops, +}; + +/* T6050: the T8142 layout; a status that stays busy is logged after settling. */ +static const struct apple_dpxbar_hw apple_dpxbar_hw_t6050 = { + .n_mux = T8142_MUX_MAX, + .n_ufp = 8, + .ops = &apple_dpxbar_t8142_ops, + .status_log_only = true, +}; + static const struct of_device_id apple_dpxbar_ids[] = { + { .compatible = "apple,t6050-display-crossbar", .data = &apple_dpxbar_hw_t6050 }, + { .compatible = "apple,t8142-display-crossbar", .data = &apple_dpxbar_hw_t8142 }, { .compatible = "apple,t8103-display-crossbar", .data = &apple_dpxbar_hw_t8103, diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig index d3ead4d9259e7..d9b13a9785de2 100644 --- a/drivers/net/wireless/apple/Kconfig +++ b/drivers/net/wireless/apple/Kconfig @@ -22,4 +22,23 @@ config APPLE_CENTAURI This driver does not include Bluetooth HCI support. To build as a module, choose M; the module is apple_centauri. + +config APPLE_CENTAURI_T6050 + tristate "Apple N1 Centauri T6050 firmware and IPC transport" + depends on ARCH_APPLE || COMPILE_TEST + depends on PCI && CFG80211 && PCIE_APPLE + select FW_LOADER + help + Boot the N1 firmware on a T6050 host that provides root-port reset. + The transport starts automatically from its device-tree firmware, + platform and protocol properties. It binds only behind a T6050 host; + the J813/J873 transport remains available independently. + +config APPLE_CENTAURI_T6050_WIFI + bool "T6050 N1 Centauri cfg80211 station interface" + depends on APPLE_CENTAURI_T6050 + default y + help + Add scanning and station connections for the T6050 protocol profile. + endif diff --git a/drivers/net/wireless/apple/Makefile b/drivers/net/wireless/apple/Makefile index ca72cd0b4da06..969c58e0ebe98 100644 --- a/drivers/net/wireless/apple/Makefile +++ b/drivers/net/wireless/apple/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_APPLE_CENTAURI) += centauri/ +obj-y += centauri/ diff --git a/drivers/net/wireless/apple/centauri/Makefile b/drivers/net/wireless/apple/centauri/Makefile index 1a17e5283e227..235e1090294dc 100644 --- a/drivers/net/wireless/apple/centauri/Makefile +++ b/drivers/net/wireless/apple/centauri/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_CENTAURI) += apple_centauri.o apple_centauri-y := centauri_pci.o centauri_port.o centauri_ipc.o centauri_wifi.o +obj-$(CONFIG_APPLE_CENTAURI_T6050) += t6050/ diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 86ad99c48db2e..57fefbc840c6f 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -197,6 +197,12 @@ static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) dma_addr_t dma; void *page; int ret; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + /* The T6050 transport owns every function behind an adopted host. */ + if (bridge && bridge->dev.parent && + of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie")) + return -ENODEV; c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); if (!c) diff --git a/drivers/net/wireless/apple/centauri/t6050/Makefile b/drivers/net/wireless/apple/centauri/t6050/Makefile new file mode 100644 index 0000000000000..ec75629ea1272 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_CENTAURI_T6050) += apple_centauri_t6050.o +apple_centauri_t6050-y := centauri_pci.o centauri_ipc.o +apple_centauri_t6050-$(CONFIG_APPLE_CENTAURI_T6050_WIFI) += centauri_wifi.o diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c b/drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c new file mode 100644 index 0000000000000..2ff1fbf61b85a --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri_t6050.h" + +struct cen_buffer { + struct list_head list; + struct device *owner; + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct cen_context { + struct cen_buffer *buffer, *messages; + u32 index[4], mtr, mcr; + u16 tr_count, cr_count, message_head; +}; + +struct centauri_t6050_ipc { + struct list_head buffers; + struct pci_dev *function[3]; + bool enabled[3], owned[3]; + bool alpha_pm_domain; + void __iomem *bar[3]; + struct cen_context control, alpha; + struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; + u16 cchi_head, cchi_tail; +}; + +static struct cen_buffer *buffer_alloc(struct centauri_t6050 *c, unsigned int function, + size_t size) +{ + struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + + if (!buffer) + return NULL; + buffer->owner = get_device(&c->ipc->function[function]->dev); + buffer->size = PAGE_ALIGN(size); + buffer->cpu = dma_alloc_coherent(buffer->owner, buffer->size, &buffer->dma, GFP_KERNEL); + if (!buffer->cpu) { + put_device(buffer->owner); + kfree(buffer); + return NULL; + } + memset(buffer->cpu, 0, buffer->size); + list_add_tail(&buffer->list, &c->ipc->buffers); + return buffer; +} + +static int ftab_entry(const void *image, size_t size, const char tag[4], + u32 *offset, u32 *length, bool sparse) +{ + u32 count, i, off, len; + const u8 *table = image; + + if (size < 0x30 || memcmp(table + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(table + 0x28); + if (!count || count > 256 || 0x30 + count * 16 > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = table + 0x30 + 16 * i; + + if (memcmp(entry, tag, 4)) + continue; + off = get_unaligned_le32(entry + 4); + len = get_unaligned_le32(entry + 8); + if (off < 0x30 + count * 16 || !len || (u64)off + len > (sparse ? SZ_64M : size)) + return -EINVAL; + *offset = off; + *length = len; + return 0; + } + return -ENOENT; +} + +static int secondary_init(struct centauri_t6050 *c) +{ + u32 offset, length, count, i, required; + const u8 *seed; + int ret; + + ret = ftab_entry(c->firmware, c->firmware_size, "2ftb", &offset, &length, false); + if (ret) + return ret; + seed = c->firmware + offset; + if (length < 0x30 || memcmp(seed + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(seed + 0x28); + if (!count || count > 256 || 0x30 + 16 * count != length) + return -EINVAL; + required = length; + for (i = 0; i < count; i++) { + u64 end = (u64)get_unaligned_le32(seed + 0x34 + i * 16) + + get_unaligned_le32(seed + 0x38 + i * 16); + + if (end > SZ_64M) + return -EINVAL; + required = max_t(u32, required, end); + } + c->ipc->secondary = buffer_alloc(c, 0, required); + if (!c->ipc->secondary) + return -ENOMEM; + memcpy(c->ipc->secondary->cpu, seed, length); + dma_wmb(); + ret = pci_write_config_dword(c->pdev, 0xf88, lower_32_bits(c->ipc->secondary->dma)); + if (!ret) + ret = pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); + if (!ret) + ret = pci_write_config_dword(c->pdev, 0xf90, required); + if (ret) + return pcibios_err_to_errno(ret); + + return 0; +} + +static int context_init(struct centauri_t6050 *c, struct cen_context *ctx, + unsigned int function, u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) +{ + u8 *header; + dma_addr_t dma; + int i; + + ctx->buffer = buffer_alloc(c, function, SZ_32K); + ctx->messages = buffer_alloc(c, function, SZ_16K); + if (!ctx->buffer || !ctx->messages) + return -ENOMEM; + ctx->tr_count = tr; + ctx->cr_count = cr; + ctx->index[0] = 0x78; + ctx->index[1] = ctx->index[0] + cr * 2; + ctx->index[2] = ctx->index[1] + tr * 2; + ctx->index[3] = ctx->index[2] + cr * 2; + ctx->mtr = ALIGN(ctx->index[3] + tr * 2, tr == 192 ? 16 : 2); + ctx->mcr = ALIGN(ctx->mtr + 256, 4); + header = ctx->buffer->cpu; + dma = ctx->buffer->dma; + put_unaligned_le16(768, header); + put_unaligned_le16(0x68, header + 2); + put_unaligned_le64(dma + 0x68, header + 8); + for (i = 0; i < 4; i++) + put_unaligned_le64(dma + ctx->index[i], header + 0x10 + 8 * i); + put_unaligned_le16(cr, header + 0x30); + put_unaligned_le16(tr, header + 0x32); + put_unaligned_le64(dma + ctx->mcr, header + 0x34); + put_unaligned_le64(dma + ctx->mtr, header + 0x3c); + put_unaligned_le16(16, header + 0x44); + put_unaligned_le16(16, header + 0x46); + put_unaligned_le64(scratch ?: dma + ALIGN(ctx->mcr + 256, 8), header + 0x58); + put_unaligned_le64(scratch_size, header + 0x60); + dma_wmb(); + return 0; +} + +static int handshake(void __iomem *bar, struct cen_context *ctx, + u32 status, u32 address, u32 control) +{ + u32 value; + int ret; + + writel(1, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); + if (ret) { + pr_err("centauri_t6050: IPC init timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; + } + writel(lower_32_bits(ctx->buffer->dma), bar + address); + writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); + dma_wmb(); + writel(2, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); + if (ret) + pr_err("centauri_t6050: IPC run timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; +} + +static int message(struct cen_context *ctx, void __iomem *doorbell, + const void *payload, size_t length) +{ + u16 head = ctx->message_head; + u8 *descriptor; + u16 tail; + int ret; + + if (head >= 15 || !length || length > 512) + return -ENOSPC; + memcpy(ctx->messages->cpu + 512 * (head + 1), payload, length); + descriptor = ctx->buffer->cpu + ctx->mtr + head * 16; + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(ctx->messages->dma + 512 * (head + 1), descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + dma_wmb(); + writel(head + 1, doorbell); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) == head + 1, + 100, 1200000, false, *(__le16 *)(ctx->buffer->cpu + ctx->index[1])); + dma_rmb(); + ctx->message_head++; + return ret; +} + +static int open_transfer(struct centauri_t6050 *c, u16 id, struct cen_buffer *buffer, + u16 footer) +{ + u8 msg[0x34] = { 1 }; + u32 words = footer >> 2, shift = min_t(u32, __ffs(words), 4); + + msg[2] = words >> shift; + msg[3] = shift << 4; + put_unaligned_le16(id, msg + 4); + put_unaligned_le64(buffer->dma, msg + 8); + put_unaligned_le64(~0ULL, msg + 0x10); + put_unaligned_le16(16, msg + 0x18); + put_unaligned_le16(0xffff, msg + 0x1a); + put_unaligned_le16(id, msg + 0x1c); + put_unaligned_le16(0x40, msg + 0x1e); + return message(&c->ipc->control, c->bar + 0x9000, msg, sizeof(msg)); +} + +static int cchi_init(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + u8 config[4] = {9, 0, 0xff, 0xff}, *entry; + u16 tail; + int i, ret; + + ipc->cchi_rx = buffer_alloc(c, 0, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, 0, SZ_32K); + ipc->debug_rx = buffer_alloc(c, 0, SZ_16K); + if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) + return -ENOMEM; + ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); + if (ret) + return ret; + ret = open_transfer(c, 4, ipc->cchi_rx, 1024); + if (!ret) + ret = open_transfer(c, 3, ipc->cchi_tx, 1024); + if (!ret) + ret = open_transfer(c, 5, ipc->debug_rx, 256); + if (ret) + return ret; + for (i = 0; i < 15; i++) { + entry = ipc->cchi_rx->cpu + i * 1040; + put_unaligned_le32(1 | (1024 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + entry = ipc->debug_rx->cpu + i * 272; + put_unaligned_le32(1 | (256 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + } + dma_wmb(); + writel(15, c->bar + 0x9010); + writel(15, c->bar + 0x9014); + entry = ipc->cchi_tx->cpu; + put_unaligned_le32(1 | (64 << 8), entry); + put_unaligned_le32(1, entry + 12); + put_unaligned_le16(64, entry + 16); + dma_wmb(); + writel(1, c->bar + 0x900c); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) != 0, + 100, 2000000, false, + *(__le16 *)(ipc->control.buffer->cpu + ipc->control.index[1] + 4 * 2)); + dma_rmb(); + if (ret) + return ret; + entry = ipc->cchi_rx->cpu; + if ((get_unaligned_le32(entry) >> 8) < 8 || entry[18] || entry[19]) + return -EPROTO; + ipc->cchi_head = ipc->cchi_tail = 1; + return 0; +} + +static int memswap_handoff(struct centauri_t6050 *c, unsigned int function, const char tag[4]) +{ + struct cen_buffer *secondary = c->ipc->secondary, *working; + u32 offset, size; + int ret; + + ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); + if (ret) + return ret; + working = buffer_alloc(c, function, size); + if (!working) + return -ENOMEM; + dma_rmb(); + memcpy(working->cpu, secondary->cpu + offset, size); + dma_wmb(); + ret = pci_write_config_dword(c->ipc->function[function], 0xf88, lower_32_bits(working->dma)); + if (!ret) + ret = pci_write_config_dword(c->ipc->function[function], 0xf8c, upper_32_bits(working->dma)); + if (!ret) + ret = pci_write_config_dword(c->ipc->function[function], 0xf90, size); + if (ret) + return pcibios_err_to_errno(ret); + return 0; +} + +static struct dev_pm_domain centauri_t6050_alpha_pm_domain; + +static int enumerate_functions(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + int i, ret; + + c->pdev->multifunction = 1; + pci_rescan_bus(c->pdev->bus); + for (i = 0; i < 3; i++) { + ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), + c->pdev->bus->number, PCI_DEVFN(PCI_SLOT(c->pdev->devfn), i)); + if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || + (i && ipc->function[i]->device != 0x1901 + i)) + return -ENODEV; + if (ipc->function[i]->cfg_size < 0xf94 || + !(pci_resource_flags(ipc->function[i], 0) & IORESOURCE_MEM) || + pci_resource_len(ipc->function[i], 0) < (i ? 0x2000 : 0x9080)) + return -ENODEV; + if (i && ipc->function[i]->dev.driver) + return -EBUSY; + ipc->owned[i] = true; + ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); + if (!ipc->bar[i]) + return -ENOMEM; + if (i) { + ret = pci_enable_device_mem(ipc->function[i]); + if (ret) + return ret; + ipc->enabled[i] = true; + } + if (i == 1 && !ipc->function[i]->dev.driver) { + dev_pm_domain_set(&ipc->function[i]->dev, ¢auri_t6050_alpha_pm_domain); + ipc->alpha_pm_domain = true; + } + ret = dma_set_mask_and_coherent(&ipc->function[i]->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + if (i == 1) { + ret = centauri_t6050_request_irqs(&c->alpha_irqs, ipc->function[i], + 16, centauri_t6050_alpha_irq, c); + if (ret) + return ret; + } + pci_set_master(ipc->function[i]); + } + return 0; +} + +static int centauri_t6050_start_ipc_locked(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc; + u32 stage; + int ret; + + if (!c->firmware || !c->link_ready || c->ipc || readl(c->bar + 0x8000) != 1) + return -EBUSY; + ipc = kzalloc(sizeof(*ipc), GFP_KERNEL); + if (!ipc) + return -ENOMEM; + INIT_LIST_HEAD(&ipc->buffers); + c->ipc = ipc; + ret = enumerate_functions(c); + if (ret) + goto failed; + ret = secondary_init(c); + if (ret) + goto failed; + ret = context_init(c, &ipc->control, 0, 13, 1, 0, 0); + if (ret) + goto failed; + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + writel(0x62746632, c->bar + 0x8074); + dma_wmb(); + writel(1, c->bar + 0x9070); + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage == 2 || stage == 3, 100, 3000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ret = handshake(c->bar, &ipc->control, 0x8050, 0x8054, 0x907c); + if (ret) + goto failed; + ret = memswap_handoff(c, 0, "mswc"); + if (!ret) + ret = memswap_handoff(c, 2, "mswb"); + if (!ret) + ret = memswap_handoff(c, 1, "msww"); + if (ret) + goto failed; + ret = cchi_init(c); + if (ret) + goto failed; + ret = readl_poll_timeout(ipc->bar[1] + 8, stage, stage == 2 || stage == 3, 100, 2000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ipc->window = buffer_alloc(c, 1, CEN_ALPHA_WINDOW_SIZE); + if (!ipc->window) { + ret = -ENOMEM; + goto failed; + } + ret = context_init(c, &ipc->alpha, 1, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); + if (ret) + goto failed; + writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); + writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); + writel(CEN_ALPHA_WINDOW_SIZE, ipc->bar[1] + 0x20); + ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); + if (ret) + goto failed; + ret = centauri_t6050_wifi_register(c); + if (ret) + return ret; + + /* Publish usable shared firmware before the Beta consumer probes. */ + smp_store_release(&c->ready, true); + ret = device_attach(&ipc->function[2]->dev); + if (ret < 0) + dev_warn(&c->pdev->dev, "Beta driver attachment failed: %d\n", ret); + return 0; +failed: + dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", + ret, readl(c->bar + 0x8000)); + if (ipc->bar[1]) + dev_err(&c->pdev->dev, "Alpha stage=%#x IPC=%#x window=%#x/%#x/%#x\n", + readl(ipc->bar[1] + 8), readl(ipc->bar[1] + 0xc), + readl(ipc->bar[1] + 0x18), readl(ipc->bar[1] + 0x1c), readl(ipc->bar[1] + 0x20)); + if (ipc->bar[2]) + dev_err(&c->pdev->dev, "Beta stage=%#x IPC=%#x\n", + readl(ipc->bar[2] + 0x18), readl(ipc->bar[2] + 0x1c)); + + return ret; +} + +int centauri_t6050_start_ipc(struct centauri_t6050 *c) +{ + int ret; + + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + ret = READ_ONCE(c->removing) ? -ECANCELED : centauri_t6050_start_ipc_locked(c); + pci_unlock_rescan_remove(); + return ret; +} + +void centauri_t6050_free_ipc(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + struct cen_buffer *buffer, *next; + int i; + + if (!ipc) + return; + if (ipc->owned[2]) + device_release_driver(&ipc->function[2]->dev); + centauri_t6050_wifi_unregister(c); + centauri_t6050_free_irqs(&c->alpha_irqs); + for (i = 0; i < 3; i++) { + if (!ipc->owned[i]) + continue; + pci_clear_master(ipc->function[i]); + } + for (i = 0; i < 3; i++) { + if (ipc->owned[i] && !centauri_t6050_stop_dma(ipc->function[i])) { + dev_warn(&c->pdev->dev, + "retaining IPC DMA after pending transaction timeout\n"); + return; + } + } + list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { + memzero_explicit(buffer->cpu, buffer->size); + dma_free_coherent(buffer->owner, buffer->size, buffer->cpu, buffer->dma); + put_device(buffer->owner); + list_del(&buffer->list); + kfree(buffer); + } + if (ipc->alpha_pm_domain) + dev_pm_domain_set(&ipc->function[1]->dev, NULL); + for (i = 0; i < 3; i++) { + if (ipc->bar[i]) + pci_iounmap(ipc->function[i], ipc->bar[i]); + if (ipc->enabled[i]) + pci_disable_device(ipc->function[i]); + pci_dev_put(ipc->function[i]); + } + kfree(ipc); + c->ipc = NULL; +} + +int centauri_t6050_alpha_resources(struct centauri_t6050 *c, struct cen_alpha_resources *resources) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) + return -ENODEV; + resources->pdev = ipc->function[1]; + resources->context = ipc->alpha.buffer->cpu; + resources->context_dma = ipc->alpha.buffer->dma; + resources->messages = ipc->alpha.messages->cpu; + resources->messages_dma = ipc->alpha.messages->dma; + resources->window = ipc->window->cpu; + resources->window_dma = ipc->window->dma; + resources->bar = ipc->bar[1]; + memcpy(resources->index, ipc->alpha.index, sizeof(resources->index)); + resources->mtr = ipc->alpha.mtr; + resources->mcr = ipc->alpha.mcr; + return 0; +} diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_pci.c b/drivers/net/wireless/apple/centauri/t6050/centauri_pci.c new file mode 100644 index 0000000000000..9fd9d5bd1d8e8 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_pci.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "centauri_t6050.h" + +static struct pci_driver centauri_t6050_driver; + +bool apple_centauri_control_ready(struct pci_dev *pdev) +{ + struct centauri_t6050 *c; + + if (pdev->driver != ¢auri_t6050_driver) + return false; + c = pci_get_drvdata(pdev); + return c && !READ_ONCE(c->removing) && smp_load_acquire(&c->ready); +} +EXPORT_SYMBOL_GPL(apple_centauri_control_ready); + +/* No DMA backing is released without verified bus-master shutdown. */ +bool centauri_t6050_stop_dma(struct pci_dev *pdev) +{ + u16 command; + int ret; + + pci_clear_master(pdev); + ret = pci_read_config_word(pdev, PCI_COMMAND, &command); + if (ret || (command & PCI_COMMAND_MASTER)) + return false; + return pci_wait_for_pending_transaction(pdev); +} + +void centauri_t6050_free_irqs(struct centauri_t6050_irqs *irqs) +{ + if (!irqs->pdev) + return; + while (irqs->count) + free_irq(pci_irq_vector(irqs->pdev, --irqs->count), irqs->data); + pci_free_irq_vectors(irqs->pdev); + irqs->pdev = NULL; +} + +int centauri_t6050_request_irqs(struct centauri_t6050_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data) +{ + int ret; + + ret = pci_alloc_irq_vectors(pdev, count, count, PCI_IRQ_MSI); + if (ret < 0) + return ret; + irqs->pdev = pdev; + irqs->data = data; + for (irqs->count = 0; irqs->count < count; irqs->count++) { + ret = request_irq(pci_irq_vector(pdev, irqs->count), handler, 0, + "centauri_t6050", data); + if (ret) { + centauri_t6050_free_irqs(irqs); + return ret; + } + } + return 0; +} + +static irqreturn_t centauri_t6050_control_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} + +static void centauri_t6050_free_boot(struct centauri_t6050 *c) +{ + struct device *dev = &c->pdev->dev; + + if (!c->link_ready && c->boot_attempted) { + while (c->alpha_irqs.count) + free_irq(pci_irq_vector(c->alpha_irqs.pdev, --c->alpha_irqs.count), c); + while (c->irqs.count) + free_irq(pci_irq_vector(c->pdev, --c->irqs.count), c); + dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); + return; + } + + centauri_t6050_free_ipc(c); + centauri_t6050_free_irqs(&c->irqs); + if (c->ipc || !centauri_t6050_stop_dma(c->pdev)) { + dev_warn(dev, "retaining boot DMA until outstanding transactions stop\n"); + return; + } + if (c->firmware) + dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); + if (c->descriptor) + dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); + c->firmware = NULL; + c->descriptor = NULL; +} + +static int centauri_t6050_rom_boot(struct centauri_t6050 *c) +{ + const struct firmware *fw; + struct device *dev = &c->pdev->dev; + u32 stage; + int ret; + + if (c->pdev->device != 0x1900 && c->pdev->device != 0x1901) + return -ENODEV; + if (!c->link_ready || c->boot_attempted) + return -EBUSY; + stage = readl(c->bar + 0x8000); + if (stage != 0) + return -EBUSY; + + ret = request_firmware_direct(&fw, c->firmware_name, dev); + if (ret) { + dev_err(dev, "required firmware %s unavailable: %d\n", c->firmware_name, ret); + return ret; + } + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } + if (fw->size < 0x30 || fw->size > SZ_64M || + memcmp(fw->data + 0x20, "rkosftab", 8)) { + ret = -EINVAL; + goto release; + } + c->firmware_size = PAGE_ALIGN(fw->size); + c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); + c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); + if (!c->firmware || !c->descriptor) { + ret = -ENOMEM; + goto free_buffers; + } + memset(c->firmware, 0, c->firmware_size); + memcpy(c->firmware, fw->data, fw->size); + memset(c->descriptor, 0, SZ_16K); + + put_unaligned_le64(c->firmware_dma, c->descriptor); + put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); + ret = centauri_t6050_request_irqs(&c->irqs, c->pdev, 4, + centauri_t6050_control_irq, c); + if (ret) + goto free_buffers; + pci_set_master(c->pdev); + writel(c->platform_id, c->bar + 0x807c); + writel(0, c->bar + 0x803c); + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + dma_wmb(); + c->boot_attempted = true; + writel(1, c->bar + 0x9000); + ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, + READ_ONCE(c->removing) || c->image_response != 0, 20000, 20000000); + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } + if (!ret && c->image_response != 1) + ret = -EIO; + if (!ret) + ret = readl_poll_timeout(c->bar + 0x8000, stage, + READ_ONCE(c->removing) || stage != 0, 20000, 20000000); + else + stage = readl(c->bar + 0x8000); + if (READ_ONCE(c->removing)) + ret = -ECANCELED; + dma_rmb(); + + if (!ret && stage != 4) + ret = -EIO; + goto release; +free_buffers: + centauri_t6050_free_boot(c); +release: + release_firmware(fw); + return ret; +} + +/* + * After the ROM boot the N1 runs its firmware from host memory and must come back on a + * retrained link: the isolated function 0 is quiesced, put in D3hot with PME enabled + * (the host's PME_Turn_Off then takes the link to L2), the host resets and retrains the + * root port (PERST# asserted and released around it; the firmware keeps running), and the + * root port's and the endpoint's state are restored. + */ +static int centauri_t6050_port_cycle(struct pci_dev *endpoint) +{ + struct pci_host_bridge *host = pci_find_host_bridge(endpoint->bus); + struct pci_dev *bridge = endpoint->bus->self, *sibling; + u16 command, pmctrl; + u32 identity; + int ret, pm; + + if (!host->reset_root_port || !bridge || !pci_is_root_bus(bridge->bus)) + return -EOPNOTSUPP; + pm = endpoint->pm_cap; + if (!pm) + return -ENODEV; + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + list_for_each_entry(sibling, &endpoint->bus->devices, bus_list) { + if (sibling != endpoint) { + ret = -EBUSY; + goto out; + } + } + ret = pci_save_state(bridge) ?: pci_save_state(endpoint); + if (ret) + goto out; + ret = pcibios_err_to_errno(pci_read_config_word(endpoint, PCI_COMMAND, &command)); + if (ret) + goto out; + if (!centauri_t6050_stop_dma(endpoint)) { + ret = -EBUSY; + goto stopped; + } + ret = pcibios_err_to_errno(pci_read_config_word(endpoint, pm + PCI_PM_CTRL, &pmctrl)); + if (ret) + goto stopped; + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + (pmctrl & ~PCI_PM_CTRL_STATE_MASK) | PCI_D3hot | PCI_PM_CTRL_PME_ENABLE)); + if (ret) + goto stopped; + usleep_range(10000, 11000); + + ret = host->reset_root_port(host, bridge); + if (ret) + goto stopped; + pci_restore_state(bridge); + ret = pcibios_err_to_errno(pci_read_config_dword(endpoint, PCI_VENDOR_ID, &identity)); + if (!ret && identity != 0x1900106b && identity != 0x1901106b) + ret = -ENODEV; + if (ret) + goto stopped; + pci_restore_state(endpoint); + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + pmctrl & ~PCI_PM_CTRL_STATE_MASK)); + if (ret) + goto stopped; + usleep_range(10000, 11000); + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, PCI_COMMAND, command)); + if (!ret) + goto out; +stopped: + /* A partial transition never restores endpoint DMA or bridge decoding. */ + pci_clear_master(endpoint); + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_err(endpoint, "N1 port cycle stopped: %d\n", ret); +out: + pci_unlock_rescan_remove(); + return ret; +} + +static int centauri_t6050_cycle(struct centauri_t6050 *c) +{ + int ret; + + if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) + return -EBUSY; + c->link_ready = false; + ret = centauri_t6050_port_cycle(c->pdev); + if (ret == -EAGAIN) { + c->link_ready = true; + } else if (!ret) { + pci_restore_msi_state(c->pdev); + c->link_ready = true; + } + return ret; +} + +static void centauri_t6050_boot_worker(struct work_struct *work) +{ + struct centauri_t6050 *c = container_of(to_delayed_work(work), struct centauri_t6050, boot_work); + bool retry = false; + int ret; + + mutex_lock(&c->boot_lock); + while (!READ_ONCE(c->removing)) { + switch (c->boot_phase) { + case CEN_BOOT_ROM: + ret = centauri_t6050_rom_boot(c); + if (!ret) + c->boot_phase = readl(c->bar + 0x8000) == 1 ? + CEN_BOOT_IPC : CEN_BOOT_CYCLE; + break; + case CEN_BOOT_CYCLE: + ret = centauri_t6050_cycle(c); + if (!ret) + c->boot_phase = CEN_BOOT_IPC; + break; + case CEN_BOOT_IPC: + ret = centauri_t6050_start_ipc(c); + if (!ret) + c->boot_phase = CEN_BOOT_READY; + break; + default: + goto out; + } + if (ret == -EAGAIN && (c->boot_phase == CEN_BOOT_CYCLE || + (c->boot_phase == CEN_BOOT_IPC && !c->ipc))) { + retry = true; + break; + } + if (ret) { + dev_err(&c->pdev->dev, "N1 automatic startup failed in phase%u: %d\n", + c->boot_phase, ret); + c->boot_phase = CEN_BOOT_FAILED; + break; + } + } +out: + if (READ_ONCE(c->removing)) + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); + if (retry && !READ_ONCE(c->removing)) + queue_delayed_work(system_long_wq, &c->boot_work, msecs_to_jiffies(20)); +} + +static int centauri_t6050_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct centauri_t6050 *c; + int ret; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + if (!bridge || !bridge->dev.parent || + !of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie")) + return -ENODEV; + if (PCI_FUNC(pdev->devfn) != 0) + return -ENODEV; + c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + c->pdev = pdev; + c->link_ready = true; + mutex_init(&c->boot_lock); + INIT_DELAYED_WORK(&c->boot_work, centauri_t6050_boot_worker); + c->boot_phase = CEN_BOOT_ROM; + ret = device_property_read_string(&pdev->dev, "firmware-name", &c->firmware_name); + if (ret) + return dev_err_probe(&pdev->dev, ret, "missing personalized firmware name\n"); + ret = device_property_read_u32(&pdev->dev, "apple,centauri-platform-id", &c->platform_id); + if (ret) + return ret; + ret = device_property_read_u32(&pdev->dev, "apple,centauri-protocol", &c->protocol); + if (ret || c->protocol != 27) + return -EINVAL; + ret = pcim_enable_device(pdev); + if (ret) + return ret; + c->size = pci_resource_len(pdev, 0); + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + c->size < 0x9080) + return dev_err_probe(&pdev->dev, -ENODEV, "invalid ACIPC BAR0\n"); + ret = pcim_iomap_regions(pdev, BIT(0), "apple_centauri_t6050"); + if (ret) + return ret; + c->bar = pcim_iomap_table(pdev)[0]; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "40-bit DMA unavailable\n"); + + pci_clear_master(pdev); + pci_set_drvdata(pdev, c); + queue_delayed_work(system_long_wq, &c->boot_work, 0); + return 0; +} + +static void centauri_t6050_remove(struct pci_dev *pdev) +{ + struct centauri_t6050 *c = pci_get_drvdata(pdev); + + WRITE_ONCE(c->removing, true); + cancel_delayed_work_sync(&c->boot_work); + mutex_lock(&c->boot_lock); + centauri_t6050_wifi_unregister(c); + centauri_t6050_free_boot(c); + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); +} + +static int centauri_t6050_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct centauri_t6050 *c = pci_get_drvdata(pdev); + + guard(mutex)(&c->boot_lock); + if (c->boot_phase != CEN_BOOT_READY) + return -EBUSY; + return pci_save_state(pdev); +} + +static int centauri_t6050_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(centauri_t6050_pm_ops, centauri_t6050_suspend, centauri_t6050_resume); + +static const struct pci_device_id centauri_t6050_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900) }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901) }, + { } +}; +MODULE_DEVICE_TABLE(pci, centauri_t6050_ids); + +static struct pci_driver centauri_t6050_driver = { + .name = "apple_centauri_t6050", + .id_table = centauri_t6050_ids, + .probe = centauri_t6050_probe, + .remove = centauri_t6050_remove, + .shutdown = centauri_t6050_remove, + .driver.pm = pm_sleep_ptr(¢auri_t6050_pm_ops), +}; + +module_pci_driver(centauri_t6050_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Centauri PCI firmware and ACIPC transport"); diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h b/drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h new file mode 100644 index 0000000000000..dfc9c013596fb --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef AURORA_CENTAURI_T6050_H +#define AURORA_CENTAURI_T6050_H +#include +#include +#include +#include +#include +#include + +#define CEN_ALPHA_WINDOW_SIZE SZ_32M +#define CEN_ALPHA_SCRATCH_OFFSET SZ_8M +#define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) + +struct centauri_t6050_ipc; +struct centauri_t6050_wifi; +struct centauri_t6050_irqs { + struct pci_dev *pdev; + void *data; + unsigned int count; +}; + +struct cen_alpha_resources { + struct pci_dev *pdev; + void *context, *messages, *window; + dma_addr_t context_dma, messages_dma, window_dma; + void __iomem *bar; + u32 index[4], mtr, mcr; +}; +enum centauri_t6050_boot_phase { + CEN_BOOT_ROM, + CEN_BOOT_CYCLE, + CEN_BOOT_IPC, + CEN_BOOT_READY, + CEN_BOOT_FAILED, + CEN_BOOT_STOPPED, +}; + +struct centauri_t6050 { + struct pci_dev *pdev; + void __iomem *bar; + resource_size_t size; + u32 platform_id, protocol; + const char *firmware_name; + struct dentry *debug; + struct mutex boot_lock; + struct delayed_work boot_work; + enum centauri_t6050_boot_phase boot_phase; + bool removing, ready; + void *firmware, *descriptor; + struct centauri_t6050_irqs irqs, alpha_irqs; + dma_addr_t firmware_dma, descriptor_dma; + size_t firmware_size; + u32 image_response; + bool boot_attempted; + bool link_ready; + struct centauri_t6050_ipc *ipc; + struct centauri_t6050_wifi *wifi; +}; + +int centauri_t6050_request_irqs(struct centauri_t6050_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data); +void centauri_t6050_free_irqs(struct centauri_t6050_irqs *irqs); + +int centauri_t6050_start_ipc(struct centauri_t6050 *c); +void centauri_t6050_free_ipc(struct centauri_t6050 *c); +int centauri_t6050_alpha_resources(struct centauri_t6050 *c, struct cen_alpha_resources *resources); +bool centauri_t6050_stop_dma(struct pci_dev *pdev); + +#if IS_ENABLED(CONFIG_APPLE_CENTAURI_T6050_WIFI) +irqreturn_t centauri_t6050_alpha_irq(int irq, void *data); +int centauri_t6050_wifi_register(struct centauri_t6050 *c); +void centauri_t6050_wifi_unregister(struct centauri_t6050 *c); +#else +static inline irqreturn_t centauri_t6050_alpha_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} +static inline int centauri_t6050_wifi_register(struct centauri_t6050 *c) +{ + return 0; +} +static inline void centauri_t6050_wifi_unregister(struct centauri_t6050 *c) +{ +} +#endif +#endif diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c new file mode 100644 index 0000000000000..5deffa85c8168 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -0,0 +1,2144 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri_t6050.h" + +#define CR_BASE 0x100000 +#define TR_BASE 0x400000 +#define RING_SPAN 0x20000 +#define TX_BUFFER 0x600000 +#define BUFFER_SIZE 2048 +#define REPLY_SIZE (BUFFER_SIZE - 24) +/* The band plan: 2.4 GHz 1-14, 5 GHz 36-64, 100-144 and 149-177, 6 GHz 1-233. */ +#define CEN_CHANNELS (14 + 28 + 59) + +#define DATA_RX_BUFFER SZ_16M +#define DATA_TX_BUFFER (SZ_16M + SZ_8M) +#define DATA_RX_SLOTS 2048 +#define DATA_TX_SLOTS 2048 +#define DATA_TX_QUEUE_LIMIT DATA_TX_SLOTS + +static_assert(DATA_TX_BUFFER >= + CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER >= + CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER + DATA_RX_SLOTS * 4096 <= DATA_TX_BUFFER); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= + CEN_ALPHA_WINDOW_SIZE); + +struct centauri_t6050_wifi { + struct centauri_t6050 *control; + struct cen_alpha_resources a; + struct wiphy *wiphy; + struct wireless_dev wdev; + struct net_device *netdev; + struct work_struct scan_work; + struct work_struct connect_work; + struct work_struct link_work; + bool irq_ready; + struct cfg80211_scan_request *scan_request; + struct mutex lock; /* Serializes firmware commands and ring consumers. */ + + u16 main_head, command_head, sequence; + u32 rx_seen[5]; + u16 rx_next[5], rx_head[5]; + u16 reply_sequence, reply_status, reply_capture; + u32 reply_command, channels_seen, scans_done, beacons_seen; + bool reply_valid, registered, configured; + u8 *reply_buffer; + size_t reply_capacity, reply_size; + bool stopping, command_failed; + bool scan_abort, connect_abort; + u8 scan_sequence; + bool connecting, connected, connect_event, disconnect_event; + bool disconnect_pending; + u16 connect_status, scan_status, disconnect_status; + struct ieee80211_channel *connect_channel; + u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + size_t connect_size, request_ie_size, ssid_len; + u32 data_rx_received, data_rx_reposted, data_rx_bad; + struct sk_buff_head data_tx_queue; + u16 data_tx_length[DATA_TX_SLOTS]; + bool data_tx_busy[DATA_TX_SLOTS]; + u32 data_tx_submitted, data_tx_completed, data_tx_failed; + u32 data_tx_coalesced, data_tx_bad; + s8 station_signal; + bool station_signal_valid; + unsigned long station_signal_at, connected_at; + struct net_device_stats station_start; + u32 data_rings_open; + bool data_failed; + struct ieee80211_supported_band bands[3]; + struct ieee80211_channel channels[CEN_CHANNELS]; + struct ieee80211_sband_iftype_data iftype_6ghz; + /* The firmware's own reports, read at bring-up (system commands 19 and 8). */ + u8 hw_channels[REPLY_SIZE], system_caps[REPLY_SIZE]; + struct debugfs_blob_wrapper hw_channels_blob, system_caps_blob; + u8 link_status[REPLY_SIZE]; +}; + +static int query_link_status(struct centauri_t6050_wifi *wifi); + +static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; +static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; +static const struct { + enum nl80211_band band; + u8 first, last, step; +} band_plan[] = { + { NL80211_BAND_2GHZ, 1, 14, 1 }, { NL80211_BAND_5GHZ, 36, 64, 4 }, + { NL80211_BAND_5GHZ, 100, 144, 4 }, { NL80211_BAND_5GHZ, 149, 177, 4 }, + { NL80211_BAND_6GHZ, 1, 233, 4 }, +}; +static struct ieee80211_rate rates[] = { + { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, + { .bitrate = 110 }, { .bitrate = 60 }, { .bitrate = 90 }, + { .bitrate = 120 }, { .bitrate = 180 }, { .bitrate = 240 }, + { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, +}; + +static const u32 akm_suites[] = { WLAN_AKM_SUITE_PSK, WLAN_AKM_SUITE_SAE }; + +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_CCMP, + WLAN_CIPHER_SUITE_AES_CMAC, +}; + +/* The firmware's band numbers: 0 is 2.4 GHz, 1 is 5 GHz, 2 is 6 GHz. */ +static int firmware_band(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return 0; + case NL80211_BAND_5GHZ: + return 1; + case NL80211_BAND_6GHZ: + return 2; + default: + return -EOPNOTSUPP; + } +} + +static u16 index_read(struct centauri_t6050_wifi *wifi, unsigned int array, + unsigned int index) +{ + return le16_to_cpu(READ_ONCE(*( + __le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index))); +} + +static void index_write(struct centauri_t6050_wifi *wifi, unsigned int array, + unsigned int index, u16 value) +{ + WRITE_ONCE( + *(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index), + cpu_to_le16(value)); +} + +static void completions(struct centauri_t6050_wifi *wifi) +{ + unsigned int vector; + + for (vector = 0; vector <= 4; vector++) + index_write(wifi, 2, vector, index_read(wifi, 0, vector)); + dma_wmb(); +} + +static int alpha_message(struct centauri_t6050_wifi *wifi, const void *message, + size_t size) +{ + u16 head = wifi->main_head; + u8 *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + if (head >= 15 || size > 512) + return -ENOSPC; + memcpy(wifi->a.messages + (head + 1) * 512, message, size); + descriptor = wifi->a.context + wifi->a.mtr + 16 * head; + put_unaligned_le32(1 | (size << 8), descriptor); + put_unaligned_le64(wifi->a.messages_dma + (head + 1) * 512, + descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + index_write(wifi, 3, 0, head + 1); + if (head) + index_write(wifi, 3, 8, head + 1); + dma_wmb(); + wifi->main_head++; + writel(1, wifi->a.bar + 0x1004); + do { + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + if (index_read(wifi, 1, 0) >= head + 1) { + dma_rmb(); + completions(wifi); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int open_pair(struct centauri_t6050_wifi *wifi, u16 id) +{ + u8 cr[0x2c] = { 2 }, tr[0x34] = { 1 }; + int ret; + + put_unaligned_le32(id | (id << 16), cr + 4); + put_unaligned_le64(wifi->a.window_dma + CR_BASE + (id - 1) * RING_SPAN, + cr + 8); + put_unaligned_le16(1024, cr + 0x10); + put_unaligned_le16(0xffff, cr + 0x12); + ret = alpha_message(wifi, cr, sizeof(cr)); + if (ret) + return ret; + put_unaligned_le32(id | (id << 16), tr + 4); + put_unaligned_le64(wifi->a.window_dma + TR_BASE + (id - 1) * RING_SPAN, + tr + 8); + put_unaligned_le64(~0ULL, tr + 0x10); + put_unaligned_le16(ring_size[id], tr + 0x18); + put_unaligned_le16(id, tr + 0x1a); + put_unaligned_le16(id, tr + 0x1c); + put_unaligned_le16(0x40, tr + 0x1e); + put_unaligned_le16(2, tr + 0x24); + return alpha_message(wifi, tr, sizeof(tr)); +} + +static void report_beacon(struct centauri_t6050_wifi *wifi, const u8 *payload, + size_t size) +{ + const u8 *frame, *ie, *end; + struct ieee80211_channel *channel; + struct cfg80211_bss *bss; + u16 frame_size; + int number = 0, frequency, signal; + enum nl80211_band band = NL80211_BAND_2GHZ; + + if (!wifi->registered || size < 0x60 + 36) + return; + + frame_size = size - 0x60; + frame = payload + 0x60; + if (!ieee80211_is_beacon(*(__le16 *)frame) && + !ieee80211_is_probe_resp(*(__le16 *)frame)) + return; + end = frame + frame_size; + for (ie = frame + 36; ie + 2 <= end && ie + 2 + ie[1] <= end; + ie += 2 + ie[1]) { + if ((ie[0] == WLAN_EID_DS_PARAMS || + ie[0] == WLAN_EID_HT_OPERATION) && + ie[1]) { + number = ie[2]; + band = number <= 14 ? NL80211_BAND_2GHZ : + NL80211_BAND_5GHZ; + } else if (ie[0] == WLAN_EID_EXTENSION && ie[1] > 1 && + ie[2] == WLAN_EID_EXT_HE_OPERATION && + ie[1] - 1 >= sizeof(struct ieee80211_he_operation) && + ie[1] >= ieee80211_he_oper_size(ie + 3)) { + /* 6 GHz beacons carry neither: the HE operation names the channel. */ + const struct ieee80211_he_6ghz_oper *oper = + ieee80211_he_6ghz_oper((const void *)(ie + 3)); + + if (oper && !number) { + number = oper->primary; + band = NL80211_BAND_6GHZ; + } + } + } + if (!number) + return; + frequency = ieee80211_channel_to_frequency(number, band); + channel = ieee80211_get_channel(wifi->wiphy, frequency); + if (!channel) + return; + signal = max_t(s8, payload[0x16], payload[0x17]); + if (signal >= 0 || signal < -127) + signal = -95; + bss = cfg80211_inform_bss_frame(wifi->wiphy, channel, + (struct ieee80211_mgmt *)frame, + frame_size, signal * 100, GFP_KERNEL); + if (bss) { + wifi->beacons_seen++; + cfg80211_put_bss(wifi->wiphy, bss); + } +} + +static void post_receive(struct centauri_t6050_wifi *wifi, unsigned int ring, + u16 slot) +{ + u8 *descriptor = + wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u32 offset = rx_buffer[ring] + slot * BUFFER_SIZE; + + memset(wifi->a.window + offset, 0, BUFFER_SIZE); + put_unaligned_le32(1 | (BUFFER_SIZE << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + offset, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); +} + +static bool data_rx_frame(const u8 *packet, unsigned int capacity, + unsigned int length, u32 footer, unsigned int *offset, + unsigned int *frame_length) +{ + unsigned int padding = (footer >> 12) & 15; + + if (!(footer & BIT(7))) + return false; + if (length < padding + ETH_HLEN || length > capacity) + return false; + if (footer & BIT(24)) { + if (capacity < 4) + return false; + *offset = get_unaligned_le16(packet + 2); + } else { + *offset = padding; + } + *frame_length = length - padding; + return *offset <= capacity && *frame_length <= capacity - *offset; +} + +static void drain_data_rx(struct centauri_t6050_wifi *wifi) +{ + u16 head, tail, post; + unsigned int processed = 0; + + if (!(wifi->data_rings_open & BIT(4)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 207); + tail = index_read(wifi, 2, 207); + post = index_read(wifi, 3, 12); + if (head >= DATA_RX_SLOTS || tail >= DATA_RX_SLOTS || + post >= DATA_RX_SLOTS) { + wifi->data_rx_bad++; + return; + } + dma_rmb(); + while (tail != head && processed < (DATA_RX_SLOTS - 1)) { + u8 *completion = wifi->a.window + 0x240000 + tail * 32; + u32 header = get_unaligned_le32(completion); + u32 info = get_unaligned_le32(completion + 4); + u16 cookie = info & 0xffff, length = info >> 16; + u16 next = (post + 1) % DATA_RX_SLOTS; + u8 *packet, *descriptor; + u32 buffer; + + if ((header >> 16) != 12 || !cookie || + cookie > (DATA_RX_SLOTS - 1) || length > 4096 || + get_unaligned_le32(completion + 8)) { + wifi->data_rx_bad++; + break; + } + + if (next == index_read(wifi, 1, 12)) + break; + buffer = DATA_RX_BUFFER + (cookie - 1) * 4096; + packet = wifi->a.window + buffer; + + wifi->data_rx_received++; + if (wifi->connected && netif_running(wifi->netdev)) { + u32 footer = get_unaligned_le32(completion + 12); + unsigned int offset, frame_length; + struct sk_buff *skb = NULL; + + if (data_rx_frame(packet, 4096, length, footer, &offset, + &frame_length)) + skb = netdev_alloc_skb_ip_align(wifi->netdev, + frame_length); + if (skb) { + skb_put_data(skb, packet + offset, + frame_length); + skb->protocol = + eth_type_trans(skb, wifi->netdev); + skb->ip_summed = CHECKSUM_NONE; + wifi->netdev->stats.rx_packets++; + wifi->netdev->stats.rx_bytes += frame_length; + netif_rx(skb); + } else { + wifi->netdev->stats.rx_dropped++; + } + } else { + wifi->netdev->stats.rx_dropped++; + } + descriptor = wifi->a.window + 0x520000 + post * 16; + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(cookie, descriptor + 12); + wifi->data_rx_reposted++; + tail = (tail + 1) % DATA_RX_SLOTS; + post = next; + processed++; + } + if (processed) { + dma_wmb(); + index_write(wifi, 2, 207, tail); + index_write(wifi, 3, 12, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +} + +static bool data_tx_completion_span(u16 cookie, u16 wire_count, u8 flags, + unsigned int *first, u16 *count) +{ + bool counted = (flags & BIT(0)) || (flags & 0x0e) == 8; + + if (!cookie || cookie > DATA_TX_SLOTS || counted != !!wire_count || + wire_count >= DATA_TX_SLOTS) + return false; + *count = counted ? wire_count : 1; + *first = (cookie + DATA_TX_SLOTS - *count) % DATA_TX_SLOTS; + return true; +} + +static void drain_data_tx(struct centauri_t6050_wifi *wifi) +{ + u16 head, tail, post; + unsigned int budget = DATA_TX_SLOTS; + bool posted = false, completed = false; + + if (wifi->command_failed || !(wifi->data_rings_open & BIT(5)) || + readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 192); + tail = index_read(wifi, 2, 192); + post = index_read(wifi, 3, 25); + if (head >= 2048 || tail >= 2048 || post >= DATA_TX_SLOTS) + return; + dma_rmb(); + while (tail != head && budget--) { + u8 *entry = wifi->a.window + 0x200000 + tail * 32; + u16 ring = get_unaligned_le32(entry) >> 16; + u16 cookie = get_unaligned_le16(entry + 4); + u16 count = get_unaligned_le16(entry + 9); + u8 status = entry[12]; + + if (ring == 25) { + unsigned int first, i; + + if (!data_tx_completion_span(cookie, count, entry[1], + &first, &count)) { + wifi->data_tx_bad++; + break; + } + for (i = 0; i < count; i++) + if (!wifi->data_tx_busy[(first + i) % + DATA_TX_SLOTS]) + break; + if (i != count) { + wifi->data_tx_bad++; + break; + } + if (count > 1) + wifi->data_tx_coalesced++; + for (i = 0; i < count; i++) { + unsigned int slot = (first + i) % DATA_TX_SLOTS; + + wifi->data_tx_completed++; + if (status == 0 || status == 0x83) { + wifi->netdev->stats.tx_packets++; + wifi->netdev->stats.tx_bytes += + wifi->data_tx_length[slot]; + } else { + wifi->data_tx_failed++; + wifi->netdev->stats.tx_errors++; + } + wifi->data_tx_busy[slot] = false; + } + } + tail = (tail + 1) % 2048; + completed = true; + } + if (completed) + index_write(wifi, 2, 192, tail); + budget = DATA_TX_SLOTS - 1; + while (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && budget--) { + u16 next = (post + 1) % DATA_TX_SLOTS; + u32 buffer = DATA_TX_BUFFER + post * BUFFER_SIZE; + u8 *descriptor = wifi->a.window + 0x540000 + post * 32; + struct sk_buff *skb; + u64 metadata = 6ULL << 27; + + if (wifi->data_tx_busy[post] || next == index_read(wifi, 1, 25)) + break; + skb = skb_dequeue(&wifi->data_tx_queue); + if (!skb) + break; + + if (skb_copy_bits(skb, 0, wifi->a.window + buffer, skb->len)) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (skb->len << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le16(post + 1, descriptor + 12); + + metadata |= 4ULL << 15; + put_unaligned_le64(metadata, descriptor + 16); + put_unaligned_le64(2000000, descriptor + 24); + wifi->data_tx_length[post] = skb->len; + wifi->data_tx_busy[post] = true; + wifi->data_tx_submitted++; + dev_kfree_skb_any(skb); + post = next; + posted = true; + } + if (posted || completed) { + dma_wmb(); + if (posted) + index_write(wifi, 3, 25, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + } + if (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && + skb_queue_len(&wifi->data_tx_queue) < DATA_TX_QUEUE_LIMIT) + netif_wake_queue(wifi->netdev); +} + +/* The interface status reply: 8 fixed bytes, then TLVs. */ +static bool link_signal_read(const u8 *packet, size_t size, s8 *signal) +{ + size_t pos = 8; + + while (pos <= size && size - pos >= 4) { + u16 tag = get_unaligned_le16(packet + pos); + u16 length = get_unaligned_le16(packet + pos + 2); + + if (length > size - pos - 4) + return false; + if (tag == 268 && length >= 115) { + *signal = (s8)packet[pos + 5]; + return *signal != 0; + } + pos += 4 + length; + } + return false; +} + +static void drain(struct centauri_t6050_wifi *wifi) +{ + unsigned int ring; + bool posted = false; + + for (ring = 2; ring <= 4; ring++) { + unsigned int budget = ring_size[ring]; + + while (budget--) { + u16 slot = wifi->rx_next[ring], size, status, sequence; + u32 command, wire_command; + u8 *descriptor = wifi->a.window + TR_BASE + + (ring - 1) * RING_SPAN + slot * 16; + u8 *packet = wifi->a.window + rx_buffer[ring] + + slot * BUFFER_SIZE; + + if (!(READ_ONCE(descriptor[15]) & 4)) + break; + dma_rmb(); + size = get_unaligned_le16(packet + 0x10); + status = get_unaligned_le16(packet + 0xe); + sequence = get_unaligned_le16(packet + 0x12); + wire_command = get_unaligned_le32(packet + 0x14); + command = wire_command & ~BIT(30); + wifi->rx_seen[ring]++; + wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; + if (size < 24 || size > BUFFER_SIZE) + goto refill; + if (ring == 4) { + report_beacon(wifi, packet + 24, size - 24); + } else if (command & BIT(31)) { + if (command == 0x80000030) + wifi->channels_seen++; + if (command == 0x80000005) { + wifi->scans_done++; + wifi->scan_status = status; + } + if (command == 0x80000000 && wifi->connecting) { + wifi->connect_event = true; + wifi->connect_status = status; + } + if (command == 0x80000003 || + command == 0x80000004) { + wifi->disconnect_event = true; + wifi->disconnect_status = status; + } + if (command == 0x80000004) + wifi->disconnect_pending = false; + + } else { + wifi->reply_valid = true; + wifi->reply_sequence = sequence; + wifi->reply_status = status; + + wifi->reply_command = wire_command; + if (wifi->reply_buffer && + sequence == wifi->reply_capture) { + wifi->reply_size = size - 24; + memcpy(wifi->reply_buffer, packet + 24, + min(wifi->reply_size, + wifi->reply_capacity)); + } + } +refill: + post_receive(wifi, ring, wifi->rx_head[ring]); + wifi->rx_head[ring] = + (wifi->rx_head[ring] + 1) % ring_size[ring]; + dma_wmb(); + index_write(wifi, 3, ring, wifi->rx_head[ring]); + posted = true; + } + } + completions(wifi); + if (posted) + writel(1, wifi->a.bar + 0x1004); + drain_data_rx(wifi); + drain_data_tx(wifi); +} + +/* Caller holds wifi->lock. Unknown completion keeps DMA payloads owned. */ +static void command_fault(struct centauri_t6050_wifi *wifi) +{ + wifi->command_failed = true; + wifi->configured = false; + wifi->data_failed = true; + netif_stop_queue(wifi->netdev); + if (smp_load_acquire(&wifi->irq_ready)) + schedule_work(&wifi->link_work); +} + +static bool command_payload_consumed(struct centauri_t6050_wifi *wifi, u16 slot) +{ + if (slot >= ring_size[1] || + index_read(wifi, 1, 1) != (slot + 1) % ring_size[1]) + return false; + dma_rmb(); + return true; +} + +/* + * Send one command and wait for its reply. With a reply buffer, the reply's + * payload is copied there (up to *reply_size bytes) and *reply_size becomes + * the payload's length. + */ +static int command_reply(struct centauri_t6050_wifi *wifi, u32 id, + const void *payload, size_t size, void *reply, + size_t *reply_size) +{ + u16 slot = wifi->command_head, sequence = ++wifi->sequence; + int ret; + u32 stage; + u8 *packet, *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(1500); + + if (wifi->command_failed) + return -EIO; + if (size > BUFFER_SIZE - 24) + return -EMSGSIZE; + if ((slot + 1) % ring_size[1] == index_read(wifi, 1, 1)) + return -ENOSPC; + packet = wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE; + memset(packet, 0, 24 + size); + put_unaligned_le16(24 + size, packet + 0x10); + put_unaligned_le16(sequence, packet + 0x12); + put_unaligned_le32(id, packet + 0x14); + if (size) + memcpy(packet + 24, payload, size); + descriptor = wifi->a.window + TR_BASE + slot * 16; + put_unaligned_le32(1 | ((24 + size) << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + TX_BUFFER + slot * BUFFER_SIZE, + descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + dma_wmb(); + wifi->command_head = (slot + 1) % ring_size[1]; + index_write(wifi, 3, 1, wifi->command_head); + wifi->reply_valid = false; + wifi->reply_buffer = reply; + wifi->reply_capacity = reply ? *reply_size : 0; + wifi->reply_capture = sequence; + wifi->reply_size = 0; + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + do { + if (READ_ONCE(wifi->stopping) || + READ_ONCE(wifi->control->removing)) { + command_fault(wifi); + ret = -ECANCELED; + goto out; + } + drain(wifi); + stage = readl(wifi->a.bar + 8); + if (stage != 2) { + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x lost transport: stage=%#x sequence=%u rx=%u\n", + id, stage, sequence, wifi->rx_seen[2]); + command_fault(wifi); + ret = -EIO; + goto out; + } + + if (wifi->reply_valid && wifi->reply_sequence == sequence && + (wifi->reply_command & ~BIT(30)) == id) { + ret = 0; + if (wifi->reply_status) { + dev_dbg(&wifi->control->pdev->dev, + "Wi-Fi command %#x: firmware status %u\n", + id, wifi->reply_status); + ret = -EREMOTEIO; + } + if (reply) + *reply_size = wifi->reply_size; + goto out; + } + usleep_range(500, 1000); + } while (time_before(jiffies, deadline)); + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); + command_fault(wifi); + ret = -ETIMEDOUT; +out: + wifi->reply_buffer = NULL; + return ret; +} + +static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload, + size_t size) +{ + return command_reply(wifi, id, payload, size, NULL, NULL); +} + +/* Caller holds wifi->lock; ret is what command() returned for the request. */ +static void command_refused(struct centauri_t6050_wifi *wifi, const char *what, + int ret) +{ + if (ret == -EREMOTEIO) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi %s refused: firmware status %u\n", what, + wifi->reply_status); + else + dev_warn(&wifi->control->pdev->dev, "Wi-Fi %s failed: %d\n", + what, ret); +} + +static int station_configure(struct centauri_t6050_wifi *wifi, const u8 *address) +{ + u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; + size_t start_size = 5; + int ret; + + ret = command(wifi, 1, address, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + return ret; +} + +/* + * The firmware owns the radio's channel list and capabilities: system + * commands 19 (the channels the hardware supports) and 8 (the system + * capabilities) report them once the interface is configured. Kept as + * read, in debugfs. + */ +static void firmware_report(struct centauri_t6050_wifi *wifi, u32 id, u8 *buffer, + struct debugfs_blob_wrapper *blob) +{ + size_t size = REPLY_SIZE; + int ret; + + ret = command_reply(wifi, id, NULL, 0, buffer, &size); + if (ret) { + command_refused(wifi, id == 19 ? "channel list" : "capabilities", + ret); + size = 0; + } else if (size > REPLY_SIZE) { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi command %u reply of %zu bytes truncated\n", id, + size); + size = REPLY_SIZE; + } + blob->data = buffer; + blob->size = size; +} + +/* + * The channel list: a count, then one 4-byte entry per channel the current + * country allows at each width: the channel number (the center channel for + * 40-160 MHz), then the band in bits 0-1, the width (20 << n MHz) in bits 2-3 + * and the flags in bits 4-7, then two bytes this driver does not use (a + * width mask and, on 6 GHz, power-mode bits). Scan requests take the same + * band and flag bits. + */ +#define CHANNEL_PASSIVE BIT(4) +#define CHANNEL_RADAR BIT(6) + +static const u8 *channel_entry(const u8 *list, size_t size, int band, + unsigned int width, unsigned int number) +{ + unsigned int count = size ? list[0] : 0, i, span = 2 * (1 << width) - 2; + + if (1 + 4 * count > size) + count = size ? (size - 1) / 4 : 0; + for (i = 0; i < count; i++) { + const u8 *entry = list + 1 + 4 * i; + + if ((entry[1] & 3) != band || ((entry[1] >> 2) & 3) != width) + continue; + if (number + span >= entry[0] && number <= entry[0] + span) + return entry; + } + return NULL; +} + +/* + * The firmware's list decides which channels exist and how: a band-plan + * channel it leaves out is disabled, its flags mark the rest, and a width no + * entry covers is not used. Applied over what cfg80211's regulatory domain + * set, after every change of it (the notifier) and at bring-up. + */ +static void channels_apply(struct centauri_t6050_wifi *wifi) +{ + const u8 *list = wifi->hw_channels; + size_t size = wifi->hw_channels_blob.size; + unsigned int counted[3] = {}, i; + + if (!size) + return; + for (i = 0; i < CEN_CHANNELS; i++) { + struct ieee80211_channel *channel = &wifi->channels[i]; + int band = firmware_band(channel->band); + const u8 *entry = channel_entry(list, size, band, 0, + channel->hw_value); + + if (!entry) { + channel->flags |= IEEE80211_CHAN_DISABLED; + continue; + } + if (entry[1] & CHANNEL_PASSIVE) + channel->flags |= IEEE80211_CHAN_NO_IR; + if (entry[1] & CHANNEL_RADAR) + channel->flags |= IEEE80211_CHAN_RADAR; + if (!channel_entry(list, size, band, 1, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_HT40; + if (!channel_entry(list, size, band, 2, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_80MHZ; + if (!channel_entry(list, size, band, 3, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_160MHZ; + if (!(channel->flags & IEEE80211_CHAN_DISABLED)) + counted[band]++; + } + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware channels: 2.4 GHz %u, 5 GHz %u, 6 GHz %u (%u entries)\n", + counted[0], counted[1], counted[2], list[0]); +} + +/* + * 6 GHz needs HE: the station's HE capabilities element as the firmware + * reports it (capabilities TLV 0xff), with the 6 GHz band capabilities taken + * from its VHT element (TLV 0xbf). Without them the band is not registered. + */ +static bool he_6ghz_from_firmware(struct centauri_t6050_wifi *wifi) +{ + struct ieee80211_sband_iftype_data *data = &wifi->iftype_6ghz; + struct ieee80211_sta_he_cap *he = &data->he_cap; + const u8 *caps = wifi->system_caps, *he_elem = NULL, *vht = NULL; + size_t size = wifi->system_caps_blob.size, pos = 0, mcs; + u32 vht_cap = 0; + + while (pos + 4 <= size) { + u16 tag = get_unaligned_le16(caps + pos); + u16 length = get_unaligned_le16(caps + pos + 2); + const u8 *value = caps + pos + 4; + + if (length > size - pos - 4) + break; + if (tag == WLAN_EID_EXTENSION && length >= 3 + 6 + 11 + 4 && + value[0] == WLAN_EID_EXTENSION && value[1] == length - 2 && + value[2] == WLAN_EID_EXT_HE_CAPABILITY) + he_elem = value; + if (tag == WLAN_EID_VHT_CAPABILITY && length >= 2 + 12 && + value[0] == WLAN_EID_VHT_CAPABILITY) + vht = value; + pos += 4 + length; + } + if (!he_elem) + return false; + memcpy(&he->he_cap_elem, he_elem + 3, sizeof(he->he_cap_elem)); + mcs = he_elem[1] - 1 - sizeof(he->he_cap_elem); + memcpy(&he->he_mcs_nss_supp, he_elem + 3 + sizeof(he->he_cap_elem), + min(mcs, sizeof(he->he_mcs_nss_supp))); + if (mcs > sizeof(he->he_mcs_nss_supp)) + memcpy(he->ppe_thres, + he_elem + 3 + sizeof(he->he_cap_elem) + + sizeof(he->he_mcs_nss_supp), + min(mcs - sizeof(he->he_mcs_nss_supp), + sizeof(he->ppe_thres))); + he->has_he = true; + if (vht) + vht_cap = get_unaligned_le32(vht + 2); + data->he_6ghz_capa.capa = cpu_to_le16( + u16_encode_bits(u32_get_bits(vht_cap, + IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK), + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) | + u16_encode_bits(vht_cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK, + IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN) | + u16_encode_bits(WLAN_HT_CAP_SM_PS_DISABLED, + IEEE80211_HE_6GHZ_CAP_SM_PS) | + (vht_cap & IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN ? + IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS : 0) | + (vht_cap & IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN ? + IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS : 0)); + data->types_mask = BIT(NL80211_IFTYPE_STATION); + return true; +} + +static void firmware_reports(struct centauri_t6050_wifi *wifi) +{ + firmware_report(wifi, 19, wifi->hw_channels, &wifi->hw_channels_blob); + firmware_report(wifi, 8, wifi->system_caps, &wifi->system_caps_blob); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware channel list: %lu bytes, capabilities: %lu bytes\n", + wifi->hw_channels_blob.size, wifi->system_caps_blob.size); +} + +/* The band plan, with the firmware's legacy rates (CCK on 2.4 GHz only). */ +static void bands_init(struct centauri_t6050_wifi *wifi) +{ + unsigned int i, n = 0, c; + + for (i = 0; i < ARRAY_SIZE(band_plan); i++) { + enum nl80211_band band = band_plan[i].band; + struct ieee80211_supported_band *sband = + &wifi->bands[firmware_band(band)]; + + if (!sband->channels) + sband->channels = wifi->channels + n; + for (c = band_plan[i].first; c <= band_plan[i].last; + c += band_plan[i].step) { + wifi->channels[n].band = band; + wifi->channels[n].hw_value = c; + wifi->channels[n].center_freq = + ieee80211_channel_to_frequency(c, band); + sband->n_channels++; + n++; + } + sband->band = band; + sband->bitrates = band == NL80211_BAND_2GHZ ? rates : rates + 4; + sband->n_bitrates = band == NL80211_BAND_2GHZ ? + ARRAY_SIZE(rates) : + ARRAY_SIZE(rates) - 4; + } + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + if (he_6ghz_from_firmware(wifi)) { + _ieee80211_set_sband_iftype_data(&wifi->bands[2], + &wifi->iftype_6ghz, 1); + wifi->wiphy->bands[NL80211_BAND_6GHZ] = &wifi->bands[2]; + } else { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi: no HE capabilities from the firmware; no 6 GHz band\n"); + } +} + +static int radio_start(struct centauri_t6050_wifi *wifi) +{ + unsigned int ring, slot; + int ret; + + for (ring = 1; ring <= 4; ring++) { + ret = open_pair(wifi, ring); + if (ret) + return ret; + } + for (ring = 2; ring <= 4; ring++) { + for (slot = 0; slot < ring_size[ring] - 1; slot++) + post_receive(wifi, ring, slot); + wifi->rx_head[ring] = ring_size[ring] - 1; + dma_wmb(); + index_write(wifi, 3, ring, ring_size[ring] - 1); + } + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + msleep(400); + ret = command(wifi, 0, NULL, 0); + if (!ret) + ret = station_configure(wifi, wifi->netdev->dev_addr); + if (ret) + return ret; + msleep(1000); + firmware_reports(wifi); + wifi->configured = !wifi->command_failed; + return wifi->configured ? 0 : -EIO; +} + +#define CEN_SCAN_CHANNELS_MAX 59 +static int scan_profile27(struct centauri_t6050_wifi *wifi) +{ + struct cfg80211_scan_request *request = wifi->scan_request; + u8 scan[14 + 5 + 4 * CEN_SCAN_CHANNELS_MAX + 9 + 4 + 2 + 32]; + unsigned int next = 0; + + if (!request || !request->n_channels || request->n_ssids > 1) + return -EINVAL; + while (next < request->n_channels) { + u8 channels[1 + 4 * CEN_SCAN_CHANNELS_MAX] = {}; + u8 options[5] = { 0, 0, 1, 0, 0 }; + unsigned int count = 0; + u32 done = wifi->scans_done; + size_t size = 14; + unsigned long deadline; + int ret; + bool abort_sent = false; + + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; + memset(scan, 0, sizeof(scan)); + scan[0] = (++wifi->scan_sequence) & 0x1f; + scan[3] = 0x80; + scan[5] = 2; + scan[7] = 0x10; + + put_unaligned_le16(40, scan + 8); + put_unaligned_le16(40, scan + 10); + put_unaligned_le16(110, scan + 12); + while (next < request->n_channels && + count < CEN_SCAN_CHANNELS_MAX) { + struct ieee80211_channel *channel = + request->channels[next++]; + unsigned int number; + int band; + + if (channel->flags & IEEE80211_CHAN_DISABLED) + continue; + band = firmware_band(channel->band); + if (band < 0) + return band; + number = ieee80211_frequency_to_channel( + channel->center_freq); + if (!number || number > 255) + return -EINVAL; + channels[1 + 4 * count] = number; + channels[2 + 4 * count] = band; + /* Flagged as in the firmware's channel list: listen only. */ + if (channel->flags & + (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR)) + channels[2 + 4 * count] |= CHANNEL_PASSIVE; + if (channel->flags & IEEE80211_CHAN_RADAR) + channels[2 + 4 * count] |= CHANNEL_RADAR; + count++; + } + if (!count) + continue; + channels[0] = count; + put_unaligned_le16(260, scan + size); + put_unaligned_le16(1 + 4 * count, scan + size + 2); + memcpy(scan + size + 4, channels, 1 + 4 * count); + size += 5 + 4 * count; + put_unaligned_le16(357, scan + size); + put_unaligned_le16(sizeof(options), scan + size + 2); + memcpy(scan + size + 4, options, sizeof(options)); + size += 4 + sizeof(options); + if (request->n_ssids) { + const struct cfg80211_ssid *ssid = &request->ssids[0]; + + if (ssid->ssid_len > 32) + return -EINVAL; + + scan[2] = !!ssid->ssid_len; + put_unaligned_le16(258, scan + size); + put_unaligned_le16(2 + ssid->ssid_len, scan + size + 2); + scan[size + 4] = ssid->ssid_len ? 5 : 4; + scan[size + 5] = ssid->ssid_len; + memcpy(scan + size + 6, ssid->ssid, ssid->ssid_len); + size += 6 + ssid->ssid_len; + } + ret = command(wifi, 65551, scan, size); + if (ret) { + command_refused(wifi, "scan", ret); + return ret; + } + deadline = jiffies + + msecs_to_jiffies(max(8000U, count * 220 + 2000)); + do { + drain(wifi); + if (READ_ONCE(wifi->stopping) || + readl(wifi->a.bar + 8) != 2) + return -EIO; + if (wifi->scans_done != done) + break; + if (READ_ONCE(wifi->scan_abort) && !abort_sent) { + ret = command(wifi, 65537, NULL, 0); + if (ret) + return ret; + abort_sent = true; + } + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (wifi->scans_done == done) { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi scan of %u channels: no completion event\n", + count); + command(wifi, 65537, NULL, 0); + return -ETIMEDOUT; + } + if (wifi->scan_status && !abort_sent) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi scan of %u channels ended with firmware status %u\n", + count, wifi->scan_status); + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; + } + return 0; +} + +static void scan_worker(struct work_struct *work) +{ + struct centauri_t6050_wifi *wifi = + container_of(work, struct centauri_t6050_wifi, scan_work); + struct cfg80211_scan_info info = {}; + struct cfg80211_scan_request *request; + int ret = 0; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->scan_abort) || READ_ONCE(wifi->stopping)) { + ret = -ECANCELED; + goto completed; + } + { + ret = scan_profile27(wifi); + goto completed; + } +completed: + info.aborted = !!ret || READ_ONCE(wifi->scan_abort) || + READ_ONCE(wifi->stopping); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + if (request) + cfg80211_scan_done(request, &info); +} + +static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) + ret = -ENODEV; + else if (wifi->command_failed) + ret = -EIO; + else if (!wifi->configured) + ret = -ENETDOWN; + else if (wifi->scan_request || wifi->connecting || + wifi->disconnect_pending) + ret = -EBUSY; + else { + WRITE_ONCE(wifi->scan_abort, false); + wifi->scan_request = request; + schedule_work(&wifi->scan_work); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + + if (wdev == &wifi->wdev) + WRITE_ONCE(wifi->scan_abort, true); +} + +static void cancel_scan(struct centauri_t6050_wifi *wifi) +{ + struct cfg80211_scan_info info = { .aborted = true }; + struct cfg80211_scan_request *request; + + WRITE_ONCE(wifi->scan_abort, true); + cancel_work_sync(&wifi->scan_work); + mutex_lock(&wifi->lock); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + + if (request) + cfg80211_scan_done(request, &info); +} + +static void cancel_connect_locked(struct centauri_t6050_wifi *wifi) +{ + if (!wifi->connecting) + return; + wifi->connecting = false; + wifi->connected = false; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, wifi->request_ie, + wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); +} + +static int rsn_caps_offset(const u8 *body, size_t size) +{ + size_t offset = 6; + u16 count; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + offset += 2 + 4 * count; + if (offset > size) + return -EINVAL; + } + return offset + 2 <= size ? offset : -EINVAL; +} + +/* The offset of the RSN element's AKM list when it offers akm. */ +static int rsn_akm_offset(const u8 *body, size_t size, u32 akm) +{ + size_t offset, end; + u16 count; + unsigned int i; + + if (size < 8 || get_unaligned_le16(body) != 1) + return -EINVAL; + count = get_unaligned_le16(body + 6); + if (!count) + return -EINVAL; + offset = 8 + 4 * count; + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + end = offset + 2 + 4 * count; + if (!count || end + 2 > size) + return -EINVAL; + for (i = 0; i < count; i++) + if (get_unaligned_be32(body + offset + 2 + 4 * i) == akm) + return offset; + return -EOPNOTSUPP; +} + +/* The station's RSN element with akm as its only AKM. */ +static int select_rsn(const struct cfg80211_crypto_settings *crypto, u32 akm, + const u8 *body, size_t size, u8 *selected) +{ + size_t tail; + unsigned int i; + int offset; + + for (i = 0; i < crypto->n_akm_suites; i++) + if (crypto->akm_suites[i] == akm) + break; + if (i == crypto->n_akm_suites) + return -EOPNOTSUPP; + offset = rsn_akm_offset(body, size, akm); + if (offset < 0) + return offset; + + if (get_unaligned_be32(body + 2) != WLAN_CIPHER_SUITE_CCMP || + get_unaligned_le16(body + 6) != 1 || + get_unaligned_be32(body + 8) != WLAN_CIPHER_SUITE_CCMP) + return -EOPNOTSUPP; + tail = offset + 2 + 4 * get_unaligned_le16(body + offset); + memcpy(selected, body, offset); + put_unaligned_le16(1, selected + offset); + put_unaligned_be32(akm, selected + offset + 2); + memcpy(selected + offset + 6, body + tail, size - tail); + return offset + 6 + size - tail; +} + +/* + * The connect command carries one authentication and AKM pair, in the bytes + * at 0x46 and 0x47: WPA2-Personal is open authentication with AKM 4 and the + * PMK at 0x4c, WPA3-Personal is SAE authentication with AKM 8 and the + * password in TLV 313, which the firmware runs SAE from. + */ +#define CONNECT_AUTH_OPEN 0 +#define CONNECT_AUTH_SAE 3 +#define CONNECT_AKM_PSK 4 +#define CONNECT_AKM_SAE 8 + +/* The AKM of the client's request whose credential it supplies, SAE first. */ +static u32 connect_akm(const struct cfg80211_crypto_settings *crypto) +{ + bool psk = false; + unsigned int i; + + for (i = 0; i < crypto->n_akm_suites; i++) { + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_SAE && + crypto->sae_pwd_len) + return WLAN_AKM_SUITE_SAE; + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_PSK && crypto->psk) + psk = true; + } + return psk ? WLAN_AKM_SUITE_PSK : 0; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, + u16 length) +{ + put_unaligned_le16(tag, payload + *size); + put_unaligned_le16(length, payload + *size + 2); + memcpy(payload + *size + 4, data, length); + *size += 4 + length; +} + +/* + * The connect request names its target: nl80211 clients pass the BSSID and + * channel they chose from their own scan results. The kernel's scan entry for + * it may already be gone (a client's flushed scan expires every BSS its last + * channel subset did not see), so the entry is looked up only when the + * request names no target. + */ +static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + static const u8 mld_defaults[14] = { + 1, 0, 1, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 3, 0, 0, + }; + + /* TLV 304: the join mode, 5 for every join. */ + static const u8 join_mode = 5; + /* TLV 316: no security roaming groups. */ + static const u8 roaming; + static const __le32 connection_timeouts[] = { + cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), + cpu_to_le32(15000), cpu_to_le32(300), + }; + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + const u8 *bssid = params->bssid ?: params->bssid_hint; + struct ieee80211_channel *channel = + params->channel ?: params->channel_hint; + struct cfg80211_bss *bss; + const u8 *rsn; + u8 rsn_body[255], ssid[35]; + u8 *payload = wifi->connect_payload; + size_t size = 130, rsn_size, rsn_prefix, rsn_tail; + int ret = 0, offset, band; + u32 akm = connect_akm(¶ms->crypto); + bool sae = akm == WLAN_AKM_SUITE_SAE, mfpc; + u8 password[2 + SAE_PASSWORD_MAX_LEN]; + + if (!params->ssid_len || params->ssid_len > 32 || !akm || + params->crypto.sae_pwd_len > SAE_PASSWORD_MAX_LEN || + params->crypto.n_ciphers_pairwise != 1 || + params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || + params->ie_len > sizeof(wifi->request_ie)) + return -EOPNOTSUPP; + rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); + if (!rsn) + return -EINVAL; + ret = select_rsn(¶ms->crypto, akm, rsn + 2, rsn[1], rsn_body); + if (ret < 0) + return ret; + rsn_size = ret; + /* + * The station's own RSN element carries its management frame + * protection to the firmware; bit 7 of its capabilities is MFPC. + */ + offset = rsn_caps_offset(rsn_body, rsn_size); + if (offset < 0) + return -EINVAL; + mfpc = get_unaligned_le16(rsn_body + offset) & BIT(7); + ret = 0; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured || wifi->connecting || wifi->connected || + wifi->scan_request || wifi->disconnect_pending) { + ret = -EBUSY; + goto out; + } + + if (bssid && channel) { + memcpy(wifi->bssid, bssid, ETH_ALEN); + } else { + bss = cfg80211_get_bss(wiphy, channel, bssid, params->ssid, + params->ssid_len, IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; + } + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + channel = bss->channel; + cfg80211_put_bss(wiphy, bss); + } + ret = firmware_band(channel->band); + if (ret < 0) + goto out; + band = ret; + ret = 0; + /* 6 GHz admits WPA3-Personal only with management frame protection. */ + if (channel->band == NL80211_BAND_6GHZ && (!sae || !mfpc)) { + ret = -EOPNOTSUPP; + goto out; + } + wifi->connect_channel = channel; + memcpy(wifi->ssid, params->ssid, params->ssid_len); + wifi->ssid_len = params->ssid_len; + memset(payload, 0, sizeof(wifi->connect_payload)); + payload[0] = 1; + memcpy(payload + 1, wifi->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(channel->center_freq); + + payload[8] = band; + payload[0x46] = sae ? CONNECT_AUTH_SAE : CONNECT_AUTH_OPEN; + payload[0x47] = sae ? CONNECT_AKM_SAE : CONNECT_AKM_PSK; + + put_unaligned_le16(4, payload + 0x48); + if (!sae) { + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + } + + append_tlv(payload, &size, 304, &join_mode, 1); + append_tlv(payload, &size, 63763, connection_timeouts, + sizeof(connection_timeouts)); + ssid[0] = 1; + ssid[1] = 4; + ssid[2] = params->ssid_len; + memcpy(ssid + 3, params->ssid, params->ssid_len); + append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); + append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + if (sae) { + put_unaligned_le16(params->crypto.sae_pwd_len, password); + memcpy(password + 2, params->crypto.sae_pwd, + params->crypto.sae_pwd_len); + append_tlv(payload, &size, 313, password, + 2 + params->crypto.sae_pwd_len); + memzero_explicit(password, sizeof(password)); + } + + append_tlv(payload, &size, 316, &roaming, 1); + append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + wifi->connect_size = size; + + rsn_prefix = rsn - params->ie; + rsn_tail = rsn_prefix + 2 + rsn[1]; + memcpy(wifi->request_ie, params->ie, rsn_prefix); + wifi->request_ie[rsn_prefix] = WLAN_EID_RSN; + wifi->request_ie[rsn_prefix + 1] = rsn_size; + memcpy(wifi->request_ie + rsn_prefix + 2, rsn_body, rsn_size); + memcpy(wifi->request_ie + rsn_prefix + 2 + rsn_size, + params->ie + rsn_tail, params->ie_len - rsn_tail); + wifi->request_ie_size = params->ie_len - rsn[1] + rsn_size; + WRITE_ONCE(wifi->connect_abort, false); + wifi->connecting = true; + wifi->connect_status = 0; + wifi->connect_event = wifi->disconnect_event = false; + schedule_work(&wifi->connect_work); +out: + mutex_unlock(&wifi->lock); + return ret; +} + +/* + * cfg80211 records a connection against a BSS entry. When the client's scans + * have expired the target's, inform one from the request until its beacons + * (reported on ring 4 once associated) replace the elements. + */ +static struct cfg80211_bss *connected_bss(struct centauri_t6050_wifi *wifi) +{ + struct cfg80211_bss *bss; + u8 ie[2 + IEEE80211_MAX_SSID_LEN]; + + bss = cfg80211_get_bss(wifi->wiphy, wifi->connect_channel, wifi->bssid, + NULL, 0, IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ANY); + if (bss) + return bss; + ie[0] = WLAN_EID_SSID; + ie[1] = wifi->ssid_len; + memcpy(ie + 2, wifi->ssid, wifi->ssid_len); + return cfg80211_inform_bss(wifi->wiphy, wifi->connect_channel, + CFG80211_BSS_FTYPE_UNKNOWN, wifi->bssid, 0, + WLAN_CAPABILITY_ESS | WLAN_CAPABILITY_PRIVACY, + 100, ie, 2 + wifi->ssid_len, -95 * 100, + GFP_KERNEL); +} + +static void connect_worker(struct work_struct *work) +{ + struct centauri_t6050_wifi *wifi = + container_of(work, struct centauri_t6050_wifi, connect_work); + unsigned long deadline; + u16 slot, status; + int ret; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) { + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + return; + } + slot = wifi->command_head; + ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + deadline = jiffies + msecs_to_jiffies(15000); + while (!ret && !wifi->connect_event && + !READ_ONCE(wifi->connect_abort) && !READ_ONCE(wifi->stopping) && + time_before(jiffies, deadline)) { + drain(wifi); + + if (wifi->disconnect_event) + ret = -ENOTCONN; + if (readl(wifi->a.bar + 8) != 2) + ret = -EIO; + usleep_range(1000, 2000); + } + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) + ret = -ECANCELED; + else if (!ret && !wifi->connect_event) + ret = -ETIMEDOUT; + if (ret == -ENOTCONN) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM: disconnect event, firmware status %u\n", + wifi->bssid, wifi->disconnect_status); + else if (ret == -ETIMEDOUT) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM: no connect event\n", + wifi->bssid); + else if (ret && ret != -ECANCELED) + command_refused(wifi, "connect", ret); + else if (!ret && wifi->connect_status) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM failed: firmware status %u\n", + wifi->bssid, wifi->connect_status); + status = !ret && wifi->connect_status == 0 ? + WLAN_STATUS_SUCCESS : + WLAN_STATUS_UNSPECIFIED_FAILURE; + wifi->connecting = false; + wifi->connected = status == WLAN_STATUS_SUCCESS; + wifi->station_signal_valid = false; + if (wifi->connected) { + wifi->connected_at = jiffies; + wifi->station_signal_at = jiffies - HZ; + wifi->station_start = wifi->netdev->stats; + } + + if (command_payload_consumed(wifi, slot)) + memzero_explicit(wifi->a.window + TX_BUFFER + + slot * BUFFER_SIZE, + BUFFER_SIZE); + + if (ret == -ECANCELED || ret == -ETIMEDOUT) + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, + wifi->request_ie, + wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); + else + cfg80211_connect_bss(wifi->netdev, wifi->bssid, + status == WLAN_STATUS_SUCCESS ? + connected_bss(wifi) : + NULL, + wifi->request_ie, wifi->request_ie_size, + NULL, 0, status, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); + if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + cfg80211_port_authorized(wifi->netdev, wifi->bssid, NULL, 0, + GFP_KERNEL); + netif_carrier_on(wifi->netdev); + if (wifi->data_rings_open & BIT(5)) + netif_wake_queue(wifi->netdev); + schedule_work(&wifi->link_work); + } + mutex_unlock(&wifi->lock); +} + +irqreturn_t centauri_t6050_alpha_irq(int irq, void *data) +{ + struct centauri_t6050 *c = data; + struct centauri_t6050_wifi *wifi = smp_load_acquire(&c->wifi); + + if (wifi && smp_load_acquire(&wifi->irq_ready) && + !READ_ONCE(wifi->stopping)) + schedule_work(&wifi->link_work); + return IRQ_HANDLED; +} + +static void link_worker(struct work_struct *work) +{ + struct centauri_t6050_wifi *wifi = + container_of(work, struct centauri_t6050_wifi, link_work); + bool disconnected, was_connected; + + mutex_lock(&wifi->lock); + was_connected = wifi->connected; + drain(wifi); + disconnected = wifi->command_failed || wifi->disconnect_event || + READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2; + if (disconnected) + wifi->connected = false; + if (disconnected && was_connected) { + netif_stop_queue(wifi->netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(wifi->netdev); + cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, false, GFP_KERNEL); + } + mutex_unlock(&wifi->lock); +} + +static int wait_disconnect(struct centauri_t6050_wifi *wifi) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(5000); + + while (wifi->disconnect_pending) { + drain(wifi); + if (READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2) + return -EIO; + if (!wifi->disconnect_pending) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(500, 1000); + } + return 0; +} + +static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, + u16 reason) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + u8 payload[16] = {}; + bool connected; + u16 reply_status; + int ret; + + WRITE_ONCE(wifi->connect_abort, true); + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + put_unaligned_le16(reason, payload); + put_unaligned_le16(328, payload + 8); + put_unaligned_le16(4, payload + 10); + + put_unaligned_le32(10, payload + 12); + wifi->disconnect_pending = !READ_ONCE(wifi->stopping); + ret = READ_ONCE(wifi->stopping) ? + 0 : + command(wifi, 131073, payload, sizeof(payload)); + reply_status = wifi->reply_status; + if (!ret && wifi->disconnect_pending && (wifi->reply_command & BIT(30))) + ret = wait_disconnect(wifi); + else if (ret != -EIO && ret != -ETIMEDOUT) + wifi->disconnect_pending = false; + + connected = wifi->connected; + wifi->connected = false; + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(netdev); + if (connected) + cfg80211_disconnected(netdev, reason, NULL, 0, true, + GFP_KERNEL); + mutex_unlock(&wifi->lock); + cancel_work_sync(&wifi->link_work); + return ret == -EREMOTEIO && reply_status == 326 ? 0 : ret; +} + +static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + u8 mode = enabled ? 0 : 3; + int ret; + + mutex_lock(&wifi->lock); + ret = wifi->configured ? command(wifi, 7, &mode, sizeof(mode)) : + -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_regulatory(struct wiphy *wiphy, + struct regulatory_request *request) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + bool named = country[0] >= 'A' && country[0] <= 'Z' && + country[1] >= 'A' && country[1] <= 'Z'; + int ret; + + mutex_lock(&wifi->lock); + if (wifi->configured && named) { + ret = command(wifi, 9, country, sizeof(country)); + if (ret) + dev_err(&wifi->control->pdev->dev, + "firmware country update failed: %d\n", ret); + else + firmware_report(wifi, 19, wifi->hw_channels, + &wifi->hw_channels_blob); + } + /* cfg80211 has just recomputed the channel flags from its own rules. */ + channels_apply(wifi); + mutex_unlock(&wifi->lock); +} + +static void fill_station(struct centauri_t6050_wifi *wifi, struct station_info *sinfo) +{ + const struct net_device_stats *stats = &wifi->netdev->stats; + + if (time_after_eq(jiffies, wifi->station_signal_at + HZ)) { + wifi->station_signal_valid = false; + wifi->station_signal_at = jiffies; + query_link_status(wifi); + } + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_TX_PACKETS); + sinfo->connected_time = (jiffies - wifi->connected_at) / HZ; + sinfo->rx_bytes = stats->rx_bytes - wifi->station_start.rx_bytes; + sinfo->tx_bytes = stats->tx_bytes - wifi->station_start.tx_bytes; + sinfo->rx_packets = stats->rx_packets - wifi->station_start.rx_packets; + sinfo->tx_packets = stats->tx_packets - wifi->station_start.tx_packets; + if (wifi->station_signal_valid) { + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->signal = wifi->station_signal; + } +} + +static int wifi_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (wdev != &wifi->wdev || !wifi->connected || + READ_ONCE(wifi->stopping) || !ether_addr_equal(mac, wifi->bssid)) + ret = -ENOENT; + else + fill_station(wifi, sinfo); + mutex_unlock(&wifi->lock); + return ret; +} + +static int wifi_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, + int index, u8 *mac, struct station_info *sinfo) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (index || wdev != &wifi->wdev || !wifi->connected || + READ_ONCE(wifi->stopping)) + ret = -ENOENT; + else { + ether_addr_copy(mac, wifi->bssid); + fill_station(wifi, sinfo); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct cfg80211_ops wifi_ops = { + .scan = wifi_scan, + .abort_scan = wifi_abort_scan, + .connect = wifi_connect, + .disconnect = wifi_disconnect, + .set_power_mgmt = wifi_power_mgmt, + .get_station = wifi_get_station, + .dump_station = wifi_dump_station, +}; + +static int data_start(struct centauri_t6050_wifi *wifi); + +static int net_open(struct net_device *netdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + int ret; + + netif_carrier_off(netdev); + netif_stop_queue(netdev); + mutex_lock(&wifi->lock); + ret = wifi->configured ? data_start(wifi) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static int net_stop(struct net_device *netdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + + cancel_scan(wifi); + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + return 0; +} + +static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + + if (READ_ONCE(wifi->stopping) || !READ_ONCE(wifi->connected) || + skb->len < ETH_HLEN || skb->len > BUFFER_SIZE) { + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) { + netif_stop_queue(netdev); + return NETDEV_TX_BUSY; + } + skb_queue_tail(&wifi->data_tx_queue, skb); + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) + netif_stop_queue(netdev); + schedule_work(&wifi->link_work); + return NETDEV_TX_OK; +} + +static int station_reconfigure(struct centauri_t6050_wifi *wifi, const u8 *address) +{ + const u8 stop[5] = {}; + int ret; + + wifi->configured = false; + ret = command(wifi, 3, stop, sizeof(stop)); + if (!ret) + ret = station_configure(wifi, address); + if (!ret) + wifi->configured = true; + return ret; +} + +static int net_set_mac_address(struct net_device *netdev, void *data) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + struct sockaddr *address = data; + int ret, restore; + + ret = eth_prepare_mac_addr_change(netdev, data); + if (ret) + return ret; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured) { + ret = -ENETDOWN; + goto out; + } + if (wifi->connecting || wifi->connected || wifi->scan_request || + wifi->disconnect_pending) { + ret = -EBUSY; + goto out; + } + if (ether_addr_equal(netdev->dev_addr, address->sa_data)) + goto out; + + ret = station_reconfigure(wifi, (const u8 *)address->sa_data); + if (!ret) { + eth_commit_mac_addr_change(netdev, data); + } else { + restore = station_reconfigure(wifi, netdev->dev_addr); + if (restore) + dev_err(&wifi->control->pdev->dev, + "Wi-Fi MAC restore failed: %d (change: %d)\n", + restore, ret); + } +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct net_device_ops net_ops = { + .ndo_open = net_open, + .ndo_stop = net_stop, + .ndo_start_xmit = net_transmit, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_mac_address = net_set_mac_address, +}; + +static int data_ring_open(struct centauri_t6050_wifi *wifi, unsigned int id) +{ + u8 message[0x34 + 10] = { 0 }; + unsigned int vector, offset, size, bit, doorbell, irq; + bool transfer = false; + int ret; + + lockdep_assert_held(&wifi->lock); + if (wifi->data_failed) + return -EIO; + if (id == 76) { + vector = 192; + offset = 0x200000; + size = 2048; + bit = 0; + doorbell = 18; + irq = 3; + } else if (id == 14) { + vector = 206; + offset = 0x220000; + size = 256; + bit = 1; + doorbell = 11; + irq = 8; + } else if (id == 15) { + vector = 207; + offset = 0x240000; + size = DATA_RX_SLOTS; + bit = 2; + doorbell = 11; + irq = 8; + } else if (id == 24 || id == 25 || id == 12) { + transfer = true; + vector = id; + offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; + size = id == 12 ? DATA_RX_SLOTS : DATA_TX_SLOTS; + bit = id == 12 ? 4 : id == 24 ? 3 : 5; + doorbell = id == 12 ? 11 : 18; + irq = id == 12 ? 8 : 3; + } else { + return -EINVAL; + } + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto out; + } + if (wifi->data_rings_open & BIT(bit)) + return 0; + if (wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + memset(wifi->a.window + offset, 0, size * 32); + index_write(wifi, transfer ? 1 : 0, vector, 0); + index_write(wifi, transfer ? 3 : 2, vector, 0); + message[0] = transfer ? 1 : 2; + message[2] = transfer && id == 12 ? 0 : 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le16(vector, message + 6); + put_unaligned_le64(wifi->a.window_dma + offset, message + 8); + if (transfer) { + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(size, message + 0x18); + put_unaligned_le16(id == 12 ? 12 : 76, message + 0x1a); + put_unaligned_le16(doorbell, message + 0x1c); + put_unaligned_le16(id == 12 ? 0x220 : 0, message + 0x1e); + put_unaligned_le32(16, message + 0x20); + put_unaligned_le16(irq, message + 0x24); + } else { + put_unaligned_le16(size, message + 0x10); + put_unaligned_le16(id == 15 ? 12 : 0xffff, message + 0x12); + put_unaligned_le16(doorbell, message + 0x14); + put_unaligned_le32(1, message + 0x18); + put_unaligned_le16(irq, message + 0x1c); + } + + ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); + if (!ret && readl(wifi->a.bar + 8) != 2) + ret = -EIO; + if (ret) { + wifi->data_failed = true; + goto out; + } + wifi->data_rings_open |= BIT(bit); + if (transfer && id == 12) { + unsigned int slot; + + for (slot = 0; slot < size - 1; slot++) { + u8 *descriptor = wifi->a.window + offset + slot * 16; + u32 buffer = DATA_RX_BUFFER + slot * 4096; + + memset(wifi->a.window + buffer, 0, 4096); + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, + descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + } + dma_wmb(); + index_write(wifi, 3, vector, size - 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +out: + return ret; +} + +static int data_start(struct centauri_t6050_wifi *wifi) +{ + static const unsigned int rings[] = { 76, 15, 12, 25 }; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(rings); i++) { + ret = data_ring_open(wifi, rings[i]); + if (ret) + return ret; + } + return 0; +} + +static int query_link_status(struct centauri_t6050_wifi *wifi) +{ + size_t size = sizeof(wifi->link_status); + int ret; + + if (!wifi->configured) + return -ENETDOWN; + ret = command_reply(wifi, 20, NULL, 0, wifi->link_status, &size); + if (!ret) + wifi->station_signal_valid = + link_signal_read(wifi->link_status, + min(size, sizeof(wifi->link_status)), + &wifi->station_signal); + wifi->station_signal_at = jiffies; + return ret; +} + +int centauri_t6050_wifi_register(struct centauri_t6050 *c) +{ + struct centauri_t6050_wifi *wifi; + u8 address[ETH_ALEN]; + int ret; + + wifi = kzalloc_obj(*wifi); + if (!wifi) + return -ENOMEM; + wifi->control = c; + mutex_init(&wifi->lock); + skb_queue_head_init(&wifi->data_tx_queue); + INIT_WORK(&wifi->scan_work, scan_worker); + INIT_WORK(&wifi->connect_work, connect_worker); + INIT_WORK(&wifi->link_work, link_worker); + smp_store_release(&c->wifi, wifi); + ret = centauri_t6050_alpha_resources(c, &wifi->a); + if (ret) + goto failed; + wifi->wiphy = wiphy_new(&wifi_ops, sizeof(struct centauri_t6050_wifi *)); + wifi->netdev = alloc_etherdev(sizeof(struct centauri_t6050_wifi *)); + if (!wifi->wiphy || !wifi->netdev) { + ret = -ENOMEM; + goto failed; + } + *(struct centauri_t6050_wifi **)wiphy_priv(wifi->wiphy) = wifi; + *(struct centauri_t6050_wifi **)netdev_priv(wifi->netdev) = wifi; + set_wiphy_dev(wifi->wiphy, &c->pdev->dev); + wifi->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + wifi->wiphy->max_scan_ssids = 1; + wifi->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + wifi->wiphy->reg_notifier = wifi_regulatory; + wifi->wiphy->cipher_suites = cipher_suites; + wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wifi->wiphy->akm_suites = akm_suites; + wifi->wiphy->n_akm_suites = ARRAY_SIZE(akm_suites); + wiphy_ext_feature_set(wifi->wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); + + ret = eth_platform_get_mac_address(&wifi->a.pdev->dev, address); + if (!ret) { + eth_hw_addr_set(wifi->netdev, address); + ether_addr_copy(wifi->wiphy->perm_addr, address); + } else if (ret == -EPROBE_DEFER) { + goto failed; + } else { + eth_hw_addr_random(wifi->netdev); + dev_warn( + &wifi->a.pdev->dev, + "No valid factory MAC property; using a random address\n"); + } + /* The bands come from the firmware's reports: start the radio first. */ + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, + "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + bands_init(wifi); + ret = wiphy_register(wifi->wiphy); + if (ret) + goto failed; + wifi->registered = true; + debugfs_create_blob("hw_channels", 0400, wifi->wiphy->debugfsdir, + &wifi->hw_channels_blob); + debugfs_create_blob("system_caps", 0400, wifi->wiphy->debugfsdir, + &wifi->system_caps_blob); + rtnl_lock(); + channels_apply(wifi); + rtnl_unlock(); + wifi->wdev.wiphy = wifi->wiphy; + wifi->wdev.iftype = NL80211_IFTYPE_STATION; + wifi->wdev.netdev = wifi->netdev; + wifi->netdev->ieee80211_ptr = &wifi->wdev; + wifi->netdev->netdev_ops = &net_ops; + SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); + strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); + wifi->netdev->max_mtu = ETH_DATA_LEN; + + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; + smp_store_release(&wifi->irq_ready, true); + schedule_work(&wifi->link_work); + + return 0; +failed: + centauri_t6050_wifi_unregister(c); + return ret; +} + +void centauri_t6050_wifi_unregister(struct centauri_t6050 *c) +{ + struct centauri_t6050_wifi *wifi = c->wifi; + + if (!wifi) + return; + WRITE_ONCE(wifi->stopping, true); + WRITE_ONCE(wifi->connect_abort, true); + WRITE_ONCE(wifi->scan_abort, true); + smp_store_release(&c->wifi, NULL); + + for (unsigned int i = 0; i < c->alpha_irqs.count; i++) + synchronize_irq(pci_irq_vector(c->alpha_irqs.pdev, i)); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + netif_tx_disable(wifi->netdev); + + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); + cancel_work_sync(&wifi->link_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + skb_queue_purge(&wifi->data_tx_queue); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(wifi->netdev); + if (wifi->registered) + wiphy_unregister(wifi->wiphy); + if (wifi->wiphy) + wiphy_free(wifi->wiphy); + if (wifi->netdev) + free_netdev(wifi->netdev); + memzero_explicit(wifi, sizeof(*wifi)); + kfree(wifi); +} diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 732d7b63be3d4..2164b13f4e787 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -210,6 +210,8 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; + /* The ANS is never reset through PMGR here, so no reset is named. */ + bool reset_optional; u32 max_queue_depth; unsigned long quirks; }; @@ -2407,7 +2409,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto out_detach_genpd; } - anv->reset = devm_reset_control_array_get_exclusive(anv->dev); + anv->reset = anv->hw->reset_optional ? + devm_reset_control_array_get_optional_exclusive(anv->dev) : + devm_reset_control_array_get_exclusive(anv->dev); if (IS_ERR(anv->reset)) { ret = dev_err_probe(dev, PTR_ERR(anv->reset), "Failed to get reset control"); @@ -2776,8 +2780,26 @@ static const struct apple_nvme_hw apple_nvme_t6040_hw = { .quirks = NVME_QUIRK_NO_APST, }; +/* + * T6050 (M5 Ultra) ANS on bare metal: T6040's split layout (a 64 KiB "nvme" + * block, a separate NVMMU, linear submission queues), 64 queue entries. Like + * every post-M4 ANS without cold_reset it is never reset through PMGR, so no + * reset is named. + */ +static const struct apple_nvme_hw apple_nvme_t6050_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .lsq_in_nvmmu = true, + .reset_optional = true, + .max_queue_depth = 64, + /* FUA is not shown durable on this ANS: flush instead. */ + .quirks = NVME_QUIRK_NO_APST | NVME_QUIRK_BROKEN_FUA, +}; + static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t6041-nvme-ans2", .data = &apple_nvme_t6040_hw }, + { .compatible = "apple,t6050-nvme-ans2", .data = &apple_nvme_t6050_hw }, { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 6ad3376dcf75e..c19f271e40d2b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -186,6 +186,35 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_MSIMAP_ENABLE BIT(31) #define PORT_MSIMAP_TARGET GENMASK(7, 0) +/* T6050: the root port's port logic, laid out as DesignWare's */ +#define RC_T6050_PORT_LINK_CTRL 0x710 +#define RC_T6050_LINK_MODE GENMASK(21, 16) +#define RC_T6050_PORT_DEBUG0 0x728 +#define RC_T6050_LTSSM_STATE GENMASK(5, 0) +#define RC_T6050_LTSSM_L1_IDLE 0x14 +#define RC_T6050_LINK_WIDTH_SPEED_CTRL 0x80c +#define RC_T6050_LINK_WIDTH GENMASK(12, 8) +#define RC_T6050_SPEED_CHANGE BIT(17) +#define RC_T6050_GEN3_RELATED 0x890 +#define RC_T6050_RATE_SHADOW_SEL GENMASK(25, 24) +#define RC_T6050_MISC_CONTROL_1 0x8bc +#define RC_T6050_DBI_RO_WR_EN BIT(0) + +/* + * T6050 PCIe-C: the 256-entry RID2SID table runs over the T8103 tunnel block, + * which moves up by 0x700 (posted request counters, TUNCTRL and TUNSTAT). + */ +#define PORT_T6050_TUNCTRL 0x1088 + +/* T6050 PCIe-C root port config space: the MPS-dependent link timers */ +#define PCIEC_RP_DEVCAP 0x074 +#define PCIEC_RP_DEVCTL 0x078 +#define PCIEC_RP_ACK_LATENCY 0x700 +#define PCIEC_RP_FC_TIMER 0x73c +#define PCIEC_RP_FC_TIMER_KEEP 0x6000ffff +#define PCIEC_RP_FC_TIMER_LIMIT GENMASK(28, 16) +#define PCIEC_RP_FC_TIMER_EN BIT(31) + /* * The doorbell address must be in the bottom 4GB because the base register * is only 32 bits wide. Exclude it from the IOVA range and share the same @@ -207,6 +236,7 @@ struct hw_info { u32 pwren_on_ms; bool sid_indexed_rid2sid; bool retain_clocks; + bool adopt_link; bool root_bus_only; bool tunneled; bool tunnel_intr2axi; @@ -224,6 +254,8 @@ struct hw_info { * lane PIPE reset status, then release PORT_T8152_PIPE_RST. */ bool pipe_rst; + bool no_port_reset; + bool mps_timers; }; static const struct hw_info t8103_hw = { @@ -281,6 +313,24 @@ static const struct hw_info t8152_hw = { .pipe_rst = true, }; +/* + * Bare-metal T6050 adoption only. Firmware owns power, PHY, reset and training. + * The 64-entry route table is never probed with test writes. + */ +static const struct hw_info t6050_hw = { + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 64, + .retain_clocks = true, + .adopt_link = true, +}; + +#define T6050_ROUTE(rid, sid) (PORT_RID2SID_VALID | \ + ((sid) << PORT_RID2SID_SID_SHIFT) | (rid)) + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -351,6 +401,32 @@ static const struct hw_info t6040_pciec_hw = { .s2idle_hold_domain = true, }; +/* + * T6050 PCIe-C keeps the t8103 doorbell, PERST and RID2SID registers, with + * the doorbell's high word at 0x16c and 256 RID2SID entries. MSIs go through + * the t602x map, indexed by the vector the endpoint sends: the bridge's slice + * of the shared window, not a local index. The port has no reset sequence of + * its own, and the root port's ACK latency and flow control timers follow + * the max payload size. + * + * Nothing hands the port over: apple,pciec-kernel-init brings it up with the + * T600x Intr2AXI sequence. + */ +static const struct hw_info t6050_pciec_hw = { + .port_msiaddr = PORT_MSIADDR, + .port_msiaddr_hi = PORT_MSIADDR + 4, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .port_tunctrl = PORT_T6050_TUNCTRL, + .max_rid2sid = 256, + .tunneled = true, + .tunnel_intr2axi = true, + .no_port_reset = true, + .msimap_by_vector = true, + .mps_timers = true, +}; + static const struct of_device_id apple_pcie_of_match[]; /** @@ -440,7 +516,7 @@ struct apple_pcie_port { struct device_node *np; void __iomem *base; void __iomem *phy; - /* PERST# of a conventional port with a single reset line. */ + /* PERST# of a conventional or adopted port with a single reset line. */ struct gpio_desc *perst; struct irq_domain *domain; struct list_head entry; @@ -456,9 +532,13 @@ struct apple_pcie_port { bool needs_stop; bool link_failed; bool irq_disabled; + void __iomem *intr2axi; /* adopt_link: this port's */ + u8 adopted_bus; /* adopt_link: the secondary bus */ + unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); +static struct apple_pcie *apple_pcie_lookup(struct device *dev); static int apple_pcie_radio_check_cold(struct apple_pcie_port *port); static void rmw_set(u32 set, void __iomem *addr) @@ -767,9 +847,20 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) struct apple_pcie *pcie = port->pcie; u32 val = 0; - /* FIXME: consider moving each interrupt under each port */ - port->irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), - port->idx); + if (pcie->hw->adopt_link) { + char name[16]; + int irq; + + snprintf(name, sizeof(name), "port%d", port->idx); + irq = of_irq_get_byname(pcie->dev->of_node, name); + + if (irq <= 0) + return irq ?: -ENXIO; + port->irq = irq; + } else { + /* FIXME: consider moving each interrupt under each port */ + port->irq = irq_of_parse_and_map(pcie->dev->of_node, port->idx); + } if (!port->irq) return -ENXIO; @@ -782,6 +873,9 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) return -ENOMEM; } + if (pcie->hw->adopt_link) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + /* Disable all interrupts */ apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); @@ -815,7 +909,22 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } + if (pcie->hw->adopt_link) { + if (apple_pcie_port_readl(port, pcie->hw->port_msiaddr) != DOORBELL_ADDR || + apple_pcie_port_readl(port, pcie->hw->port_msiaddr_hi)) + return -EIO; + for (int i = 0; i < pcie->nvecs; i++) + if (apple_pcie_port_readl(port, pcie->hw->port_msimap + 4 * i) != + (PORT_MSIMAP_ENABLE | (pcie->msi_vector_base + i))) + return -EIO; + /* Enable command is not readable; +0x84 is only a completion read. */ + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + } apple_pcie_port_writel(port, val | PORT_MSICFG_EN, PORT_MSICFG); + if (pcie->hw->adopt_link && + !(apple_pcie_port_readl(port, PORT_MSICFG) & PORT_MSICFG_EN)) + return -EIO; return 0; } @@ -831,6 +940,8 @@ static irqreturn_t apple_pcie_port_irq(int irq, void *data) complete_all(&port->pcie->event); break; case PORT_INT_LINK_DOWN: + if (port->pcie->hw->adopt_link) + WRITE_ONCE(port->link_failed, true); /* A later link-up alone does not prove downstream state survived. */ if (port->pcie->hw->tunneled && READ_ONCE(port->started)) { WRITE_ONCE(port->link_failed, true); @@ -1008,6 +1119,10 @@ static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) { int i; + /* T6050 has no such sequence: only its RID2SID table is cleared. */ + if (port->pcie->hw->no_port_reset) + goto clear_rid2sid; + for (i = 0; i < ARRAY_SIZE(apple_pcie_tunnel_reset_regs); i++) { u32 value = apple_pcie_tunnel_reset_regs[i].value; @@ -1035,6 +1150,7 @@ static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) apple_pcie_port_writel(port, value, apple_pcie_tunnel_reset_regs[i].offset); } +clear_rid2sid: for (i = 0; i < port->pcie->hw->max_rid2sid; i++) apple_pcie_rid2sid_write(port, i, 0); if (port->pcie->hw->t8132_reset) @@ -1107,8 +1223,8 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) /* * Bring up a port selected by apple,pciec-kernel-init that no firmware - * handoff left running: tunables and the reset table, then PERST and APPCLK, - * and only then the root-port config-space + * handoff left running: tunables and the reset table (T6050 has none), + * then PERST and APPCLK, and only then the root-port config-space * tunables. Stop before releasing the tunnel reset or starting link * training. */ @@ -1233,10 +1349,48 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) */ #define APPLE_PCIE_TUNNEL_READY_MS 1000 +/* + * The T6050 PCIe-C root port's ACK latency limit (0x700, low half) and flow + * control update timer (0x73c, bits 28:16) are sized for the max payload + * size. Each word holds both, for MPS 128, 256 and 512. + */ +static const u32 apple_pcie_mps_timers[] = { + 0x003c003c, 0x005a005a, 0x006b006b, +}; + +static void apple_pcie_update_mps_timers(struct apple_pcie *pcie, u32 mps) +{ + void __iomem *rp = pcie->cfg->win; + u32 cap, word, val; + + cap = apple_pcie_tunnel_readl(rp, PCIEC_RP_DEVCAP) & PCI_EXP_DEVCAP_PAYLOAD; + if (mps > cap || mps >= ARRAY_SIZE(apple_pcie_mps_timers)) { + dev_warn(pcie->dev, "no link timers for a %u-byte MPS\n", 128 << mps); + return; + } + + word = apple_pcie_mps_timers[mps]; + writew_relaxed(lower_16_bits(word), rp + PCIEC_RP_ACK_LATENCY); + /* Complete the write, as apple_pcie_tunnel_writel() does. */ + mb(); + isb(); + val = apple_pcie_tunnel_readl(rp, PCIEC_RP_FC_TIMER); + val &= PCIEC_RP_FC_TIMER_KEEP; + val |= FIELD_PREP(PCIEC_RP_FC_TIMER_LIMIT, upper_16_bits(word)) | + PCIEC_RP_FC_TIMER_EN; + apple_pcie_tunnel_writel(rp, val, PCIEC_RP_FC_TIMER); +} + +/* Process context: the link of a tunnel came up. */ static void apple_pcie_tunnel_link_up(struct apple_pcie *pcie) { WRITE_ONCE(pcie->tunnel_ready_at, jiffies + msecs_to_jiffies(APPLE_PCIE_TUNNEL_READY_MS)); + if (pcie->hw->mps_timers) + apple_pcie_update_mps_timers(pcie, + FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, + apple_pcie_tunnel_readl(pcie->cfg->win, + PCIEC_RP_DEVCTL))); } /* Call before scanning the bus behind a tunnel; elapsed time counts. */ @@ -1509,9 +1663,21 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) apple_pcie_port_disable_irq(port); /* Busy managed admission has not acquired or programmed this IRQ block. */ - if (port->base && (!port->pcie->managed_sids || port->domain)) + if (port->base && + (port->domain || + (!port->pcie->managed_sids && !port->pcie->hw->adopt_link))) apple_pcie_port_writel(port, ~0, PORT_INTMSK); + if (port->pcie->hw->adopt_link && port->domain) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + if (port->adopted_slots) { + int slot; + + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + apple_pcie_rid2sid_write(port, slot, 0); + port->adopted_slots = 0; + } + if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) apple_pcie_tunnel_stop(port); @@ -1595,6 +1761,591 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return apple_pcie_port_readl(port, offset); } +/* + * The DART stream an adopted port's function `rid` maps to: one T6050 DART stream, nonzero + * and below the RID2SID table size (slot == SID), from the host's iommu-map. + */ +static int apple_pcie_adopted_sid(struct apple_pcie_port *port, u32 rid, u32 *sid) +{ + struct apple_pcie *pcie = port->pcie; + struct of_phandle_args iommu = {}; + int ret; + + if (rid >> 8 != port->adopted_bus) { + /* The PCI core numbers root ports' buses in devfn order: the bootloader must too */ + dev_err(pcie->dev, "port %d: RID %04x is off bus %02x, which the bootloader numbered\n", + port->idx, rid, port->adopted_bus); + return -ENODEV; + } + ret = of_map_iommu_id(pcie->dev->of_node, rid, &iommu); + if (ret) + return ret; + ret = iommu.args_count == 1 && iommu.args[0] && + iommu.args[0] < pcie->hw->max_rid2sid && + iommu.args[0] < BITS_PER_LONG && + of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; + *sid = iommu.args[0]; + of_node_put(iommu.np); + return ret; +} + +/* Route an adopted port's function `rid` to its stream, or confirm the route is in place. */ +static int apple_pcie_adopted_route(struct apple_pcie_port *port, u32 rid) +{ + u32 sid, value, cur; + int ret = apple_pcie_adopted_sid(port, rid, &sid); + + if (ret) + return ret; + value = T6050_ROUTE(rid, sid); + cur = apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)); + if (cur == value && test_bit(sid, &port->adopted_slots)) + return 0; + if (cur & PORT_RID2SID_VALID) + return -EBUSY; + set_bit(sid, &port->adopted_slots); + if (apple_pcie_rid2sid_write(port, sid, value) != value) + return -EIO; + dev_info(port->pcie->dev, "port %d: RID %04x -> SID %u, slot %u\n", + port->idx, rid, sid, sid); + return 0; +} + +/* + * Qualified J775D handoff of one bootloader-trained port: its root port + * 00:.0 and the endpoint on its secondary bus (function 0 routed here; a + * multi-function endpoint's other functions when they are enabled), with DMA and + * endpoint MSI disabled. Every resource comes from the DT by the port's index + * (port, phy, intr2axi, interrupt port, the iommu-map entry for the + * endpoint's RID). All checks precede the first PCIe register write. + */ +static int apple_pcie_adopt_link(struct apple_pcie_port *port) +{ + static const char *const names[] = { "port%d", "phy%d", "intr2axi%d" }; + struct apple_pcie *pcie = port->pcie; + struct platform_device *pdev = to_platform_device(pcie->dev); + void __iomem *rp, *ep; + char name[16]; + u32 value, bus; + u8 cap; + bool msi = false; + int ret, i; + + if (pcie->cfg->busr.start != 0 || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "config") || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "rc")) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + snprintf(name, sizeof(name), names[i], port->idx); + if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) + return -EINVAL; + } + + if (!(readl(pcie->base + CORE_RC_CTL) & CORE_RC_CTL_RUN) || + !(readl(pcie->base + CORE_RC_STAT) & CORE_RC_STAT_READY) || + !(apple_pcie_port_readl(port, PORT_STATUS) & PORT_STATUS_READY) || + (apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP || + !(apple_pcie_port_readl(port, PORT_LTSSMCTL) & PORT_LTSSMCTL_START) || + (apple_pcie_port_readl(port, PORT_APPCLK) & 0x101) != 0x101 || + (apple_pcie_port_readl(port, pcie->hw->port_perst) & 0x10001) != 1 || + (readl(port->phy + PHY_LANE_CFG) & 0x1000061f) != 0x60f) + return -ENOLINK; + + /* The bootloader numbered the root port's buses: primary 0, one secondary. */ + rp = pcie->cfg->win + (PCI_DEVFN(port->idx, 0) << 12); + value = readl(rp + PCI_PRIMARY_BUS); + bus = (value >> 8) & 0xff; + if (readl(rp) != 0x100c106b || (value & 0xff) != 0 || !bus || + ((value >> 16) & 0xff) != bus || bus > pcie->cfg->busr.end || + (readw(rp + PCI_BRIDGE_CONTROL) & PCI_BRIDGE_CTL_BUS_RESET)) + return -ENODEV; + ep = pcie->cfg->win + (bus << 20); + value = readl(ep); + if (value == 0xffffffff || !(value & 0xffff) || + (readb(ep + PCI_HEADER_TYPE) & 0x7f) != PCI_HEADER_TYPE_NORMAL || + (readw(ep + PCI_COMMAND) & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY | + PCI_COMMAND_MASTER))) + return -ENODEV; + + port->adopted_bus = bus; + ret = apple_pcie_adopted_sid(port, bus << 8, &value); + if (ret) + return ret; + + snprintf(name, sizeof(name), "intr2axi%d", port->idx); + port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->intr2axi)) + return PTR_ERR(port->intr2axi); + /* + * Only a port whose endpoint needs a root-port reset names its PERST# line + * (reset-gpios, on the port node as on every other Apple host). The pin + * controller owns the pad; the line is taken as the bootloader left it, + * released, and driven only by the reset. + */ + port->perst = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(port->np), "reset", + GPIOD_ASIS, "PERST#"); + if (IS_ERR(port->perst)) { + if (PTR_ERR(port->perst) != -ENOENT) + return PTR_ERR(port->perst); + port->perst = NULL; + } + + /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ + cap = readb(rp + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + if (readb(rp + cap + PCI_CAP_LIST_ID) == PCI_CAP_ID_EXP) + break; + cap = readb(rp + cap + PCI_CAP_LIST_NEXT); + } + if (!cap || i == 48) + return -EINVAL; + /* + * x1 at up to the Gen3 target the bootloader set. The endpoint picks the speed: the N1's + * ROM personality links at Gen1 and comes back at Gen3 after its root-port reset. + */ + value = readw(rp + cap + PCI_EXP_LNKSTA); + if ((value & (PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != (1 << PCI_EXP_LNKSTA_NLW_SHIFT) || + (value & PCI_EXP_LNKSTA_CLS) < PCI_EXP_LNKSTA_CLS_2_5GB || + (value & PCI_EXP_LNKSTA_CLS) > PCI_EXP_LNKSTA_CLS_8_0GB) + return -ENOLINK; + + /* Refuse an active endpoint or malformed capability chain. */ + cap = readb(ep + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + value = readw(ep + cap + PCI_MSI_FLAGS); + switch (readb(ep + cap + PCI_CAP_LIST_ID)) { + case PCI_CAP_ID_MSI: + if (value & PCI_MSI_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + case PCI_CAP_ID_MSIX: + if (value & PCI_MSIX_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + } + cap = readb(ep + cap + PCI_CAP_LIST_NEXT); + } + if (cap || !msi) + return -EINVAL; + + for (i = 0; i < pcie->hw->max_rid2sid; i++) + if (apple_pcie_port_readl(port, port_rid2sid_offset(port, i)) & + PORT_RID2SID_VALID) + return -EBUSY; + + /* The route goes in the slot whose index is the SID, as on J775D's adopted ports. */ + ret = apple_pcie_adopted_route(port, bus << 8); + if (!ret) + dev_info(pcie->dev, "adopted port %d, endpoint %02x:00\n", port->idx, bus); + return ret; +} + +/* + * T6050 port bring-up, for an adopted port whose endpoint must come back on a + * retrained link: port disable, the port reset table, port enable, root complex + * configuration for Gen3 x1, then the lane PHY, PERST# and the LTSSM. The port's + * registers are its port, phy and intr2axi blocks; the root complex's + * are the root port's config space. The boot loader copies the port's tunables + * onto its node. + */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001001ff }, + { 0x82c, 0x00010000 }, +}; + +/* Written after the RID2SID and MSI map tables are cleared */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs_late[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static void apple_pcie_t6050_rmw(void __iomem *addr, u32 clear, u32 set) +{ + writel((readl(addr) & ~clear) | set, addr); +} + +static int apple_pcie_t6050_rc_rmw(struct pci_dev *rp, int where, u32 clear, u32 set) +{ + u32 value; + int ret; + + ret = pci_read_config_dword(rp, where, &value); + if (!ret) + ret = pci_write_config_dword(rp, where, (value & ~clear) | set); + return pcibios_err_to_errno(ret); +} + +static int apple_pcie_t6050_wait(struct apple_pcie_port *port, void __iomem *base, + u32 offset, u32 mask, u32 value, unsigned int timeout_us) +{ + u32 stat; + int ret; + + ret = readl_poll_timeout(base + offset, stat, (stat & mask) == value, 1, timeout_us); + if (ret) + dev_err(port->pcie->dev, "port %d: %#x = %#x, waiting for %#x/%#x\n", + port->idx, offset, stat, value, mask); + return ret; +} + +/* + * A tunable list the boot loader copies onto the root port node, groups of + * (offset, 4, mask, value): for the port's registers, or with @rp for the root + * port's config space. A list it did not copy is skipped. + */ +static int apple_pcie_t6050_tune(struct apple_pcie_port *port, struct pci_dev *rp, + const char *name) +{ + u32 cells[64]; + int count, ret, i; + + if (!of_property_present(port->np, name)) + return 0; + count = of_property_count_u32_elems(port->np, name); + if (count <= 0 || count > ARRAY_SIZE(cells) || count % 4) + return -EINVAL; + ret = of_property_read_u32_array(port->np, name, cells, count); + if (ret) + return ret; + for (i = 0; i < count; i += 4) + if (cells[i + 1] != 4 || (cells[i] & 3) || + cells[i] >= (rp ? PCI_CFG_SPACE_EXP_SIZE : SZ_16K)) + return -EINVAL; + + for (i = 0; i < count; i += 4) { + if (!rp) { + apple_pcie_t6050_rmw(port->base + cells[i], cells[i + 2], cells[i + 3]); + continue; + } + ret = apple_pcie_t6050_rc_rmw(rp, cells[i], cells[i + 2], cells[i + 3]); + if (ret) + return ret; + } + return 0; +} + +/* + * PERST# through the pin controller: the pad as a GPIO output (its function bits + * cleared, mode output) at the line's logical level, which the DT's + * GPIO_ACTIVE_LOW turns into the pad level. + */ +static int apple_pcie_t6050_perst(struct apple_pcie_port *port, bool asserted) +{ + return gpiod_direction_output(port->perst, asserted); +} + +/* Let the link go idle, hold the endpoint in reset and stop the port and its lane clocks */ +static int apple_pcie_t6050_port_disable(struct apple_pcie_port *port, struct pci_dev *rp) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + void __iomem *lane = port->phy + PHY_LANE_CFG; + u32 stat; + int ret, err; + + if (!(readl(port->base + PORT_STATUS) & BIT(2))) { + ret = apple_pcie_t6050_wait(port, port->base, 0x820, BIT(2), BIT(2), 10000); + if (ret) + return ret; + } + ret = readl_poll_timeout_atomic(port->base + 0x820, stat, + !(stat & GENMASK(8, 0)) || !(stat & GENMASK(23, 16)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + 0x13c, 0, BIT(8)); + + /* The link out of L1 */ + ret = read_poll_timeout_atomic(pci_read_config_dword, err, + err || FIELD_GET(RC_T6050_LTSSM_STATE, stat) != + RC_T6050_LTSSM_L1_IDLE, + 5, 5000, false, rp, RC_T6050_PORT_DEBUG0, &stat); + if (ret || err) + return ret ?: pcibios_err_to_errno(err); + + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(24)); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(16), BIT(16), 2000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(16)); + udelay(1); + apple_pcie_t6050_rmw(perst, PORT_PERST_OFF, 0); + udelay(1); + apple_pcie_t6050_rmw(perst, BIT(24), 0); + apple_pcie_t6050_rmw(port->base + 0x13c, BIT(8), 0); + + apple_pcie_t6050_rmw(lane, BIT(31), 0); + apple_pcie_t6050_rmw(lane, BIT(10), 0); + apple_pcie_t6050_rmw(lane, BIT(9), 0); + apple_pcie_t6050_rmw(lane, 0, BIT(4)); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK0REQ, 0); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK1REQ, 0); + + writel(~0, port->base + PORT_INTSTAT); + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, PORT_APPCLK_EN, 0); + return apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, 0, + 250000); +} + +/* The port reset table, then the port clocked and out of reset, ready to train */ +static int apple_pcie_t6050_port_enable(struct apple_pcie_port *port) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + u32 stat; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs); i++) + writel(apple_pcie_t6050_reset_regs[i].value, + port->base + apple_pcie_t6050_reset_regs[i].offset); + for (i = 0; i < port->pcie->hw->max_rid2sid; i++) + writel(0, port->base + port_rid2sid_offset(port, i)); + for (i = 0; i < 256; i++) + writel(0, port->base + PORT_T602X_MSIMAP + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs_late); i++) + writel(apple_pcie_t6050_reset_regs_late[i].value, + port->base + apple_pcie_t6050_reset_regs_late[i].offset); + + if (!(readl(perst) & BIT(16))) + return -EIO; + ret = apple_pcie_t6050_tune(port, NULL, "apple,apcie-config-tunables"); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_EN); + + ret = readl_poll_timeout_atomic(port->base + 0xa8, stat, + (stat & (BIT(16) | BIT(0))) != (BIT(16) | BIT(0)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, PORT_PERST_OFF); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 20000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, BIT(16), 0); + ret = apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, + PORT_STATUS_READY, 250000); + if (ret) + return ret; + + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_CGDIS); + return apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_BUSY, 0, + 250000); +} + +/* Root complex tunables and a Gen3 x1 link, with the read-only registers opened for it */ +static int apple_pcie_t6050_port_configure(struct apple_pcie_port *port, struct pci_dev *rp) +{ + static const char * const shadow[] = { + "apple,pcie-rc-gen3-shadow-tunables", + "apple,pcie-rc-gen4-shadow-tunables", + "apple,pcie-rc-gen5-shadow-tunables", + }; + static const u32 shadow_sel[] = { 0, BIT(24), 0 }; + int cap = pci_pcie_cap(rp); + u32 value; + u16 ctl2; + int ret, err, i; + + if (!cap) + return -EIO; + ret = apple_pcie_t6050_rc_rmw(rp, RC_T6050_MISC_CONTROL_1, 0, RC_T6050_DBI_RO_WR_EN); + if (ret) + return ret; + ret = apple_pcie_t6050_tune(port, rp, "apple,pcie-rc-tunables"); + if (ret) + goto lock; + for (i = 0; i < ARRAY_SIZE(shadow); i++) { + ret = apple_pcie_t6050_rc_rmw(rp, RC_T6050_GEN3_RELATED, + RC_T6050_RATE_SHADOW_SEL, shadow_sel[i]) ?: + apple_pcie_t6050_tune(port, rp, shadow[i]); + if (ret) + goto lock; + } + ret = apple_pcie_t6050_rc_rmw(rp, 0x194, 0, BIT(0)); + if (ret) + goto lock; + + ret = pcibios_err_to_errno(pci_read_config_dword(rp, cap + PCI_EXP_LNKCAP2, &value)); + if (ret) + goto lock; + if (!(value & PCI_EXP_LNKCAP2_SLS_8_0GB)) { + ret = -EIO; + goto lock; + } + ret = pcibios_err_to_errno(pci_read_config_word(rp, cap + PCI_EXP_LNKCTL2, &ctl2)) ?: + pcibios_err_to_errno(pci_write_config_word(rp, cap + PCI_EXP_LNKCTL2, + (ctl2 & ~PCI_EXP_LNKCTL2_TLS) | + PCI_EXP_LNKCTL2_TLS_8_0GT)) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_PORT_LINK_CTRL, RC_T6050_LINK_MODE, + FIELD_PREP(RC_T6050_LINK_MODE, 1)) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_LINK_WIDTH_SPEED_CTRL, RC_T6050_LINK_WIDTH, + FIELD_PREP(RC_T6050_LINK_WIDTH, 1)) ?: + apple_pcie_t6050_rc_rmw(rp, cap + PCI_EXP_LNKCAP, PCI_EXP_LNKCAP_MLW, + FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1)); +lock: + err = apple_pcie_t6050_rc_rmw(rp, RC_T6050_MISC_CONTROL_1, RC_T6050_DBI_RO_WR_EN, 0); + ret = ret ?: err; + if (ret) + return ret; + + ret = apple_pcie_t6050_rc_rmw(rp, cap + PCI_EXP_LNKCTL, 0, PCI_EXP_LNKCTL_LBMIE) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_LINK_WIDTH_SPEED_CTRL, 0, RC_T6050_SPEED_CHANGE); + if (ret) + return ret; + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + return 0; +} + +/* Disable, reset, enable and configure an adopted port whose endpoint is held in PERST# */ +static int apple_pcie_t6050_port_restart(struct apple_pcie_port *port, struct pci_dev *rp) +{ + return apple_pcie_t6050_port_disable(port, rp) ?: + apple_pcie_t6050_port_enable(port) ?: + apple_pcie_t6050_port_configure(port, rp); +} + +/* Lane clocks, PERST# released, then the LTSSM to link up */ +static int apple_pcie_t6050_port_train(struct apple_pcie_port *port) +{ + void __iomem *lane = port->phy + PHY_LANE_CFG; + int ret; + + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK0REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK0ACK, + PHY_LANE_CFG_REFCLK0ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK1REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK1ACK, + PHY_LANE_CFG_REFCLK1ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, BIT(4), 0); + udelay(1); + apple_pcie_t6050_rmw(lane, 0, BIT(9)); + apple_pcie_t6050_rmw(lane, 0, BIT(10)); + + /* Tperst-clk is at least 100 us (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 110); + ret = apple_pcie_t6050_perst(port, false); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_LTSSMCTL, 0, PORT_LTSSMCTL_START); + ret = apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_UP, + PORT_LINKSTS_UP, 500000); + if (ret) + return ret; + + /* Wait for 100ms after the link trains (PCIe r5.0, 6.6.1) */ + msleep(100); + return 0; +} + +/* + * Root-port reset of an adopted T6050 port, for an endpoint that must come back on a + * retrained link (the N1 after its ROM boot): PME_Turn_Off to L2, PERST# asserted, the + * port disabled, reset, enabled and configured, its routes and MSI map rewritten, then the + * lane PHY, PERST# release and LTSSM to a Gen3 x1 link. The caller has quiesced the + * endpoint and saved the root port's state; the PCI core restores it. + */ +static int apple_pcie_t6050_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *rp) +{ + struct pci_config_window *cfg = rp->sysdata; + struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); + struct apple_pcie_port *port = NULL, *p; + u32 routes[BITS_PER_LONG], value; + int ret, slot, i; + + if (!pcie || !pcie->hw->adopt_link || !pci_is_root_bus(rp->bus)) + return -EOPNOTSUPP; + list_for_each_entry(p, &pcie->ports, entry) + if (p->idx == PCI_SLOT(rp->devfn)) + port = p; + if (!port || !port->perst || !port->intr2axi) + return -EOPNOTSUPP; + + mutex_lock(&pcie->lock); + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + routes[slot] = apple_pcie_port_readl(port, port_rid2sid_offset(port, slot)); + + /* Bridge decoding off and the secondary bus unnumbered while the link is down */ + ret = pcibios_err_to_errno(pci_write_config_word(rp, PCI_COMMAND, 0)) ?: + pcibios_err_to_errno(pci_read_config_dword(rp, PCI_PRIMARY_BUS, &value)) ?: + pcibios_err_to_errno(pci_write_config_dword(rp, PCI_PRIMARY_BUS, + value & 0xff0000ff)); + if (ret) + goto out; + /* PME_Turn_Off, then L2 */ + apple_pcie_port_writel(port, apple_pcie_port_readl(port, PORT_APPCLK) | 0x100, PORT_APPCLK); + apple_pcie_port_writel(port, 0x11, 0x88); + ret = apple_pcie_t6050_wait(port, port->base, 0x88, 1, 0, 500000) ?: + apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, BIT(6), BIT(6), 500000); + if (ret) + goto out; + WRITE_ONCE(port->link_failed, true); + ret = apple_pcie_t6050_perst(port, true); + if (ret) + goto out; + writel(readl(port->phy) & ~BIT(31), port->phy); + ret = apple_pcie_t6050_port_restart(port, rp); + if (ret) + goto out; + + /* The reset table cleared the routes and the MSI setup: put back what Linux set */ + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + if (apple_pcie_rid2sid_write(port, slot, routes[slot]) != routes[slot]) + ret = -EIO; + apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + apple_pcie_port_writel(port, 0, pcie->hw->port_msiaddr_hi); + for (i = 0; i < pcie->nvecs; i++) + apple_pcie_port_writel(port, FIELD_PREP(PORT_MSIMAP_TARGET, pcie->msi_vector_base + i) | + PORT_MSIMAP_ENABLE, pcie->hw->port_msimap + 4 * i); + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + apple_pcie_port_writel(port, PORT_MSICFG_EN, PORT_MSICFG); + if (ret) + goto out; + + ret = apple_pcie_t6050_port_train(port); + if (ret) + goto out; + /* Link interrupts stay masked after the reset table: the link is checked on access */ + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + WRITE_ONCE(port->link_failed, false); + dev_info(pcie->dev, "port %d: link retrained after a root-port reset\n", port->idx); +out: + mutex_unlock(&pcie->lock); + if (ret) + dev_err(pcie->dev, "port %d: root-port reset stopped: %d\n", port->idx, ret); + return ret; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) @@ -1881,6 +2632,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, goto err_teardown; } } + } else if (pcie->hw->adopt_link) { + ret = apple_pcie_adopt_link(port); + if (ret) + goto err_teardown; } else { if (pcie->hw->retain_clocks) rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); @@ -1922,9 +2677,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (ret) goto err_teardown; - if (pcie->managed_sids) { + if (pcie->managed_sids || pcie->hw->adopt_link) goto rid2sid_ready; - } /* Reset all RID/SID mappings, and check for RAZ/WI registers */ for (i = 0; i < pcie->hw->max_rid2sid; i++) { @@ -1949,6 +2703,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, unsigned long timeout, left = 0; int attempt, attempts = 1; + if (pcie->hw->adopt_link) { + ret = -ENOLINK; + goto err_teardown; + } if (pcie->hw->tunneled) { if (pcie->kernel_init) { apple_pcie_tunnel_apply_tunable(port->base, @@ -2095,6 +2853,34 @@ static int apple_msi_init(struct apple_pcie *pcie) return ret; } + if (pcie->hw->adopt_link) { + ret = of_property_read_u32(to_of_node(fwnode), "apple,msi-vector-base", + &pcie->msi_vector_base); + if (ret || pcie->msi_vector_base != 64 || + (pcie->nvecs != 32 && pcie->nvecs != 64) || + args.args_count != 4 || args.args[0] != 0 || args.args[1] != 0 || + args.args[2] != 1888 || args.args[3] != IRQ_TYPE_EDGE_RISING || + DOORBELL_ADDR != 0xfffff000) + goto invalid_slice; + } + + /* + * T6050's PCIe hosts share one MSI window, vector n at AIC 1824 + n. + * The internal host's bridges own vectors 0..127 and each PCIe-C + * bridge a 32-vector slice of the rest. + */ + if (pcie->hw == &t6050_pciec_hw) { + ret = of_property_read_u32(to_of_node(fwnode), "apple,msi-vector-base", + &pcie->msi_vector_base); + if (ret || pcie->msi_vector_base < 128 || pcie->msi_vector_base % 32 || + pcie->nvecs != 32 || + pcie->msi_vector_base + pcie->nvecs > FIELD_MAX(PORT_MSIMAP_TARGET) + 1 || + args.args_count != 4 || args.args[0] != 0 || args.args[1] != 0 || + args.args[2] != 1824 + pcie->msi_vector_base || + args.args[3] != IRQ_TYPE_EDGE_RISING || DOORBELL_ADDR != 0xfffff000) + goto invalid_slice; + } + /* * The controller-wide ADT count (512) is not a per-host allocation. * Internal PCIe owns vectors 0..31. Each PCIe-C bridge owns a distinct @@ -2218,7 +3004,7 @@ static int apple_msi_init(struct apple_pcie *pcie) invalid_slice: of_node_put(args.np); return dev_err_probe(pcie->dev, -EINVAL, - "invalid or missing PCIe-C MSI slice\n"); + "invalid or missing MSI slice\n"); } static void apple_pcie_cleanup(void *data) @@ -2419,6 +3205,14 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d of_node_put(iommu_spec.np); sid = iommu_spec.args[0]; + if (pcie->hw->adopt_link) { + if (READ_ONCE(port->link_failed)) + return -ENODEV; + mutex_lock(&pcie->lock); + err = apple_pcie_adopted_route(port, rid); + mutex_unlock(&pcie->lock); + return err; + } mutex_lock(&port->pcie->lock); if (port->pcie->managed_sids || port->pcie->hw->sid_indexed_rid2sid) { @@ -2485,7 +3279,7 @@ static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci int idx; port = apple_pcie_get_port(pdev); - if (!port) + if (!port || port->pcie->hw->adopt_link) return; mutex_lock(&port->pcie->lock); @@ -2526,9 +3320,13 @@ static int apple_pcie_init(struct pci_config_window *cfg) ret = apple_pcie_setup_port(pcie, of_port); if (ret) { dev_err(dev, "Port %pOF setup fail: %d\n", of_port, ret); - return ret; + /* Adopted ports are independent: one refused leaves the others. */ + if (!pcie->hw->adopt_link) + return ret; } } + if (pcie->hw->adopt_link && list_empty(&pcie->ports)) + return -ENODEV; /* The bus scan follows this callback. */ apple_pcie_tunnel_wait_ready(pcie); @@ -2548,6 +3346,27 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); struct apple_pcie_port *port; + if (pcie->hw->adopt_link) { + /* Only adopted ports' root ports, and the endpoint behind each. */ + list_for_each_entry(port, &pcie->ports, entry) { + if (bus->number == 0) { + if (devfn == PCI_DEVFN(port->idx, 0)) + return pci_ecam_map_bus(bus, devfn, where); + continue; + } + if (bus->number != port->adopted_bus) + continue; + if (PCI_SLOT(devfn) || READ_ONCE(port->link_failed)) + return NULL; + if ((apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP) { + WRITE_ONCE(port->link_failed, true); + return NULL; + } + return pci_ecam_map_bus(bus, devfn, where); + } + return NULL; + } if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); @@ -2578,11 +3397,17 @@ static int apple_pcie_config_write(struct pci_bus *bus, unsigned int devfn, int ret; ret = pci_generic_config_write(bus, devfn, where, size, val); - if (ret || !pcie || !pcie->kernel_init) + if (ret || !pcie) return ret; - /* Root-port prefetchable window. The port block has to be told too. */ if (bus->number != cfg->busr.start || devfn) return ret; + /* The root port's link timers follow its max payload size. */ + if (pcie->hw->mps_timers && where == PCIEC_RP_DEVCTL) + apple_pcie_update_mps_timers(pcie, + FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, val)); + if (!pcie->kernel_init) + return ret; + /* Root-port prefetchable window. The port block has to be told too. */ if (where < 0x24 || where >= 0x30) return ret; @@ -3121,7 +3946,32 @@ static int apple_pcie_probe(struct platform_device *pdev) * reset-gpios provider here rejects the valid synthesized port before * the tunneled setup path can consume that handoff. */ - if (!hw->tunneled) { + if (hw->adopt_link) { + if (!of_get_available_child_count(dev->of_node) || + of_property_present(dev->of_node, "power-domains")) + return -EINVAL; + for_each_available_child_of_node_scoped(dev->of_node, child) { + u32 reg; + + /* A root port 00:.0 on bus 0, n a GP port (0-3) */ + if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || + (reg >> 11) > 3 || + !of_property_read_bool(child, "apple,link-trained-by-bootloader") || + of_property_present(child, "pwren-gpios")) + return -EINVAL; + /* + * A port whose endpoint needs a root-port reset names its PERST# + * line: probe waits for the line's controller before any port is + * touched (a failed adoption would otherwise swallow the deferral). + */ + if (of_property_present(child, "reset-gpios")) { + ret = apple_pcie_probe_port(child); + if (ret) + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", + child); + } + } + } else if (!hw->tunneled) { /* Check for probe dependencies for all ports first */ for_each_available_child_of_node(dev->of_node, of_port) { ret = apple_pcie_probe_port(of_port); @@ -3164,15 +4014,15 @@ static int apple_pcie_probe(struct platform_device *pdev) return ret; } pcie->piodma_supplier = piodma_supplier; - /* Only this retained-DMA radio hierarchy forbids user reset/removal. */ - if (piodma_supplier || pcie->managed_sids) { + /* Firmware-owned links cannot be recovered by user reset/removal. */ + if (piodma_supplier || pcie->managed_sids || hw->adopt_link) { bridge->no_user_remove = true; bridge->reset_sensitive = true; bridge->native_pcie_hotplug = false; bridge->native_shpc_hotplug = false; bridge->native_aer = false; bridge->native_dpc = false; - dev_dbg(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); + dev_dbg(dev, "firmware-owned host: automatic AER/DPC recovery disabled\n"); } if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); @@ -3238,6 +4088,8 @@ static int apple_pcie_probe(struct platform_device *pdev) } } + if (hw->adopt_link) + bridge->reset_root_port = apple_pcie_t6050_reset_root_port; ret = pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); if (ret) { apple_pcie_cleanup(pcie); @@ -3952,6 +4804,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) */ if (pcie->managed_sids) return pm_suspend_target_state == PM_SUSPEND_TO_IDLE ? 0 : -EBUSY; + if (pcie->hw->adopt_link) + return -EBUSY; if (!pcie->hw->tunneled) return 0; pcie->reset_on_resume = false; @@ -4085,6 +4939,7 @@ static const struct dev_pm_ops apple_pcie_pm_ops = { static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t6040-pciec", .data = &t6040_pciec_hw }, + { .compatible = "apple,t6050-pciec", .data = &t6050_pciec_hw }, { .compatible = "apple,t8132-pciec", .data = &t8132_pciec_hw }, { .compatible = "apple,t8122-pciec", .data = &t8122_pciec_hw }, { .compatible = "apple,t8112-pciec", .data = &t8112_pciec_hw }, @@ -4095,6 +4950,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8152-pcie", .data = &t8152_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, + { .compatible = "apple,t6050-pcie", .data = &t6050_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } }; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 27008e2ea5afc..d625dd5e778a8 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -3517,6 +3517,13 @@ void pci_lock_rescan_remove(void) } EXPORT_SYMBOL_GPL(pci_lock_rescan_remove); +/* Work cancelled by removal must not wait on the remover's own lock. */ +bool pci_trylock_rescan_remove(void) +{ + return mutex_trylock(&pci_rescan_remove_lock); +} +EXPORT_SYMBOL_GPL(pci_trylock_rescan_remove); + void pci_unlock_rescan_remove(void) { mutex_unlock(&pci_rescan_remove_lock); diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 5348987cf1dc8..36ed85b7935f2 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -62,3 +62,16 @@ config PHY_APPLE_DPTX such as the M2. This driver provides support for DisplayPort and is used on the Mac mini (M2 and M2 Pro, 2023). + +config PHY_APPLE_T6050_DP + tristate "Apple T6050 DisplayPort and Type-C combo PHY" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF && PM && TYPEC + select GENERIC_PHY + select APPLE_TUNABLE + select APPLE_RTKIT + select RESET_CONTROLLER + help + Fixed HDMI bridges and Type-C USB host/DisplayPort alternate mode. + One owner coordinates the shared PHY, firmware, PIPE and USB2 state. + Requires named register banks, power domains and firmware tunables. diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile index e7156408e792f..e0b94bb1dcaa5 100644 --- a/drivers/phy/apple/Makefile +++ b/drivers/phy/apple/Makefile @@ -8,3 +8,5 @@ phy-apple-t6040-usb2-y := t6040-usb2.o obj-$(CONFIG_PHY_APPLE_DPTX) += phy-apple-dptx.o phy-apple-dptx-y += dptx.o + +obj-$(CONFIG_PHY_APPLE_T6050_DP) += atc-t6050-dp.o diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h new file mode 100644 index 0000000000000..f1e0ea6238ab9 --- /dev/null +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -0,0 +1,527 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* One Type-C owner reuses the established DP and USB backend sequences. + * All calls below hold dp->lock; only PHY power_off authorizes PIPE teardown + * while the controller reset is deasserted. Lane changes require reset/off. + */ +static u32 combo_read(void *ctx, unsigned int bank, unsigned int offset) +{ + struct t6050_dp *dp = ctx; + return readl(dp->banks[bank] + offset); +} +static void combo_write(void *ctx, unsigned int bank, unsigned int offset, u32 value) +{ + struct t6050_dp *dp = ctx; + writel(value, dp->banks[bank] + offset); +} +static int combo_poll(void *ctx, unsigned int bank, unsigned int offset, + unsigned int mask, unsigned int expected, unsigned int usec) +{ + struct t6050_dp *dp = ctx; + u32 value; + int ret = readl_poll_timeout(dp->banks[bank] + offset, value, + (value & mask) == expected, + (bank == 44 || bank == 35 || usec <= 1000) ? 1 : 1000, usec); + + /* The PIPE switch carries on without the lock ACK; say so. */ + if (ret && bank == 44 && offset == 0x14 && usec == 6000) + dev_warn(dp->dev, + "PIPE %slock not acknowledged; continuing (override=%x values=%x mux=%x req=%x ack=%x aon=%x nonsel=%x)\n", + expected ? "" : "un", readl(dp->banks[44]), + readl(dp->banks[44] + 4), readl(dp->banks[44] + 0xc), + readl(dp->banks[44] + 0x10), value, + readl(dp->banks[44] + 0x1c), readl(dp->banks[44] + 0x20)); + return ret; +} +static void combo_delay(void *ctx, unsigned int usec) { udelay(usec); } +static void combo_pause(void *ctx, unsigned int nsec) { ndelay(nsec); } +static void combo_sleep(void *ctx, unsigned int msec) { msleep(msec); } +static int combo_wrapper(void *ctx, unsigned int active) +{ + struct t6050_dp *dp = ctx; + return atc_domain_set(&dp->domain[2], active); +} +static int combo_firmware(void *ctx, unsigned int active) +{ + return atc_firmware(ctx, active); +} +static int combo_tunable(void *ctx, unsigned int bank, const char *name) +{ + static const struct { u8 bank, index; const char *name; } common[] = { + {41, 0, "tunable_ATC0AXI2AF"}, {40, 1, "tunable_ATC_FABRIC"}, + {43, 2, "tunable_ATC_COMMON_CFG"}, {42, 3, "tunable_ATC0AXI2AF_LIOA"}, + {20, 4, "tunable_LN0_RX_CFG_TX_OF_RXCLK"}, {29, 5, "tunable_LN1_RX_CFG_TX_OF_RXCLK"}, + {18, 6, "tunable_LN0_RX_EQ_DFLT"}, {27, 7, "tunable_LN1_RX_EQ_DFLT"}, + {8, 8, "tunable_AUSPLL_CORE"}, {5, 9, "tunable_AUS40CMN_SHM"}, + {10, 10, "tunable_CIO4PLL_CORE"}, + }; + static const struct { u8 bank, index; const char *name; } cio[] = { + {3, 0, "tunable_UC_REGS_CIO_DFLT"}, {16, 1, "tunable_LN0_RX_TOP_CIO_DFLT"}, + {25, 2, "tunable_LN1_RX_TOP_CIO_DFLT"}, {18, 3, "tunable_LN0_RX_EQ_CIO_DFLT"}, + {27, 4, "tunable_LN1_RX_EQ_CIO_DFLT"}, + }; + struct t6050_dp *dp = ctx; + + if (bank == 0 && !strcmp(name, "tunable_USB2PHY_DFLT")) { + apple_tunable_apply(dp->banks[0], dp->usb2_default); + return 0; + } + if (bank == 0 && !strcmp(name, "tunable_USB2PHY_HOST")) { + apple_tunable_apply(dp->banks[0], dp->usb2_host); + return 0; + } + for (unsigned int i = 0; i < ARRAY_SIZE(common); i++) + if (bank == common[i].bank && !strcmp(name, common[i].name)) { + apple_tunable_apply(dp->banks[bank], dp->tunables[common[i].index]); + return 0; + } + /* A missing CIO80 tunable is skipped. */ + for (unsigned int i = 0; i < ARRAY_SIZE(cio); i++) + if (bank == cio[i].bank && !strcmp(name, cio[i].name)) { + apple_tunable_apply(dp->banks[bank], dp->cio_tunables[cio[i].index]); + return 0; + } + return -EINVAL; +} + +static int combo_quiesce(void *ctx) +{ + struct t6050_dp *dp = ctx; + int ret = 0; + + if (dp->usb_prepared) { + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_shutdown); + if (!ret) + dp->usb_prepared = false; + } else if (dp->cio_prepared) { + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_cio_shutdown); + if (!ret) + dp->cio_prepared = false; + } else if (dp->enabled) { + /* Multifunction has a distinct native shutdown; never run the + * USB-only analog sequence over live DisplayPort lane state. */ + ret = atc_stop(dp); + } + return ret; +} + +/* The pipehandler bank is the controller's: read it only while the controller holds its domain. */ +static void combo_pipe_fault(struct t6050_dp *dp, const char *what, int ret) +{ + dev_err(dp->dev, "%s failed (%d): pipe 00=%08x 04=%08x 0c=%08x 10=%08x 14=%08x 1c=%08x 20=%08x\n", + what, ret, readl(dp->banks[44]), readl(dp->banks[44] + 0x4), + readl(dp->banks[44] + 0xc), readl(dp->banks[44] + 0x10), + readl(dp->banks[44] + 0x14), readl(dp->banks[44] + 0x1c), + readl(dp->banks[44] + 0x20)); + dp->faulted = true; +} + +static int combo_stop(struct t6050_dp *dp) +{ + int ret; + + lockdep_assert_held(&dp->lock); + if (dp->faulted) + return -EIO; + /* No tunnel clock outlives USB4 mode. */ + atc_dp_tunnels_stop(dp); + if (dp->controller_active || dp->pipe_active) + return -EBUSY; + ret = combo_quiesce(dp); + if (!ret && dp->usb2_active) { + ret = apple_t6050_usb2_stop(&dp->usb_platform); + if (!ret) + dp->usb2_active = false; + } + if (!ret && dp->ndomains > 3) + ret = atc_domain_set(&dp->domain[3], false); + if (!ret) + ret = atc_domain_set(&dp->domain[1], false); + if (!ret) + ret = atc_domain_set(&dp->domain[0], false); + if (ret) { + dev_err(dp->dev, "stop failed (%d)\n", ret); + dp->faulted = true; + return ret < 0 ? ret : -EIO; + } + dp->combo_mode = COMBO_SAFE; + return 0; +} + +static int combo_start(struct t6050_dp *dp, enum combo_mode mode) +{ + int ret; + + lockdep_assert_held(&dp->lock); + if (dp->faulted) + return -EIO; + if (dp->orientation == TYPEC_ORIENTATION_NONE) + return -ENOLINK; + ret = atc_domain_set(&dp->domain[0], true); + if (!ret) + ret = atc_domain_set(&dp->domain[1], true); + /* + * The USB2 PHY and digital banks are unclocked without the USB domain, + * which a USB controller holds only while one is bound to this port. + */ + if (!ret && dp->ndomains > 3) + ret = atc_domain_set(&dp->domain[3], true); + if (!ret) + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb2_host); + if (!ret) + dp->usb2_active = true; + if (!ret && mode == COMBO_USB) { + ret = dp->reverse ? + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_init_reverse) : + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_init_normal); + if (!ret) + dp->usb_prepared = true; + } else if (!ret && mode == COMBO_CIO) { + ktime_t start = ktime_get(); + + /* USB2 stays native beside a USB4/TBT link; USB3 comes tunnelled. */ + ret = dp->reverse ? + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_cio_init_reverse) : + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_cio_init_normal); + if (!ret) { + dp->cio_prepared = true; + atc_dp_tunnels_park(dp); + } + dev_info(dp->dev, "CIO mode %s (%d), %s orientation, %lld ms\n", + ret ? "failed" : "up: UC_TNBL_DONE", ret, + dp->reverse ? "reverse" : "normal", + ktime_ms_delta(ktime_get(), start)); + } else if (!ret) { + dp->multifunction = mode == COMBO_DP2; + ret = atc_start(dp); + } + if (ret) { + /* A partial shared transition has no cold-start ownership proof. */ + dev_err(dp->dev, "start of mode %d failed (%d)\n", mode, ret); + dp->faulted = true; + return ret < 0 ? ret : -EIO; + } + dp->combo_mode = mode; + return 0; +} + +static int combo_orientation(struct t6050_dp *dp, enum typec_orientation orientation) +{ + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (orientation != TYPEC_ORIENTATION_NONE && orientation != TYPEC_ORIENTATION_NORMAL && + orientation != TYPEC_ORIENTATION_REVERSE) + return -EINVAL; + if (dp->orientation == orientation) + return 0; + ret = combo_stop(dp); + if (!ret) { + dp->orientation = orientation; + dp->reverse = orientation == TYPEC_ORIENTATION_REVERSE; + } + return ret; +} + +static int combo_mux(struct t6050_dp *dp, struct typec_mux_state *state) +{ + enum combo_mode mode; + int ret; + + if (state->mode == TYPEC_STATE_SAFE) + mode = COMBO_SAFE; + else if (state->mode == TYPEC_STATE_USB && !state->alt) + mode = COMBO_USB; + else if (!state->alt && state->mode == TYPEC_MODE_USB4) + mode = COMBO_CIO; + else if (state->alt && state->alt->svid == USB_TYPEC_TBT_SID && + state->mode == TYPEC_TBT_MODE) + mode = COMBO_CIO; + else if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: mode = COMBO_DP4; break; + case TYPEC_DP_STATE_D: mode = COMBO_DP2; break; + default: return -EOPNOTSUPP; + } + } else { + return -EOPNOTSUPP; + } + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (mode == dp->combo_mode && + (mode == COMBO_SAFE || dp->usb_prepared || dp->cio_prepared || dp->enabled)) + return 0; + if (mode != COMBO_SAFE && dp->orientation == TYPEC_ORIENTATION_NONE) + return -ENOLINK; + ret = combo_stop(dp); + if (ret || mode == COMBO_SAFE) + return ret; + return combo_start(dp, mode); +} + +static int combo_usb2_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (mode != PHY_MODE_USB_HOST) + return -EOPNOTSUPP; + if (!dp->usb_prepared && !dp->cio_prepared && !dp->enabled) + return -ENOLINK; + if (dp->usb2_active) + return 0; + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb2_host); + if (ret) { + dev_err(dp->dev, "USB2 host start failed (%d)\n", ret); + dp->faulted = true; + } else { + dp->usb2_active = true; + } + return ret; +} + +static int combo_usb2_reset(struct phy *phy) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted || !dp->usb2_active) + return -ENOLINK; + ret = apple_t6050_usb2_port_reset(&dp->usb_io, dp, combo_sleep); + if (ret) { + dev_err(dp->dev, "USB2 port reset failed (%d)\n", ret); + dp->faulted = true; + } + return ret ? -EIO : 0; +} + +/* A tunnelled PIPE leaves the CIO PHY to the router: nothing to quiesce on the PHY side. */ +static int combo_pipe_keep(void *ctx) +{ + return 0; +} + +/* + * In CIO mode the lanes carry USB4/TBT and USB3 reaches DWC3 through the router's USB3 adapter: + * move DWC3 from its dummy PIPE producer to the tunnel producer, as is done on the older PHYs. + */ +static int combo_pipe_tunnel(struct t6050_dp *dp) +{ + int ret; + + lockdep_assert_held(&dp->lock); + if (dp->pipe_active) + return dp->pipe_tunnel ? 0 : -EBUSY; + ret = apple_t6050_usb3_pipe_tunnel(&dp->usb_io); + if (ret) { + combo_pipe_fault(dp, "USB3 tunnel PIPE switch", ret); + return -EIO; + } + dp->pipe_active = true; + dp->pipe_tunnel = true; + dev_info(dp->dev, "USB3 PIPE on the USB4 tunnel\n"); + return 0; +} + +static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (mode != PHY_MODE_USB_HOST) + return -EOPNOTSUPP; + if (!dp->controller_active || !dp->usb2_active) + return -ENOLINK; + /* Four DP lanes leave DWC3 on its established dummy PIPE; USB2 still + * uses its independent D+/D- PHY. Do not switch USB3 onto DP lanes. */ + if (dp->combo_mode == COMBO_DP4 && dp->enabled) + return 0; + if (dp->combo_mode == COMBO_CIO && dp->cio_prepared) + return combo_pipe_tunnel(dp); + if (!dp->usb_prepared && !(dp->combo_mode == COMBO_DP2 && dp->enabled)) + return -ENOLINK; + if (dp->pipe_active) + return (readl(dp->banks[44] + 0xc) & 0x73) == 0x10 ? 0 : -EIO; + apple_tunable_apply(dp->banks[44], dp->pipe_default); + ret = apple_t6050_usb3_pipe_switch(&dp->usb_io); + if (ret >= -6 && ret < 0 && (readl(dp->banks[44] + 0xc) & 0x73) == 0x42) { + /* The PIPE never left its dummy producer: the port stays USB2, as with four DP lanes. */ + dev_warn(dp->dev, "USB3 PIPE switch failed (%d), the port runs USB2 only\n", ret); + return 0; + } + if (ret) + combo_pipe_fault(dp, "USB3 PIPE switch", ret); + else + dp->pipe_active = true; + return ret ? -EIO : 0; +} + +static int combo_usb3_poweroff(struct phy *phy) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret = 0; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + /* DWC3 calls power_off only after host/gadget shutdown and SUSPHY. + * PIPE must select its dummy producer before the controller is reset. */ + if (dp->pipe_active) { + ret = apple_t6050_usb3_pipe_stop(&dp->usb_io, + dp->pipe_tunnel ? combo_pipe_keep : combo_quiesce, dp); + if (!ret) + dp->pipe_tunnel = false; + if (!ret) + dp->pipe_active = false; + } + if (!ret && dp->usb2_active) { + ret = apple_t6050_usb2_stop(&dp->usb_platform); + if (!ret) + dp->usb2_active = false; + } + if (ret) { + dev_err(dp->dev, "USB3 power-off failed (%d)\n", ret); + dp->faulted = true; + } + return ret ? -EIO : 0; +} +static const struct phy_ops combo_usb2_ops = { + .set_mode = combo_usb2_mode, .reset = combo_usb2_reset, .owner = THIS_MODULE, +}; +static const struct phy_ops combo_usb3_ops = { + .set_mode = combo_usb3_mode, .power_off = combo_usb3_poweroff, .owner = THIS_MODULE, +}; + +static int combo_reset_assert(struct reset_controller_dev *reset, unsigned long id) +{ + struct t6050_dp *dp = container_of(reset, struct t6050_dp, reset); + int ret; + + guard(mutex)(&dp->lock); + if (id || dp->pipe_active) + return -EBUSY; + if (dp->faulted) + return -EIO; + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 1, 0); + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 0x10, 0x10); + ret = apple_t6050_usb3_pipe_cold_dummy(&dp->usb_io); + if (ret) + combo_pipe_fault(dp, "dummy PIPE park", ret); + else + dp->controller_active = false; + return ret ? -EIO : 0; +} +static int combo_reset_deassert(struct reset_controller_dev *reset, unsigned long id) +{ + struct t6050_dp *dp = container_of(reset, struct t6050_dp, reset); + + guard(mutex)(&dp->lock); + if (id) + return -EINVAL; + if (dp->faulted || !dp->usb2_active) + return -ENOLINK; + /* + * The PIPE lock that starts the USB3 handoff is acknowledged only while the dummy PHY + * behind the mux is enabled (NONSELECTED_OVERRIDE bit 15), as on the earlier ATC PHYs. + * The bit resets to clear with the controller's domain, so set it for every session. + */ + apple_t6050_usb_mask(&dp->usb_io, 44, 0x20, BIT(15), BIT(15)); + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 0x10, 0); + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 1, 1); + dp->controller_active = true; + return 0; +} +static const struct reset_control_ops combo_reset_ops = { + .assert = combo_reset_assert, .deassert = combo_reset_deassert, +}; + +static struct phy *combo_xlate(struct device *dev, const struct of_phandle_args *args) +{ + struct t6050_dp *dp = dev_get_drvdata(dev); + if (args->args_count != 1) + return ERR_PTR(-EINVAL); + switch (args->args[0]) { + case PHY_TYPE_USB2: return dp->combo_phy[0]; + case PHY_TYPE_USB3: return dp->combo_phy[1]; + case PHY_TYPE_DP: return dp->combo_phy[2]; + default: return ERR_PTR(-EINVAL); + } +} + +static int combo_setup(struct platform_device *pdev, struct t6050_dp *dp) +{ + /* Same order as the cio[] table in combo_tunable(). */ + static const struct { u8 bank; const char *name; } cio_tunables[] = { + {3, "apple,tunable-uc-regs-cio"}, {16, "apple,tunable-lane0-rx-top-cio"}, + {25, "apple,tunable-lane1-rx-top-cio"}, {18, "apple,tunable-lane0-rx-eq-cio"}, + {27, "apple,tunable-lane1-rx-eq-cio"}, + }; + static const struct { u8 bank; u32 minimum; const char *name; } extra[] = { + {0, 0x20, "usb2phy"}, {1, 0x24, "usb2-digital"}, {44, 0x48, "pipehandler"}, + }; + struct device *dev = dp->dev; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(extra); i++) { + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, extra[i].name); + if (!res || resource_size(res) < extra[i].minimum) + return -EINVAL; + dp->banks[extra[i].bank] = devm_ioremap_resource(dev, res); + if (IS_ERR(dp->banks[extra[i].bank])) + return PTR_ERR(dp->banks[extra[i].bank]); + dp->resources[extra[i].bank] = *res; + } + dp->usb2_default = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-usb2-default", &dp->resources[0]); + if (IS_ERR(dp->usb2_default)) + return PTR_ERR(dp->usb2_default); + dp->usb2_host = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-usb2-host", &dp->resources[0]); + if (IS_ERR(dp->usb2_host)) + return PTR_ERR(dp->usb2_host); + dp->pipe_default = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-pipe-default", &dp->resources[44]); + if (IS_ERR(dp->pipe_default)) + return PTR_ERR(dp->pipe_default); + for (unsigned int i = 0; i < ARRAY_SIZE(cio_tunables); i++) { + struct apple_tunable *t = devm_apple_tunable_parse(dev, dev->of_node, + cio_tunables[i].name, &dp->resources[cio_tunables[i].bank]); + + if (IS_ERR(t) && PTR_ERR(t) != -ENOENT) + return PTR_ERR(t); + dp->cio_tunables[i] = IS_ERR(t) ? NULL : t; + } + dp->usb_io = (struct apple_t6050_usb_io) { + .context = dp, .read = combo_read, .write = combo_write, + .poll = combo_poll, .delay = combo_delay, + }; + dp->usb_platform = (struct apple_t6050_usb_platform) { + .context = dp, .io = &dp->usb_io, .wrapper_power = combo_wrapper, + .firmware = combo_firmware, .tunable = combo_tunable, .pause_ns = combo_pause, + }; + dp->reset = (struct reset_controller_dev) { + .owner = THIS_MODULE, .ops = &combo_reset_ops, .dev = dev, + .of_node = dev->of_node, .nr_resets = 1, + }; + ret = devm_reset_controller_register(dev, &dp->reset); + if (ret) + return ret; + for (unsigned int i = 0; i < 2; i++) { + dp->combo_phy[i] = devm_phy_create(dev, NULL, i ? &combo_usb3_ops : &combo_usb2_ops); + if (IS_ERR(dp->combo_phy[i])) + return PTR_ERR(dp->combo_phy[i]); + phy_set_drvdata(dp->combo_phy[i], dp); + } + return 0; +} + diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c new file mode 100644 index 0000000000000..62d8ee9aae0f2 --- /dev/null +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -0,0 +1,1034 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* T6050 DisplayPort PHY for fixed bridges and Type-C DP alternate mode. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "usb-t6050/apple_t6050_usb_phy.h" +#include "usb-t6050/apple_t6050_usb3_pipe.h" +#include "usb-t6050/apple_t6050_usb2_reset.h" +#include "usb-t6050/apple_t6050_cio_phy.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum atc_operation { ATC_MASK, ATC_TUNABLE, ATC_DELAY, ATC_POLL, ATC_POWER, ATC_FIRMWARE }; +struct atc_op { + u8 op, bank; + u16 offset; + u32 mask, value, timeout; +}; +struct atc_rate_op { + struct atc_op op; + u8 rates; + u32 values[4]; +}; +#include "atc-t6050-dp-data.h" +#include "atc-t6050-typec-data.h" + +static const struct { u8 bank; u32 offset; } core_banks[] = { + {2, 0}, {3, 0x400}, {4, 0x800}, {5, 0xa00}, {6, 0x1000}, + {7, 0x3000}, {8, 0x3200}, {9, 0x3800}, {10, 0x3a00}, + {14, 0x7000}, {16, 0x9000}, {18, 0xb000}, {19, 0xc000}, + {20, 0xd000}, {21, 0xe000}, {22, 0xf000}, {25, 0x19000}, + {27, 0x1b000}, {28, 0x1c000}, {29, 0x1d000}, {30, 0x1e000}, + {31, 0x1f000}, {33, 0x28000}, {34, 0x28400}, {35, 0x29000}, +}; +static const struct { u8 bank; const char *name; } extra_banks[] = { + {39, "reset"}, {40, "fabric"}, {41, "axi2af"}, + {42, "lioa"}, {43, "common-cfg"}, + {38, "usb-control"}, +}; +static const struct { u8 bank; const char *name; } tunable_names[] = { + {41, "apple,tunable-axi2af"}, {40, "apple,tunable-fabric"}, + {43, "apple,tunable-common-cfg"}, {42, "apple,tunable-lioa"}, + {20, "apple,tunable-lane0-rx-clock"}, {29, "apple,tunable-lane1-rx-clock"}, + {18, "apple,tunable-lane0-rx-eq"}, {27, "apple,tunable-lane1-rx-eq"}, + {8, "apple,tunable-auspll"}, + {5, "apple,tunable-aus40cmn-shm"}, {10, "apple,tunable-cio4pll"}, +}; +static const unsigned int link_rates[] = {1620, 2700, 5400, 8100}; +static const char * const training_names[] = { + "apple,dp-training-table-rbr", "apple,dp-training-table-hbr", + "apple,dp-training-table-hbr2", "apple,dp-training-table-hbr3", +}; +struct atc_domain { + struct device *dev; + bool active; +}; +enum combo_mode { COMBO_SAFE, COMBO_USB, COMBO_DP4, COMBO_DP2, COMBO_CIO }; + +/* + * USB4 DisplayPort tunnel pixel clocks, k = 1..4. Each has a clock gate, a + * source select word in the power manager's clock block (the DP link rate's + * source, then a busy bit to wait out), and a switch in the PHY's CFG block + * (bank 39): +0x14 holds a 3-bit field per clock (k = on, 7 = parked) and a + * UHBR bit 11 + k, +0x18 an enable bit per clock, +0x1c its status. Switch k + * feeds the display crossbar's pixel clock k. + */ +#define ATC_DP_TUNNELS 4 +#define ATC_CFG_DP_SWITCH 0x14 +#define ATC_CFG_DP_ENABLE 0x18 +#define ATC_CFG_DP_STATUS 0x1c +#define ATC_DP_SWITCH_PARKED 7 +#define ATC_DP_SRC_SELECT GENMASK(27, 24) +#define ATC_DP_SRC_BUSY BIT(30) +struct t6050_dp { + bool combo, usb_prepared, usb2_active, pipe_active, controller_active, cio_prepared; + bool pipe_tunnel; /* the active PIPE is a USB4 router's USB3 tunnel */ + enum combo_mode combo_mode; + enum typec_orientation orientation; + struct phy *combo_phy[3]; + struct reset_controller_dev reset; + struct apple_t6050_usb_io usb_io; + struct apple_t6050_usb_platform usb_platform; + struct apple_tunable *usb2_default, *usb2_host, *pipe_default; + struct apple_tunable *cio_tunables[5]; /* optional; empty on J716c except RX_TOP */ + struct device *dev; + struct mutex lock; + void __iomem *banks[45]; + struct resource resources[45]; + struct apple_tunable *tunables[ARRAY_SIZE(tunable_names)]; + /* USB AON, common, wrapper: native order; a combo may add USB. */ + struct atc_domain domain[4]; + unsigned int ndomains; + u32 training[4][16]; + u32 voltage[4], pre[4]; + int rate; + bool enabled, faulted; + bool typec, reverse, multifunction; + struct apple_rtkit *rtkit; + void __iomem *fw_data; + bool fw_running; + void *fw_data_saved; /* the firmware data SRAM as the first start found it */ + /* USB4 DP tunnel pixel clocks: source words, gates, running link rates */ + void __iomem *dp_clk_src; + struct atc_domain dp_gate[ATC_DP_TUNNELS]; + bool dp_gates; + u8 tunnel_rate[ATC_DP_TUNNELS]; +}; + +static int atc_domain_set(struct atc_domain *pd, bool active) +{ + int ret; + + if (pd->active == active) + return 0; + if (active) { + ret = pm_runtime_resume_and_get(pd->dev); + if (ret < 0) + return ret; + } else { + ret = pm_runtime_put_sync_suspend(pd->dev); + if (ret < 0) { + /* put decremented usage even when suspend failed. */ + pm_runtime_get_noresume(pd->dev); + return ret; + } + } + pd->active = active; + return 0; +} + +static void atc_domain_release(void *data) +{ + struct atc_domain *pd = data; + + if (pd->active) + pm_runtime_put_sync_suspend(pd->dev); + dev_pm_domain_detach(pd->dev, true); +} + +/* The bootloader preloads CIO80 firmware into local SRAM. The firmware's + * data segment is 0x10000000..0x10010000, not guest system RAM. */ +static int atc_fw_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct t6050_dp *dp = cookie; + + if (bfr->iova < 0x10000000 || + bfr->size > 0x10000 || bfr->iova - 0x10000000 > 0x10000 - bfr->size) { + dev_err(dp->dev, "unsupported PHY firmware buffer %llx size %zx\n", + (unsigned long long)bfr->iova, bfr->size); + return -EINVAL; + } + bfr->iomem = dp->fw_data + (bfr->iova - 0x10000000); + bfr->is_mapped = true; + return 0; +} + +static void atc_fw_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + /* SRAM is owned by the PHY and mapped for the device lifetime. */ +} + +static void atc_fw_message(void *cookie, u8 endpoint, u64 message) +{ + struct t6050_dp *dp = cookie; + + dev_dbg(dp->dev, "PHY firmware ep%x: %016llx\n", endpoint, message); +} + +static void atc_fw_crashed(void *cookie, const void *log, size_t size) +{ + struct t6050_dp *dp = cookie; + + WRITE_ONCE(dp->faulted, true); + dev_err(dp->dev, "PHY firmware crashed\n"); +} + +static const struct apple_rtkit_ops atc_fw_ops = { + .shmem_setup = atc_fw_shmem_setup, + .shmem_destroy = atc_fw_shmem_destroy, + .recv_message = atc_fw_message, + .crashed = atc_fw_crashed, +}; + +static int atc_firmware(struct t6050_dp *dp, bool start) +{ + int ret; + + if (!dp->rtkit) + return -ENODEV; + if (dp->fw_running == start) + return 0; + if (start) { + /* + * The firmware is the boot loader's, preloaded with its data segment, and keeps + * its state there: restarted on the state its last run left, it never finishes + * calibrating the lanes (UC_TNBL_DONE). Give every start the data segment the + * first one found. The PHY's domains are on here, so the SRAM is accessible. + */ + if (!dp->fw_data_saved) { + dp->fw_data_saved = devm_kmalloc(dp->dev, 0x10000, GFP_KERNEL); + if (!dp->fw_data_saved) + return -ENOMEM; + memcpy_fromio(dp->fw_data_saved, dp->fw_data, 0x10000); + } else { + memcpy_toio(dp->fw_data, dp->fw_data_saved, 0x10000); + } + /* Native coprocessor run, after its +0x40 firmware enable. */ + writel(BIT(4), dp->banks[38] + 0x28); + ret = apple_rtkit_wake(dp->rtkit); + if (!ret) + ret = apple_rtkit_start_ep(dp->rtkit, 0x20); + } else { + /* CIO80PhyMBI::sendPowerDown, then the RTBuddy power transition. */ + ret = apple_rtkit_send_message(dp->rtkit, 0x20, BIT_ULL(56), NULL, false); + if (!ret) + ret = apple_rtkit_shutdown(dp->rtkit); + if (!ret) + writel((u32)~BIT(4), dp->banks[38] + 0x28); + } + if (ret) + dev_err(dp->dev, "PHY firmware %s failed: %d\n", start ? "start" : "stop", ret); + else + dp->fw_running = start; + return ret; +} + +static int atc_validate_step(struct t6050_dp *dp, const struct atc_op *s) +{ + if (s->op == ATC_DELAY) + return s->value <= 10000 ? 0 : -EINVAL; + if (s->op == ATC_POWER) + return s->value <= 1 ? 0 : -EINVAL; + if (s->op == ATC_FIRMWARE) + return dp->typec && s->value <= 1 ? 0 : -EINVAL; + if (s->bank >= ARRAY_SIZE(dp->banks) || !dp->banks[s->bank] || + s->offset % 4 || s->offset + 4 > resource_size(&dp->resources[s->bank])) + return -EINVAL; + if (s->op == ATC_TUNABLE) + return s->value < ARRAY_SIZE(tunable_names) && + tunable_names[s->value].bank == s->bank ? 0 : -EINVAL; + if (s->op == ATC_MASK || s->op == ATC_POLL) + return s->value & ~s->mask ? -EINVAL : 0; + return -EINVAL; +} + +static int atc_step(struct t6050_dp *dp, const struct atc_op *s) +{ + void __iomem *reg; + u32 value; + int ret = atc_validate_step(dp, s); + + if (ret) + return ret; + if (s->op == ATC_DELAY) { + udelay(s->value); + return 0; + } + if (s->op == ATC_FIRMWARE) + return atc_firmware(dp, s->value); + if (s->op == ATC_POWER) + return atc_domain_set(&dp->domain[2], s->value); + if (s->bank >= ARRAY_SIZE(dp->banks) || !dp->banks[s->bank] || + s->offset + 4 > resource_size(&dp->resources[s->bank])) + return -EINVAL; + reg = dp->banks[s->bank] + s->offset; + switch (s->op) { + case ATC_MASK: + value = readl(reg); + writel((value & ~s->mask) | s->value, reg); + return 0; + case ATC_TUNABLE: + if (s->value >= ARRAY_SIZE(dp->tunables)) + return -EINVAL; + apple_tunable_apply(dp->banks[s->bank], dp->tunables[s->value]); + return 0; + case ATC_POLL: + ret = readl_poll_timeout(reg, value, (value & s->mask) == s->value, + s->timeout > 1000 ? 1000 : 1, s->timeout); + if (ret) + dev_err(dp->dev, "bank%u+%x timeout: %x & %x != %x\n", + s->bank, s->offset, value, s->mask, s->value); + return ret; + default: + return -EINVAL; + } +} + +static int atc_sequence(struct t6050_dp *dp, const struct atc_op *s, size_t count) +{ + for (size_t i = 0; i < count; i++) { + int ret = atc_step(dp, &s[i]); + + if (ret) { + dp->faulted = true; + return ret; + } + } + return 0; +} +#define ATC_SEQUENCE(dp, seq) atc_sequence(dp, seq, ARRAY_SIZE(seq)) + +static int atc_link_rate(struct t6050_dp *dp, int rate) +{ + const struct atc_rate_op *on = dp_link_on; + size_t nr_on = ARRAY_SIZE(dp_link_on); + int ret; + + if (dp->multifunction) { + on = dp->reverse ? dp_group0_on : dp_group1_on; + nr_on = dp->reverse ? ARRAY_SIZE(dp_group0_on) : ARRAY_SIZE(dp_group1_on); + } + if (dp->rate == rate) + return 0; + if (dp->rate >= 0) { + if (dp->multifunction) + ret = dp->reverse ? ATC_SEQUENCE(dp, dp_group0_off) : + ATC_SEQUENCE(dp, dp_group1_off); + else + ret = ATC_SEQUENCE(dp, dp_link_off); + if (ret) + return ret; + dp->rate = -1; + } + if (rate < 0) + return 0; + for (size_t i = 0; i < nr_on; i++) { + struct atc_op op = on[i].op; + + if (!(on[i].rates & BIT(rate))) + continue; + op.value = on[i].values[rate]; + ret = atc_step(dp, &op); + if (ret) { + dp->faulted = true; + return ret; + } + } + dp->rate = rate; + memset(dp->voltage, 0, sizeof(dp->voltage)); + memset(dp->pre, 0, sizeof(dp->pre)); + return 0; +} + +static int atc_start(struct t6050_dp *dp) +{ + int ret; + + if (dp->faulted) + return -EIO; + if (dp->enabled) + return 0; + ret = atc_domain_set(&dp->domain[0], true); + if (ret) + return ret; + ret = atc_domain_set(&dp->domain[1], true); + if (ret) { + atc_domain_set(&dp->domain[0], false); + return ret; + } + if (dp->multifunction) + ret = dp->reverse ? ATC_SEQUENCE(dp, mf_reverse_init) : + ATC_SEQUENCE(dp, mf_normal_init); + else + ret = dp->reverse ? ATC_SEQUENCE(dp, dp_reverse_init) : + ATC_SEQUENCE(dp, dp_init); + if (!ret) + ret = ATC_SEQUENCE(dp, aux_on); + if (ret) + return ret; + dp->enabled = true; + return 0; +} + +static int atc_stop(struct t6050_dp *dp) +{ + int ret; + + if (dp->faulted) + return -EIO; + if (!dp->enabled) + return 0; + ret = atc_link_rate(dp, -1); + if (!ret) + ret = ATC_SEQUENCE(dp, aux_off); + if (!ret) + ret = dp->multifunction ? ATC_SEQUENCE(dp, mf_poweroff) : + ATC_SEQUENCE(dp, dp_poweroff); + if (ret) + return ret; + dp->enabled = false; + /* Combo owns domain shutdown after USB2 and PIPE have also quiesced. */ + if (dp->combo) + return 0; + ret = atc_domain_set(&dp->domain[1], false); + if (!ret) + ret = atc_domain_set(&dp->domain[0], false); + return ret; +} + +static void atc_mask(void __iomem *reg, u32 mask, u32 value) +{ + writel((readl(reg) & ~mask) | value, reg); +} + +static void atc_drive(struct t6050_dp *dp, unsigned int lane, u32 preset) +{ + /* Native DP lane order is RX/TX in each orientation-selected group. */ + unsigned int group = (lane < 2 ? 1 : 0) ^ dp->reverse; + bool tx = lane & 1; + void __iomem *base = dp->banks[(tx ? 21 : 20) + 9 * group]; + u32 ctrl = ((preset >> 2) & 0xf) | ((preset & 3) << 14); + u32 eq = ((preset >> 5) & 0x7fffe) | 1; + + if (tx) + atc_mask(base + 0x100, BIT(20), BIT(20)); + atc_mask(base + (tx ? 0x58 : 0x68), 0xc00f, ctrl); + atc_mask(base + (tx ? 0x104 : 0xac), BIT(29), BIT(29)); + atc_mask(base + (tx ? 0x100 : 0xa8), 0x7ffff, eq); +} + +static u32 atc_dp_switch_mask(unsigned int k) +{ + return GENMASK(3 * k - 1, 3 * k - 3) | BIT(11 + k); +} + +/* Enable (and status) bits 1, 8, 15 and 22 */ +static u32 atc_dp_enable_bit(unsigned int k) +{ + return BIT(7 * k - 6); +} + +/* The power manager's source code for a DP link rate */ +static int atc_dp_source(u8 rate) +{ + switch (rate) { + case 0x06: return 10; /* RBR */ + case 0x0a: return 9; /* HBR */ + case 0x14: return 8; /* HBR2 */ + case 0x1e: return 7; /* HBR3 */ + case 0x81: return 6; /* UHBR10 */ + case 0x82: return 5; /* UHBR20 */ + default: return -EINVAL; + } +} + +/* Switch tunnel clock @k off and park it; its gate stays as it is. */ +static void atc_dp_tunnel_switch_off(struct t6050_dp *dp, unsigned int k) +{ + void __iomem *cfg = dp->banks[39]; + u32 en = atc_dp_enable_bit(k), v; + + writel(readl(cfg + ATC_CFG_DP_ENABLE) & ~en, cfg + ATC_CFG_DP_ENABLE); + writel((readl(cfg + ATC_CFG_DP_SWITCH) & ~atc_dp_switch_mask(k)) | + (ATC_DP_SWITCH_PARKED << (3 * k - 3)), cfg + ATC_CFG_DP_SWITCH); + if (readl_poll_timeout(cfg + ATC_CFG_DP_STATUS, v, !(v & en), 10, 10000)) + dev_warn(dp->dev, "DP tunnel clock %u: switch status %08x still on\n", k, v); + dp->tunnel_rate[k - 1] = 0; +} + +/* + * Park every tunnel clock switch that runs no clock (field 7, enable off), + * once the PHY is in USB4 mode: an unparked switch selects the native DP + * clock for its crossbar pixel clock. + */ +static void atc_dp_tunnels_park(struct t6050_dp *dp) +{ + void __iomem *cfg = dp->banks[39]; + u32 sw = readl(cfg + ATC_CFG_DP_SWITCH), en = readl(cfg + ATC_CFG_DP_ENABLE); + + for (unsigned int k = 1; k <= ATC_DP_TUNNELS; k++) { + if (dp->tunnel_rate[k - 1]) + continue; + sw = (sw & ~atc_dp_switch_mask(k)) | (ATC_DP_SWITCH_PARKED << (3 * k - 3)); + en &= ~atc_dp_enable_bit(k); + } + writel(en, cfg + ATC_CFG_DP_ENABLE); + writel(sw, cfg + ATC_CFG_DP_SWITCH); +} + +/* Stop every tunnel clock, before the PHY leaves USB4 mode. */ +static void atc_dp_tunnels_stop(struct t6050_dp *dp) +{ + for (unsigned int k = 1; k <= ATC_DP_TUNNELS; k++) { + if (dp->tunnel_rate[k - 1]) + atc_dp_tunnel_switch_off(dp, k); + /* A failed start or gate suspend can leave a held domain at rate zero. */ + if (dp->dp_gates) + atc_domain_set(&dp->dp_gate[k - 1], false); + } +} + +#include "atc-t6050-combo.h" + +static int t6050_dp_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + + guard(mutex)(&dp->lock); + if (mode == PHY_MODE_DP) + return dp->typec ? (dp->enabled ? 0 : -ENOLINK) : atc_start(dp); + if (mode == PHY_MODE_INVALID) + return dp->combo ? atc_link_rate(dp, -1) : atc_stop(dp); + return -EINVAL; +} + +static int t6050_dp_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + + guard(mutex)(&dp->lock); + if (mode != PHY_MODE_DP) + return -EINVAL; + memset(opts, 0, sizeof(*opts)); + opts->dp.lanes = dp->multifunction ? 2 : 4; + opts->dp.link_rate = 8100; + opts->dp.ssc = 0; + return 0; +} + +static int t6050_dp_configure(struct phy *phy, union phy_configure_opts *opts) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + struct phy_configure_opts_dp *cfg = &opts->dp; + int rate = -1, ret; + + /* Validate the whole request before altering a running transmitter. */ + if (cfg->set_rate && cfg->link_rate) { + for (unsigned int i = 0; i < ARRAY_SIZE(link_rates); i++) + if (cfg->link_rate == link_rates[i]) + rate = i; + if (rate < 0 || cfg->ssc) + return -EINVAL; + } + if ((cfg->set_lanes || cfg->set_voltages) && + cfg->lanes != 0 && cfg->lanes != 1 && cfg->lanes != 2 && cfg->lanes != 4) + return -EINVAL; + if (cfg->set_voltages) + for (unsigned int i = 0; i < cfg->lanes; i++) + if (cfg->voltage[i] > 3 || cfg->pre[i] > 3 || + cfg->voltage[i] + cfg->pre[i] > 3) + return -EINVAL; + if (cfg->set_voltages && cfg->set_rate && rate < 0 && cfg->lanes) + return -EINVAL; + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (!dp->enabled) + return -EPIPE; + if ((cfg->set_lanes || cfg->set_voltages) && dp->multifunction && cfg->lanes > 2) + return -EINVAL; + if (cfg->set_rate) { + ret = atc_link_rate(dp, rate); + if (ret) + return ret; + } + /* Native setActiveLaneCount is a no-op; DCP controls the active count. */ + if (cfg->set_voltages) { + if (dp->rate < 0) + return -EPIPE; + for (unsigned int i = 0; i < cfg->lanes; i++) { + u32 preset = dp->training[dp->rate][4 * cfg->voltage[i] + cfg->pre[i]]; + + atc_drive(dp, i, preset); + dp->voltage[i] = cfg->voltage[i]; + dp->pre[i] = cfg->pre[i]; + } + } + return 0; +} + +static const struct phy_ops t6050_dp_ops = { + .set_mode = t6050_dp_set_mode, + .validate = t6050_dp_validate, + .configure = t6050_dp_configure, + .owner = THIS_MODULE, +}; + +int apple_dp_phy_link_config(struct phy *phy, unsigned int *pclk, bool *uhbr) +{ + struct t6050_dp *dp; + + if (!phy || phy->ops != &t6050_dp_ops) + return -EOPNOTSUPP; + dp = phy_get_drvdata(phy); + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (!dp->enabled || dp->rate < 0) + return -ENOLINK; + /* Native direct DPPHY uses logical clock0, mapped to xbar PCLK1. + * The other four logical clocks are separately managed tunnel clocks. */ + *pclk = 1; + *uhbr = false; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dp_phy_link_config); + +int apple_t6050_dp_tunnel_clock(struct phy *phy, unsigned int clock, u8 rate) +{ + struct t6050_dp *dp; + void __iomem *cfg, *word; + unsigned int i = clock - 1; + int src = 0, ret; + u32 en, v; + + if (!phy || phy->ops != &t6050_dp_ops) + return -EOPNOTSUPP; + if (clock < 1 || clock > ATC_DP_TUNNELS) + return -EINVAL; + if (rate) { + src = atc_dp_source(rate); + if (src < 0) + return src; + } + dp = phy_get_drvdata(phy); + guard(mutex)(&dp->lock); + if (!dp->combo || !dp->dp_clk_src) + return -EOPNOTSUPP; + if (rate && dp->tunnel_rate[i] == rate) + return 0; + /* A new rate takes the clock off its old source first. */ + if (dp->tunnel_rate[i]) + atc_dp_tunnel_switch_off(dp, clock); + if (!rate) { + ret = dp->dp_gates ? atc_domain_set(&dp->dp_gate[i], false) : 0; + if (!ret) + dev_info(dp->dev, "DP tunnel clock %u stopped\n", clock); + return ret; + } + if (dp->faulted) + return -EIO; + if (dp->combo_mode != COMBO_CIO) + return -ENOLINK; + if (dp->dp_gates) { + ret = atc_domain_set(&dp->dp_gate[i], true); + if (ret) + return ret; + } + word = dp->dp_clk_src + 4 * i; + writel((readl(word) & ~ATC_DP_SRC_SELECT) | FIELD_PREP(ATC_DP_SRC_SELECT, src), word); + ret = readl_poll_timeout(word, v, !(v & ATC_DP_SRC_BUSY), 10, 200000); + if (ret) { + dev_err(dp->dev, "DP tunnel clock %u: source %d not taken (%08x)\n", + clock, src, v); + if (dp->dp_gates) + atc_domain_set(&dp->dp_gate[i], false); + return ret; + } + cfg = dp->banks[39]; + en = atc_dp_enable_bit(clock); + writel((readl(cfg + ATC_CFG_DP_SWITCH) & ~atc_dp_switch_mask(clock)) | + (clock << (3 * clock - 3)) | + (rate == 0x81 || rate == 0x82 ? BIT(11 + clock) : 0), + cfg + ATC_CFG_DP_SWITCH); + writel(readl(cfg + ATC_CFG_DP_ENABLE) | en, cfg + ATC_CFG_DP_ENABLE); + ret = readl_poll_timeout(cfg + ATC_CFG_DP_STATUS, v, v & en, 10, 10000); + if (ret) { + dev_err(dp->dev, "DP tunnel clock %u: switch status %08x not on\n", clock, v); + /* Undo the partial start before leaving the rate uncached for retry. */ + atc_dp_tunnel_switch_off(dp, clock); + if (dp->dp_gates) + atc_domain_set(&dp->dp_gate[i], false); + return ret; + } + dp->tunnel_rate[i] = rate; + dev_info(dp->dev, "DP tunnel clock %u: link rate 0x%02x, source %d, switch status %08x\n", + clock, rate, src, v); + return 0; +} +EXPORT_SYMBOL_GPL(apple_t6050_dp_tunnel_clock); + +/* The connector sets orientation before changing the alternate-mode mux. */ +static int t6050_set_orientation(struct t6050_dp *dp, + enum typec_orientation orientation) +{ + bool reverse = orientation == TYPEC_ORIENTATION_REVERSE; + int ret; + + if (dp->combo) + return combo_orientation(dp, orientation); + guard(mutex)(&dp->lock); + if (orientation == TYPEC_ORIENTATION_NONE) + return atc_stop(dp); + if (dp->reverse == reverse) + return 0; + ret = atc_stop(dp); + if (!ret) + dp->reverse = reverse; + return ret; +} + +static int t6050_set_typec_mode(struct t6050_dp *dp, struct typec_mux_state *state) +{ + bool multifunction; + int ret; + + if (dp->combo) + return combo_mux(dp, state); + guard(mutex)(&dp->lock); + if (state->mode == TYPEC_STATE_SAFE) + return atc_stop(dp); + /* USB-only and USB4 require their controller's separate PHY interface. */ + if (!state->alt || state->alt->svid != USB_TYPEC_DP_SID) + return -EOPNOTSUPP; + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + multifunction = false; + break; + case TYPEC_DP_STATE_D: + multifunction = true; + break; + default: + return -EOPNOTSUPP; + } + if (dp->enabled && dp->multifunction == multifunction) + return 0; + ret = atc_stop(dp); + if (ret) + return ret; + dp->multifunction = multifunction; + ret = atc_start(dp); + if (!ret) + dev_info(dp->dev, "Type-C DP: %u lanes, %s orientation\n", + multifunction ? 2 : 4, dp->reverse ? "reverse" : "normal"); + return ret; +} + +static int t6050_typec_orientation(struct typec_switch_dev *sw, + enum typec_orientation orientation) +{ + return t6050_set_orientation(typec_switch_get_drvdata(sw), orientation); +} + +static int t6050_typec_mode(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + return t6050_set_typec_mode(typec_mux_get_drvdata(mux), state); +} + +static void t6050_typec_switch_release(void *data) +{ + typec_switch_unregister(data); +} + +static void t6050_typec_mux_release(void *data) +{ + typec_mux_unregister(data); +} + +static int t6050_typec_register(struct t6050_dp *dp) +{ + struct typec_switch_desc sw_desc = { + .fwnode = dev_fwnode(dp->dev), .drvdata = dp, + .set = t6050_typec_orientation, + }; + struct typec_mux_desc mux_desc = { + .fwnode = dev_fwnode(dp->dev), .drvdata = dp, + .set = t6050_typec_mode, + }; + struct typec_switch_dev *sw; + struct typec_mux_dev *mux; + int ret; + + sw = typec_switch_register(dp->dev, &sw_desc); + if (IS_ERR(sw)) + return PTR_ERR(sw); + ret = devm_add_action_or_reset(dp->dev, t6050_typec_switch_release, sw); + if (ret) + return ret; + mux = typec_mux_register(dp->dev, &mux_desc); + if (IS_ERR(mux)) + return PTR_ERR(mux); + return devm_add_action_or_reset(dp->dev, t6050_typec_mux_release, mux); +} + +static int t6050_dp_probe(struct platform_device *pdev) +{ + static const char * const domains[] = {"usb-aon", "common", "wrapper", "usb"}; + struct device *dev = &pdev->dev; + struct t6050_dp *dp; + struct phy_provider *provider; + struct phy *phy; + struct resource *core; + void __iomem *base; + int ndomains, gates, ret; + + dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL); + if (!dp) + return -ENOMEM; + dp->dev = dev; + dp->combo = of_device_is_compatible(dev->of_node, "apple,t6050-combo-phy"); + if (dp->combo) { + u32 port; + /* Only ports explicitly handed over by the boot producer. */ + if (of_property_read_u32(dev->of_node, "apple,port-index", &port) || + port > 5 || + !of_property_present(dev->of_node, "apple,guest-port-reserved") || + !of_property_present(dev->of_node, "mode-switch")) + return -EINVAL; + } + dp->typec = of_property_present(dev->of_node, "mode-switch"); + dp->rate = -1; + mutex_init(&dp->lock); + platform_set_drvdata(pdev, dp); + core = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core"); + if (!core || resource_size(core) < (dp->combo ? 0x2c000 : 0x2c400)) + return -EINVAL; + base = devm_ioremap_resource(dev, core); + if (IS_ERR(base)) + return PTR_ERR(base); + for (unsigned int i = 0; i < ARRAY_SIZE(core_banks); i++) { + u8 bank = core_banks[i].bank; + + dp->banks[bank] = base + core_banks[i].offset; + dp->resources[bank] = *core; + dp->resources[bank].start += core_banks[i].offset; + dp->resources[bank].end = core->end; + } + for (unsigned int i = 0; i < ARRAY_SIZE(extra_banks); i++) { + u8 bank = extra_banks[i].bank; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + extra_banks[i].name); + + if (bank == 38 && !dp->typec) + continue; + if (!res || resource_size(res) < 4) + return -EINVAL; + dp->banks[bank] = devm_ioremap_resource(dev, res); + if (IS_ERR(dp->banks[bank])) + return PTR_ERR(dp->banks[bank]); + dp->resources[bank] = *res; + } + /* Check every register step before any domain is enabled. */ + { + static const struct { const struct atc_op *ops; size_t count; } sequences[] = { + { dp_init, ARRAY_SIZE(dp_init) }, + { aux_on, ARRAY_SIZE(aux_on) }, + { aux_off, ARRAY_SIZE(aux_off) }, + { dp_link_off, ARRAY_SIZE(dp_link_off) }, + { dp_poweroff, ARRAY_SIZE(dp_poweroff) }, + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(sequences); i++) + for (size_t j = 0; j < sequences[i].count; j++) { + ret = atc_validate_step(dp, &sequences[i].ops[j]); + if (ret) + return ret; + } + for (unsigned int i = 0; i < ARRAY_SIZE(dp_link_on); i++) + for (unsigned int rate = 0; rate < ARRAY_SIZE(link_rates); rate++) { + struct atc_op op = dp_link_on[i].op; + + if (!(dp_link_on[i].rates & BIT(rate))) + continue; + op.value = dp_link_on[i].values[rate]; + ret = atc_validate_step(dp, &op); + if (ret) + return ret; + } + } + if (dp->typec) { + static const struct { const struct atc_op *ops; size_t count; } sequences[] = { + { dp_reverse_init, ARRAY_SIZE(dp_reverse_init) }, + { mf_normal_init, ARRAY_SIZE(mf_normal_init) }, + { mf_reverse_init, ARRAY_SIZE(mf_reverse_init) }, + { mf_poweroff, ARRAY_SIZE(mf_poweroff) }, + { dp_group0_off, ARRAY_SIZE(dp_group0_off) }, + { dp_group1_off, ARRAY_SIZE(dp_group1_off) }, + }; + const struct atc_rate_op *rates[] = { dp_group0_on, dp_group1_on }; + const size_t counts[] = { ARRAY_SIZE(dp_group0_on), ARRAY_SIZE(dp_group1_on) }; + + for (unsigned int i = 0; i < ARRAY_SIZE(sequences); i++) + for (size_t j = 0; j < sequences[i].count; j++) { + ret = atc_validate_step(dp, &sequences[i].ops[j]); + if (ret) + return ret; + } + for (unsigned int group = 0; group < ARRAY_SIZE(rates); group++) + for (size_t i = 0; i < counts[group]; i++) + for (unsigned int rate = 0; rate < ARRAY_SIZE(link_rates); rate++) { + struct atc_op op = rates[group][i].op; + if (!(rates[group][i].rates & BIT(rate))) + continue; + op.value = rates[group][i].values[rate]; + ret = atc_validate_step(dp, &op); + if (ret) + return ret; + } + } + for (unsigned int i = 0; i < ARRAY_SIZE(tunable_names); i++) { + if (i >= 9 && !dp->typec) + continue; + dp->tunables[i] = devm_apple_tunable_parse(dev, dev->of_node, + tunable_names[i].name, &dp->resources[tunable_names[i].bank]); + if (IS_ERR(dp->tunables[i])) + return dev_err_probe(dev, PTR_ERR(dp->tunables[i]), + "missing or invalid %s\n", tunable_names[i].name); + } + for (unsigned int i = 0; i < ARRAY_SIZE(training_names); i++) { + const char *name = training_names[i]; + + if (!of_property_present(dev->of_node, name)) + name = "apple,dp-training-table"; + if (!of_property_present(dev->of_node, name)) { + memcpy(dp->training[i], dp_training_default, sizeof(dp_training_default)); + continue; + } + if (of_property_count_u32_elems(dev->of_node, name) != 16) + return -EINVAL; + ret = of_property_read_u32_array(dev->of_node, name, dp->training[i], 16); + if (ret) + return ret; + } + /* + * Named domains prevent automatic wrapper enable at probe time. A combo + * PHY may also name the USB controller's domain, which its USB2 banks + * are in (combo_start()). + */ + ndomains = of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells"); + /* A combo's tunnel clock gates, "dp0".."dp3", follow its own domains. */ + gates = dp->combo ? + of_property_match_string(dev->of_node, "power-domain-names", "dp0") : -EINVAL; + if (gates > 0) { + if (ndomains != gates + ATC_DP_TUNNELS) + return -EINVAL; + ndomains = gates; + } + if (ndomains != 3 && !(dp->combo && ndomains == 4)) + return -EINVAL; + dp->ndomains = ndomains; + for (unsigned int i = 0; i < dp->ndomains; i++) { + dp->domain[i].dev = dev_pm_domain_attach_by_name(dev, domains[i]); + if (IS_ERR_OR_NULL(dp->domain[i].dev)) + return dev_err_probe(dev, dp->domain[i].dev ? PTR_ERR(dp->domain[i].dev) : -ENODEV, + "cannot attach %s domain\n", domains[i]); + ret = devm_add_action_or_reset(dev, atc_domain_release, &dp->domain[i]); + if (ret) + return ret; + } + for (unsigned int i = 0; gates > 0 && i < ATC_DP_TUNNELS; i++) { + char name[4]; + + snprintf(name, sizeof(name), "dp%u", i); + dp->dp_gate[i].dev = dev_pm_domain_attach_by_name(dev, name); + if (IS_ERR_OR_NULL(dp->dp_gate[i].dev)) { + ret = dp->dp_gate[i].dev ? PTR_ERR(dp->dp_gate[i].dev) : -ENODEV; + return dev_err_probe(dev, ret, "cannot attach %s domain\n", name); + } + ret = devm_add_action_or_reset(dev, atc_domain_release, &dp->dp_gate[i]); + if (ret) + return ret; + dp->dp_gates = true; + } + if (dp->combo) { + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "dp-clock-source"); + + if (res) { + if (resource_size(res) < 4 * ATC_DP_TUNNELS) + return -EINVAL; + dp->dp_clk_src = devm_ioremap_resource(dev, res); + if (IS_ERR(dp->dp_clk_src)) + return PTR_ERR(dp->dp_clk_src); + } + } + if (dp->typec) { + dp->fw_data = devm_platform_ioremap_resource_byname(pdev, "firmware-data"); + if (IS_ERR(dp->fw_data)) + return PTR_ERR(dp->fw_data); + dp->rtkit = devm_apple_rtkit_init(dev, dp, NULL, 0, &atc_fw_ops); + if (IS_ERR(dp->rtkit)) + return dev_err_probe(dev, PTR_ERR(dp->rtkit), "cannot initialize PHY firmware IPC\n"); + } + if (dp->combo) { + ret = combo_setup(pdev, dp); + if (ret) + return ret; + } + phy = devm_phy_create(dev, NULL, &t6050_dp_ops); + if (IS_ERR(phy)) + return PTR_ERR(phy); + phy_set_drvdata(phy, dp); + dp->combo_phy[2] = phy; + provider = devm_of_phy_provider_register(dev, dp->combo ? combo_xlate : of_phy_simple_xlate); + if (IS_ERR(provider)) + return PTR_ERR(provider); + return dp->typec ? t6050_typec_register(dp) : 0; +} + +static void t6050_dp_shutdown(struct platform_device *pdev) +{ + struct t6050_dp *dp = platform_get_drvdata(pdev); + + guard(mutex)(&dp->lock); + if (dp->combo ? combo_stop(dp) : atc_stop(dp)) + dev_warn(dp->dev, "PHY shutdown incomplete after an earlier error\n"); +} + +static const struct of_device_id t6050_dp_match[] = { + { .compatible = "apple,t6050-dp-phy" }, + { .compatible = "apple,t6050-combo-phy" }, + {}, +}; +MODULE_DEVICE_TABLE(of, t6050_dp_match); +static struct platform_driver t6050_dp_driver = { + .probe = t6050_dp_probe, + .remove = t6050_dp_shutdown, + .shutdown = t6050_dp_shutdown, + .driver = { + .name = "apple-t6050-dp-phy", + .of_match_table = t6050_dp_match, + }, +}; +module_platform_driver(t6050_dp_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple T6050 DisplayPort and Type-C USB combo PHY"); diff --git a/drivers/phy/apple/atc-t6050-typec-data.h b/drivers/phy/apple/atc-t6050-typec-data.h new file mode 100644 index 0000000000000..096307b657c08 --- /dev/null +++ b/drivers/phy/apple/atc-t6050-typec-data.h @@ -0,0 +1,700 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Native firmware27 Type-C DP recipes; board tunables remain DT data. */ +static const struct atc_op dp_reverse_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x3c000fc, 0x48, 0 }, + { ATC_MASK, 2, 0x78, 0x1f, 0x14, 0 }, + { ATC_MASK, 2, 0x78, 0x3ffe0, 0x100, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op mf_normal_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_TUNABLE, 5, 0x0, 0x0, 0x9, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_TUNABLE, 10, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x28000fc, 0x44, 0 }, + { ATC_MASK, 2, 0x78, 0x1ffff, 0x110, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x10000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x100, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x1000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x400, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 38, 0x40, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_FIRMWARE, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1388, 0 }, + { ATC_POLL, 2, 0x260, 0x400, 0x400, 1000000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op mf_reverse_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_TUNABLE, 5, 0x0, 0x0, 0x9, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_TUNABLE, 10, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x14000fc, 0x28, 0 }, + { ATC_MASK, 2, 0x78, 0x1ffff, 0x111, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x20000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x200, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x2000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x800, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 38, 0x40, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_FIRMWARE, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1388, 0 }, + { ATC_POLL, 2, 0x260, 0x400, 0x400, 1000000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op mf_poweroff[] = { + { ATC_MASK, 35, 0x4, 0x1, 0x1, 0 }, + { ATC_MASK, 35, 0x10c, 0x100000, 0x0, 0 }, + { ATC_MASK, 35, 0x110, 0x60, 0x20, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x8, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x0, 0 }, + { ATC_POLL, 35, 0x104, 0x100, 0x100, 1000000 }, + { ATC_MASK, 35, 0x10c, 0x1e000000, 0x4000000, 0 }, + { ATC_MASK, 35, 0x110, 0x600, 0x200, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x8, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x0, 0 }, + { ATC_POLL, 35, 0x104, 0x100, 0x100, 1000000 }, + { ATC_MASK, 35, 0x4, 0x1, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x100, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x200, 0x200, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 5, 0x0, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xc8, 0 }, + { ATC_FIRMWARE, 0, 0x0, 0x0, 0x0, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x0, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x100, 1000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x4, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, +}; + +static const struct atc_op dp_group0_off[] = { + { ATC_MASK, 39, 0x14, 0x1007, 0x7, 0 }, + { ATC_MASK, 39, 0x18, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x1c, 0x1, 0x0, 5000 }, + { ATC_MASK, 14, 0x0, 0x1, 0x1, 0 }, + { ATC_MASK, 22, 0x1c, 0xc00000, 0xc00000, 0 }, + { ATC_MASK, 22, 0x10, 0x600, 0x400, 0 }, + { ATC_MASK, 22, 0x4, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x100000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x400000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x4000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x1000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x1000, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x100, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x44, 0x6000000, 0x4000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x44, 0x18000000, 0x10000000, 0 }, + { ATC_MASK, 19, 0x24, 0xc0000000, 0x80000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x24, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x28, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x40, 0xc000, 0xc000, 0 }, + { ATC_MASK, 16, 0x1d0, 0x3, 0x1, 0 }, + { ATC_MASK, 19, 0x40, 0x10000, 0x0, 0 }, + { ATC_MASK, 2, 0xc, 0x4000, 0x4000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x400, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 14, 0x0, 0xc, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x40000, 0x40000, 0 }, + { ATC_MASK, 3, 0x8, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x80000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x80000, 0x80000, 0 }, + { ATC_MASK, 3, 0x8, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x40000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_POLL, 14, 0x34, 0x8, 0x0, 500000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 6, 0x34, 0xf, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x0, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_MASK, 8, 0x0, 0xaaa, 0x0, 0 }, +}; + +static const struct atc_rate_op dp_group0_on[] = { + { { ATC_MASK, 39, 0x14, 0x7, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 39, 0x18, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x6c, 0x1c, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 14, 0x0, 0x40fff0, 0, 0 }, 0xf, { 0x1110, 0x1110, 0x1110, 0x1110 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x1, { 0x1e0e021c, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x1, { 0x156600, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x1, { 0x70000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x1, { 0x1, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x2, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x2, { 0x0, 0x1e0e01c2, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x2, { 0x0, 0x7fffffe, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x2, { 0x0, 0x155200, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x2, { 0x0, 0x70000, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x2, { 0x0, 0x1, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x4, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x1e0e01c2, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x7fffffe, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x554800, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x4, { 0x0, 0x0, 0x70000, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x4, { 0x0, 0x0, 0x1, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1e0e02a3 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0xbff7ffc } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x564800 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x70000 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1 } }, + { { ATC_MASK, 8, 0x0, 0xaaa, 0, 0 }, 0xf, { 0xaaa, 0xaaa, 0xaaa, 0xaaa } }, + { { ATC_MASK, 6, 0x30, 0x4000, 0, 0 }, 0xf, { 0x4000, 0x4000, 0x4000, 0x4000 } }, + { { ATC_MASK, 6, 0x34, 0xf, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x18, 0x1fe000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_MASK, 6, 0x20, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x1c, 0x7, 0, 0 }, 0xf, { 0x7, 0x7, 0x7, 0x7 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 14, 0x34, 0x8, 0, 500000 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 19, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 14, 0x0, 0xc, 0, 0 }, 0xf, { 0xc, 0xc, 0xc, 0xc } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x10, 0, 0 }, 0xf, { 0x10, 0x10, 0x10, 0x10 } }, + { { ATC_MASK, 2, 0x120, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x4, 0, 0 }, 0xf, { 0x4, 0x4, 0x4, 0x4 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x100, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0x120, 0x400, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0xfffffc0, 0, 0 }, 0xf, { 0xaafb800, 0xaafb800, 0xaafb800, 0xaafb800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x1000000, 0, 0 }, 0xf, { 0x1000000, 0x1000000, 0x1000000, 0x1000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x4000000, 0, 0 }, 0xf, { 0x4000000, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x7c0, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x6c000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 21, 0x100, 0x100000, 0, 0 }, 0xf, { 0x100000, 0x100000, 0x100000, 0x100000 } }, + { { ATC_MASK, 21, 0x58, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 21, 0x104, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 21, 0x100, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x100000, 0, 0 }, 0x7, { 0x100000, 0x100000, 0x100000, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0x7, { 0x1, 0x1, 0x1, 0x0 } }, + { { ATC_MASK, 22, 0x44, 0x400000, 0, 0 }, 0x7, { 0x400000, 0x400000, 0x400000, 0x0 } }, + { { ATC_MASK, 22, 0x4, 0x3000000, 0, 0 }, 0xf, { 0x2000000, 0x2000000, 0x2000000, 0x2000000 } }, + { { ATC_MASK, 22, 0x4, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_MASK, 22, 0xc, 0x7000000, 0, 0 }, 0x1, { 0x5000000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 22, 0xc, 0x7000000, 0, 0 }, 0xe, { 0x0, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_MASK, 22, 0x8, 0x3, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 22, 0x10, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_MASK, 22, 0x1c, 0xc00000, 0, 0 }, 0xf, { 0x800000, 0x800000, 0x800000, 0x800000 } }, + { { ATC_MASK, 19, 0xf4, 0x3000, 0, 0 }, 0xf, { 0x3000, 0x3000, 0x3000, 0x3000 } }, + { { ATC_MASK, 16, 0x188, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d4, 0x700, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_MASK, 19, 0x24, 0x3f0000, 0, 0 }, 0xf, { 0x2a0000, 0x2a0000, 0x2a0000, 0x2a0000 } }, + { { ATC_MASK, 19, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 19, 0x78, 0xc0000, 0, 0 }, 0xf, { 0x80000, 0x80000, 0x80000, 0x80000 } }, + { { ATC_MASK, 19, 0x7c, 0xc0, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 19, 0x84, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_MASK, 19, 0xd4, 0x600000, 0, 0 }, 0xf, { 0x400000, 0x400000, 0x400000, 0x400000 } }, + { { ATC_MASK, 19, 0xd8, 0x300, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_MASK, 19, 0x40, 0xc00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 19, 0x48, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 19, 0x28, 0x1ff030, 0, 0 }, 0xf, { 0x11c020, 0x11c020, 0x11c020, 0x11c020 } }, + { { ATC_MASK, 19, 0x2c, 0xfc, 0, 0 }, 0xf, { 0xb0, 0xb0, 0xb0, 0xb0 } }, + { { ATC_MASK, 19, 0x28, 0xe00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 19, 0x28, 0x1c0, 0, 0 }, 0xf, { 0x180, 0x180, 0x180, 0x180 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x10, 0, 0 }, 0xf, { 0x10, 0x10, 0x10, 0x10 } }, + { { ATC_MASK, 2, 0x10, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x4, 0, 0 }, 0xf, { 0x4, 0x4, 0x4, 0x4 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x400, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 2, 0xc, 0x1000, 0, 0 }, 0xf, { 0x1000, 0x1000, 0x1000, 0x1000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_MASK, 2, 0xc, 0x100, 0, 0 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x4000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0xc, 0x10000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d0, 0x3, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d0, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x40, 0x30000, 0, 0 }, 0xf, { 0x30000, 0x30000, 0x30000, 0x30000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x40, 0xc000, 0, 0 }, 0xf, { 0xc000, 0xc000, 0xc000, 0xc000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x3010, 0, 0 }, 0xf, { 0x1010, 0x1010, 0x1010, 0x1010 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x3000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x24, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x24, 0xc0000000, 0, 0 }, 0xf, { 0xc0000000, 0xc0000000, 0xc0000000, 0xc0000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 19, 0x44, 0x18000000, 0, 0 }, 0xf, { 0x18000000, 0x18000000, 0x18000000, 0x18000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 19, 0x40, 0x38, 0, 0 }, 0x1, { 0x28, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 19, 0x40, 0x38, 0, 0 }, 0xe, { 0x0, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 19, 0x44, 0x60000000, 0, 0 }, 0xf, { 0x40000000, 0x40000000, 0x40000000, 0x40000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x44, 0x6000000, 0, 0 }, 0xf, { 0x6000000, 0x6000000, 0x6000000, 0x6000000 } }, + { { ATC_MASK, 19, 0x40, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 20, 0x68, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 20, 0xac, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 20, 0xa8, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 20, 0x18, 0x38, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 14, 0x0, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 39, 0x1c, 0x1, 0, 5000 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, +}; + +static const struct atc_op dp_group1_off[] = { + { ATC_MASK, 39, 0x14, 0x1007, 0x7, 0 }, + { ATC_MASK, 39, 0x18, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x1c, 0x1, 0x0, 5000 }, + { ATC_MASK, 14, 0x0, 0x1, 0x1, 0 }, + { ATC_MASK, 31, 0x1c, 0xc00000, 0xc00000, 0 }, + { ATC_MASK, 31, 0x10, 0x600, 0x400, 0 }, + { ATC_MASK, 31, 0x4, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x100000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x400000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x4000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x1000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x1000, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x200, 0x200, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x44, 0x6000000, 0x4000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x44, 0x18000000, 0x10000000, 0 }, + { ATC_MASK, 28, 0x24, 0xc0000000, 0x80000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x24, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x28, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x40, 0xc000, 0xc000, 0 }, + { ATC_MASK, 25, 0x1d0, 0x3, 0x1, 0 }, + { ATC_MASK, 28, 0x40, 0x10000, 0x0, 0 }, + { ATC_MASK, 2, 0xc, 0x8000, 0x8000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x800, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 14, 0x0, 0xc, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x400000, 0x400000, 0 }, + { ATC_MASK, 3, 0x8, 0x4, 0x4, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x800000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x800000, 0x800000, 0 }, + { ATC_MASK, 3, 0x8, 0x8, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x400000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_POLL, 14, 0x34, 0x8, 0x0, 500000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 6, 0x34, 0xf, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x0, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_MASK, 8, 0x0, 0xaaa, 0x0, 0 }, +}; + +static const struct atc_rate_op dp_group1_on[] = { + { { ATC_MASK, 39, 0x14, 0x7, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 39, 0x18, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x6c, 0xe0, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_MASK, 14, 0x0, 0x40fff0, 0, 0 }, 0xf, { 0x1110, 0x1110, 0x1110, 0x1110 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x1, { 0x1e0e021c, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x1, { 0x156600, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x1, { 0x70000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x1, { 0x1, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x2, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x2, { 0x0, 0x1e0e01c2, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x2, { 0x0, 0x7fffffe, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x2, { 0x0, 0x155200, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x2, { 0x0, 0x70000, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x2, { 0x0, 0x1, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x4, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x1e0e01c2, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x7fffffe, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x554800, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x4, { 0x0, 0x0, 0x70000, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x4, { 0x0, 0x0, 0x1, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1e0e02a3 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0xbff7ffc } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x564800 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x70000 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1 } }, + { { ATC_MASK, 8, 0x0, 0xaaa, 0, 0 }, 0xf, { 0xaaa, 0xaaa, 0xaaa, 0xaaa } }, + { { ATC_MASK, 6, 0x30, 0x4000, 0, 0 }, 0xf, { 0x4000, 0x4000, 0x4000, 0x4000 } }, + { { ATC_MASK, 6, 0x34, 0xf, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x18, 0x1fe000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_MASK, 6, 0x20, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x1c, 0x7, 0, 0 }, 0xf, { 0x7, 0x7, 0x7, 0x7 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 14, 0x34, 0x8, 0, 500000 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 28, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 14, 0x0, 0xc, 0, 0 }, 0xf, { 0xc, 0xc, 0xc, 0xc } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x20, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 2, 0x120, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 2, 0x120, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x200, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0x120, 0x800, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0xfffffc0, 0, 0 }, 0xf, { 0xaafb800, 0xaafb800, 0xaafb800, 0xaafb800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x1000000, 0, 0 }, 0xf, { 0x1000000, 0x1000000, 0x1000000, 0x1000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x4000000, 0, 0 }, 0xf, { 0x4000000, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x7c0, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x6c000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 30, 0x100, 0x100000, 0, 0 }, 0xf, { 0x100000, 0x100000, 0x100000, 0x100000 } }, + { { ATC_MASK, 30, 0x58, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 30, 0x104, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 30, 0x100, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x100000, 0, 0 }, 0x7, { 0x100000, 0x100000, 0x100000, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0x7, { 0x1, 0x1, 0x1, 0x0 } }, + { { ATC_MASK, 31, 0x44, 0x400000, 0, 0 }, 0x7, { 0x400000, 0x400000, 0x400000, 0x0 } }, + { { ATC_MASK, 31, 0x4, 0x3000000, 0, 0 }, 0xf, { 0x2000000, 0x2000000, 0x2000000, 0x2000000 } }, + { { ATC_MASK, 31, 0x4, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_MASK, 31, 0xc, 0x7000000, 0, 0 }, 0x1, { 0x5000000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 31, 0xc, 0x7000000, 0, 0 }, 0xe, { 0x0, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_MASK, 31, 0x8, 0x3, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 31, 0x10, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_MASK, 31, 0x1c, 0xc00000, 0, 0 }, 0xf, { 0x800000, 0x800000, 0x800000, 0x800000 } }, + { { ATC_MASK, 28, 0xf4, 0x3000, 0, 0 }, 0xf, { 0x3000, 0x3000, 0x3000, 0x3000 } }, + { { ATC_MASK, 25, 0x188, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d4, 0x700, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_MASK, 28, 0x24, 0x3f0000, 0, 0 }, 0xf, { 0x2a0000, 0x2a0000, 0x2a0000, 0x2a0000 } }, + { { ATC_MASK, 28, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 28, 0x78, 0xc0000, 0, 0 }, 0xf, { 0x80000, 0x80000, 0x80000, 0x80000 } }, + { { ATC_MASK, 28, 0x7c, 0xc0, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 28, 0x84, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_MASK, 28, 0xd4, 0x600000, 0, 0 }, 0xf, { 0x400000, 0x400000, 0x400000, 0x400000 } }, + { { ATC_MASK, 28, 0xd8, 0x300, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_MASK, 28, 0x40, 0xc00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 28, 0x48, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 28, 0x28, 0x1ff030, 0, 0 }, 0xf, { 0x11c020, 0x11c020, 0x11c020, 0x11c020 } }, + { { ATC_MASK, 28, 0x2c, 0xfc, 0, 0 }, 0xf, { 0xb0, 0xb0, 0xb0, 0xb0 } }, + { { ATC_MASK, 28, 0x28, 0xe00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 28, 0x28, 0x1c0, 0, 0 }, 0xf, { 0x180, 0x180, 0x180, 0x180 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x20, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 2, 0x10, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 2, 0x10, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x800, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 2, 0xc, 0x2000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 2, 0xc, 0x200, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x8000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0xc, 0x20000, 0, 0 }, 0xf, { 0x20000, 0x20000, 0x20000, 0x20000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d0, 0x3, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d0, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x40, 0x30000, 0, 0 }, 0xf, { 0x30000, 0x30000, 0x30000, 0x30000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x40, 0xc000, 0, 0 }, 0xf, { 0xc000, 0xc000, 0xc000, 0xc000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x3010, 0, 0 }, 0xf, { 0x1010, 0x1010, 0x1010, 0x1010 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x3000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x24, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x24, 0xc0000000, 0, 0 }, 0xf, { 0xc0000000, 0xc0000000, 0xc0000000, 0xc0000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 28, 0x44, 0x18000000, 0, 0 }, 0xf, { 0x18000000, 0x18000000, 0x18000000, 0x18000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 28, 0x40, 0x38, 0, 0 }, 0x1, { 0x28, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 28, 0x40, 0x38, 0, 0 }, 0xe, { 0x0, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 28, 0x44, 0x60000000, 0, 0 }, 0xf, { 0x40000000, 0x40000000, 0x40000000, 0x40000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x44, 0x6000000, 0, 0 }, 0xf, { 0x6000000, 0x6000000, 0x6000000, 0x6000000 } }, + { { ATC_MASK, 28, 0x40, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 29, 0x68, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 29, 0xac, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 29, 0xa8, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 29, 0x18, 0x38, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 14, 0x0, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 39, 0x1c, 0x1, 0, 5000 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, +}; diff --git a/drivers/phy/apple/dptx.c b/drivers/phy/apple/dptx.c index 5e58f149ce96a..f191efe4c6286 100644 --- a/drivers/phy/apple/dptx.c +++ b/drivers/phy/apple/dptx.c @@ -16,11 +16,14 @@ #include #include "linux/of.h" #include +#include +#include #include #include #include #include #include +#include #include #define DPTX_MAX_LANES 4 @@ -31,6 +34,23 @@ enum apple_dptx_type { DPTX_PHY_T8112, DPTX_PHY_T6020, + DPTX_PHY_T6050, +}; + +/* The PLL core calibration the board describes, in t6050_pll_core_cfg() order. */ +enum { + T6050_DCOCFG, + T6050_DTCVREG, + T6050_DTCCAL, + T6050_DCDADJ, + T6050_DLFCFG, + T6050_RODCOROLEAKCOMP, + T6050_NR_CAL, +}; + +static const char *const t6050_cal_props[T6050_NR_CAL] = { + "apple,pll-dcocfg", "apple,pll-dtcvreg", "apple,pll-dtccal", + "apple,pll-dcdadj", "apple,pll-dlfcfg", "apple,pll-rodcoroleakcomp", }; struct apple_dptx_phy_hw { @@ -49,6 +69,14 @@ struct apple_dptx_phy { void __iomem *dptx; } regs; + /* T6050 */ + u32 cal[T6050_NR_CAL]; + bool cal_set[T6050_NR_CAL]; + u32 source_select; + bool pclkdrv_select; + bool active; + u32 rate; /* Mbit/s; 0: none programmed */ + struct phy *phy_dp; struct phy_provider *phy_provider; @@ -523,10 +551,398 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) return 0; } +/* + * T6050 (M5 Ultra) lpdptx. The two banks are T602x's ("core" and "dptx", the + * PLL command register at dptx+0x100c), but the PLL calibration, clocking, + * rate programming, AUX calibration and PHY power are T8122/T8132's. Every + * write is a field update; nothing below replays a captured register word. + */ +static void t6050_rmw(void __iomem *base, u32 off, u32 shift, u32 mask, u32 val) +{ + mask32(base + off, mask, (val << shift) & mask); +} + +#define T6050_D(phy, off, shift, mask, val) \ + t6050_rmw((phy)->regs.dptx, off, shift, mask, val) +#define T6050_C(phy, off, shift, mask, val) \ + t6050_rmw((phy)->regs.core, off, shift, mask, val) + +/* core+0x48: PLL (bits 6:4), AUX (10:8) and PHY (2:0) power, 1 us apart. */ +static void t6050_block_power(struct apple_dptx_phy *phy, u32 lsb, bool on) +{ + if (on) { + T6050_C(phy, 0x48, lsb, BIT(lsb), 1); + udelay(1); + T6050_C(phy, 0x48, lsb + 1, BIT(lsb + 1), 1); + udelay(1); + T6050_C(phy, 0x48, lsb + 2, BIT(lsb + 2), 0); + udelay(1); + } else { + T6050_C(phy, 0x48, lsb + 2, BIT(lsb + 2), 1); + udelay(1); + T6050_C(phy, 0x48, lsb + 1, BIT(lsb + 1), 0); + udelay(1); + T6050_C(phy, 0x48, lsb, BIT(lsb), 0); + udelay(1); + } +} + +static void t6050_device_power(struct apple_dptx_phy *phy, bool on) +{ + T6050_C(phy, 0x10, 0, 0x7, phy->source_select); + t6050_block_power(phy, 4, on); /* PLL */ + t6050_block_power(phy, 8, on); /* AUX */ + t6050_block_power(phy, 0, on); /* PHY */ +} + +/* The board's PLL core calibration (an absent value leaves its field), then fixed settings. */ +static void t6050_pll_core_cfg(struct apple_dptx_phy *phy) +{ + const u32 *v = phy->cal; + const bool *set = phy->cal_set; + + if (set[T6050_DCOCFG]) { + T6050_D(phy, 0x2240, 1, 0x6, (v[T6050_DCOCFG] >> 3) & 3); + T6050_D(phy, 0x2240, 4, 0x70, v[T6050_DCOCFG] & 7); + } + if (set[T6050_DTCVREG]) + T6050_D(phy, 0x2230, 0, 0x1f, v[T6050_DTCVREG]); + if (set[T6050_DTCCAL]) + T6050_D(phy, 0x22e8, 17, 0x60000, v[T6050_DTCCAL]); + if (set[T6050_DCDADJ]) + T6050_D(phy, 0x2240, 22, 0x7c00000, v[T6050_DCDADJ]); + if (set[T6050_DLFCFG]) + T6050_D(phy, 0x22b0, 20, 0x700000, v[T6050_DLFCFG]); + if (set[T6050_RODCOROLEAKCOMP]) + T6050_D(phy, 0x2240, 19, 0x180000, v[T6050_RODCOROLEAKCOMP]); + + T6050_D(phy, 0x22e8, 0, 0x1, 1); + T6050_D(phy, 0x2240, 3, 0x8, 1); + T6050_D(phy, 0x2240, 0, 0x1, 1); + T6050_D(phy, 0x2224, 10, 0xc00, 2); + T6050_D(phy, 0x22b0, 27, 0x08000000, 0); + T6050_D(phy, 0x2338, 23, 0x0f800000, 0xf); +} + +/* The pixel clock driver: dptx+0x2200 bit 3 with pclkdrv-select, else bit 1. */ +static bool t6050_pclk(struct apple_dptx_phy *phy, bool on) +{ + u32 bit = phy->pclkdrv_select ? 3 : 1; + + if (!on) { + if (!(readl(phy->regs.dptx + 0x2200) & BIT(bit))) + return false; + } + T6050_D(phy, 0x2200, bit, BIT(bit), on); + return true; +} + +static void t6050_configure_pll(struct apple_dptx_phy *phy) +{ + T6050_D(phy, 0x2218, 0, 0x1, 1); + T6050_D(phy, 0x2234, 0, 0x3, 0); + t6050_pclk(phy, true); + T6050_D(phy, 0x2320, 1, 0x6, 1); /* single-ended clock output */ +} + +/* AUX impedance calibration: start, sample completion once, copy the result. */ +static int t6050_impedance_calibration(struct apple_dptx_phy *phy) +{ + u32 done, code; + + T6050_D(phy, 0x4224, 0, 0x1, 0); + usleep_range(1000, 1200); + T6050_D(phy, 0x4224, 0, 0x1, 1); + udelay(1); + T6050_D(phy, 0x4200, 4, 0x3f0, 0x3c); + udelay(1); + T6050_D(phy, 0x4200, 0, 0x1, 1); + udelay(125); + done = readl(phy->regs.dptx + 0x4204) & 1; + udelay(10); + T6050_D(phy, 0x4200, 0, 0x1, 0); + code = (readl(phy->regs.dptx + 0x4210) >> 18) & 0x7f; + if (!done) { + dev_err(phy->dev, "impedance calibration did not complete (0x4204 %#x)\n", + readl(phy->regs.dptx + 0x4204)); + return -ETIMEDOUT; + } + T6050_D(phy, 0x4228, 16, 0x7f0000, code); + T6050_D(phy, 0x4228, 15, 0x8000, 1); + dev_dbg(phy->dev, "impedance calibration code %#x\n", code); + return 0; +} + +static int t6050_activate(struct apple_dptx_phy *phy) +{ + t6050_device_power(phy, true); + phy->active = true; + t6050_pll_core_cfg(phy); /* PLL top, PHY and AUX configuration: nothing on this board */ + t6050_configure_pll(phy); + /* PLL activation */ + T6050_D(phy, 0x1000, 1, 0x2, 0); + T6050_D(phy, 0x4004, 3, 0x8, 1); + /* PHY activation: every lane, highest first */ + udelay(1); + for (int l = DPTX_MAX_LANES - 1; l >= 0; l--) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l, 9, 0x200, 0); + /* AUX power-up */ + T6050_D(phy, 0xd000, 0, 0x1, 0); + return t6050_impedance_calibration(phy); +} + +/* One PLL command (7 up, 0 down): request, 100 us, acknowledgment, lock state. */ +static int t6050_pll_command(struct apple_dptx_phy *phy, u32 cmd) +{ + bool up = cmd == 7; + u32 ack, sts; + + T6050_D(phy, 0x100c, 0, 0x7, cmd); + T6050_D(phy, 0x100c, 3, 0x8, 1); + udelay(100); + ack = readl(phy->regs.dptx + 0x1014); + T6050_D(phy, 0x100c, 3, 0x8, 0); + sts = readl(phy->regs.dptx + 0x1008); + if (!(ack & 1)) { + dev_err(phy->dev, "PLL power %s: no acknowledgment (0x1014 %#x)\n", + up ? "up" : "down", ack); + return -ETIMEDOUT; + } + if (!!(sts & BIT(3)) != up) { + dev_err(phy->dev, "PLL power %s: lock is %s (0x1008 %#x)\n", + up ? "up" : "down", up ? "missing" : "still set", sts); + return -EIO; + } + return 0; +} + +static int t6050_pll_power_down(struct apple_dptx_phy *phy) +{ + int ret; + + if (!t6050_pclk(phy, false)) + return 0; /* already down */ + ret = t6050_pll_command(phy, 0); + udelay(1); + return ret; +} + +static int t6050_pll_power_up(struct apple_dptx_phy *phy) +{ + int ret = t6050_pll_command(phy, 7); + + if (!ret) + t6050_pclk(phy, true); + return ret; +} + +static void t6050_phy_power_down(struct apple_dptx_phy *phy) +{ + T6050_D(phy, 0x4004, 0, 0x1, 1); + T6050_D(phy, 0x4000, 6, 0x40, 0); + T6050_D(phy, 0x4004, 3, 0x8, 0); + T6050_D(phy, 0x4000, 25, 0x2000000, 0); + T6050_D(phy, 0x4000, 24, 0x1000000, 0); +} + +static void t6050_phy_power_up(struct apple_dptx_phy *phy) +{ + T6050_D(phy, 0x4004, 3, 0x8, 1); + for (int l = 0; l < DPTX_MAX_LANES; l++) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l + 0x10, + 27, 0x38000000, 4); + T6050_D(phy, 0x4030, 0, 0x1, 1); /* TXA LDO low-power mode */ + udelay(1); + T6050_D(phy, 0x4000, 24, 0x1000000, 1); + ndelay(250); + T6050_D(phy, 0x4000, 25, 0x2000000, 1); + udelay(1); + T6050_D(phy, 0x4000, 6, 0x40, 1); + udelay(1); + T6050_D(phy, 0x4004, 0, 0x1, 0); +} + +static int t6050_set_lanes(struct apple_dptx_phy *phy, u32 n) +{ + if (n == 3 || n > DPTX_MAX_LANES) + return -EINVAL; + if (!phy->active) + return 0; + T6050_D(phy, 0x4000, 26, 0x4000000, 1); + udelay(1); + for (u32 l = 0; l < DPTX_MAX_LANES; l++) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l, 8, 0x100, n <= l); + if (n) + T6050_D(phy, 0x4000, 26, 0x4000000, 0); + phy->active_lanes = n; + return 0; +} + +/* + * The PLL record of a link rate (after the T8122 adjustment of its first two + * words): the integer/fraction divider q08 + q0c / q10, the analog fields and + * whether the PLL clock output is bypassed. + */ +struct t6050_pll_cfg { + u32 rate; + u8 q00, q04, q14, q18, q1c; + u16 q08, q0c, q10, q24, q28; + bool bypass; +}; + +static const struct t6050_pll_cfg t6050_pll_cfgs[] = { + { 1620, 1, 0, 1, 5, 0x13, 0x87, 0, 0, 0x80, 0x8804, true }, + { 2700, 1, 0, 1, 5, 0x09, 0x70, 0x1fff, 0x3ffe, 0x80, 0x8804, true }, + { 5400, 1, 1, 1, 5, 0x04, 0x70, 0x1fff, 0x3ffe, 0x100, 0x8984, true }, + { 8100, 1, 1, 1, 6, 0x04, 0xa8, 0x2ffd, 0x3ffc, 0x100, 0x8984, false }, +}; + +static void t6050_shadow_update(struct apple_dptx_phy *phy) +{ + set32(phy->regs.core + 0xa0, 0x8); + set32(phy->regs.core + 0xa0, 0x4); + set32(phy->regs.core + 0xa0, 0x40000); + clear32(phy->regs.core + 0xa0, 0x40000); + set32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x4); + clear32(phy->regs.core + 0xa0, 0x8); +} + +/* Program a link rate's PLL frequency and half-rate fields (the PLL and PHY stay as they are). */ +static int t6050_program_rate(struct apple_dptx_phy *phy, u32 rate) +{ + const struct t6050_pll_cfg *c = NULL; + u32 q10, f, e; + bool half; + + for (int i = 0; i < ARRAY_SIZE(t6050_pll_cfgs); i++) + if (t6050_pll_cfgs[i].rate == rate) + c = &t6050_pll_cfgs[i]; + if (!c) + return -EINVAL; + + q10 = c->q10 ? c->q10 : 1; + f = 4 * c->q08 + (4 * c->q0c) / q10; + e = c->q0c > 1 ? c->q10 : 0; + half = rate >= 2000; + + T6050_D(phy, 0x222c, 1, 0x2, c->bypass); + T6050_D(phy, 0x222c, 0, 0x1, !c->bypass); + T6050_D(phy, 0x2244, 0, 0xffff, c->q28); + T6050_D(phy, 0x2234, 23, 0xff800000, c->q24); + T6050_D(phy, 0x20b0, 0, 0x3ff, f); + T6050_D(phy, 0x20b0, 10, 0x400, 0); + T6050_D(phy, 0x20b4, 14, 0x0fffc000, c->q0c); + T6050_D(phy, 0x20b4, 0, 0x3fff, e); + T6050_D(phy, 0x20b8, 9, 0x3e00, c->q1c); + T6050_D(phy, 0x20b8, 14, 0xc000, c->q14); + T6050_D(phy, 0x20b8, 16, 0xf0000, c->q18); + T6050_D(phy, 0x20b8, 20, 0x300000, c->q00); + T6050_D(phy, 0x20b8, 22, 0x400000, c->q04); + t6050_shadow_update(phy); + + /* half-link-rate: PHY, then PLL */ + T6050_D(phy, 0x4004, 5, 0x20, !half); + T6050_D(phy, 0x4004, 6, 0x40, half); + T6050_D(phy, 0x2000, 29, 0x20000000, !half); + T6050_D(phy, 0x201c, 5, 0x20, !half); + phy->rate = rate; + return 0; +} + +/* + * A link reconfiguration as the DCP brackets it: PHY and PLL down, the new + * lane count and rate, then (for a nonzero rate) PLL and PHY up. + */ +static int t6050_reconfigure(struct apple_dptx_phy *phy, bool set_lanes, + u32 lanes, bool set_rate, u32 rate) +{ + int ret; + + if (!phy->active) + return -ENODEV; + t6050_phy_power_down(phy); + ret = t6050_pll_power_down(phy); + if (ret) + return ret; + if (set_lanes) { + ret = t6050_set_lanes(phy, lanes); + if (ret) + return ret; + } + if (set_rate) { + if (!rate) { + phy->rate = 0; + return 0; + } + ret = t6050_program_rate(phy, rate); + if (ret) + return ret; + } + if (!phy->rate) + return 0; + ret = t6050_pll_power_up(phy); + if (ret) + return ret; + t6050_phy_power_up(phy); + return 0; +} + +static int t6050_deactivate(struct apple_dptx_phy *phy) +{ + int ret; + + if (!phy->active) + return 0; + t6050_set_lanes(phy, 0); + t6050_phy_power_down(phy); + for (int l = 0; l < DPTX_MAX_LANES; l++) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l, 9, 0x200, 1); + ret = t6050_pll_power_down(phy); + T6050_D(phy, 0xd000, 0, 0x1, 1); /* AUX power-down */ + phy->active = false; + phy->rate = 0; + t6050_device_power(phy, false); + return ret; +} + +static int t6050_probe(struct apple_dptx_phy *phy) +{ + struct device_node *np = phy->dev->of_node; + u32 v; + + for (int i = 0; i < T6050_NR_CAL; i++) { + if (of_property_read_u32(np, t6050_cal_props[i], &v) || v == 0xffffffff) + continue; + phy->cal[i] = v; + phy->cal_set[i] = true; + } + /* The crossbar's ufp endpoint u: 5 and 6 select themselves, 0xf none, any other u + 1. */ + if (of_property_read_u32(np, "apple,ufp-endpoint", &v)) + return dev_err_probe(phy->dev, -EINVAL, "no apple,ufp-endpoint\n"); + phy->source_select = (v == 5 || v == 6) ? v : v == 0xf ? 0 : v + 1; + phy->pclkdrv_select = !of_property_read_u32(np, "apple,pclkdrv-select", &v) && v; + + return 0; +} + static int dptx_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + if (dptx_phy->hw.type == DPTX_PHY_T6050) { + int ret = -EINVAL; + + mutex_lock(&dptx_phy->lock); + if (mode == PHY_MODE_INVALID) + ret = t6050_deactivate(dptx_phy); + else if (mode == PHY_MODE_DP) + ret = dptx_phy->active ? 0 : t6050_activate(dptx_phy); + mutex_unlock(&dptx_phy->lock); + return ret; + } + switch (mode) { case PHY_MODE_INVALID: return dptx_phy_deactivate(dptx_phy); @@ -574,6 +990,16 @@ static int dptx_phy_configure(struct phy *phy, union phy_configure_opts *opts_) enum dptx_phy_link_rate link_rate; int ret = 0; + if (dptx_phy->hw.type == DPTX_PHY_T6050) { + if (!opts->set_lanes && !opts->set_rate) + return 0; + mutex_lock(&dptx_phy->lock); + ret = t6050_reconfigure(dptx_phy, opts->set_lanes, opts->lanes, + opts->set_rate, opts->link_rate); + mutex_unlock(&dptx_phy->lock); + return ret; + } + if (opts->set_lanes) { mutex_lock(&dptx_phy->lock); ret = dptx_phy_set_active_lane_count(dptx_phy, opts->lanes); @@ -656,6 +1082,8 @@ static int dptx_phy_probe(struct platform_device *pdev) if (IS_ERR(dptx_phy->phy_provider)) return PTR_ERR(dptx_phy->phy_provider); + if (dptx_phy->hw.type == DPTX_PHY_T6050) + return t6050_probe(dptx_phy); return 0; } @@ -667,9 +1095,14 @@ static const struct apple_dptx_phy_hw apple_dptx_hw_t8112 = { .type = DPTX_PHY_T8112, }; +static const struct apple_dptx_phy_hw apple_dptx_hw_t6050 = { + .type = DPTX_PHY_T6050, +}; + static const struct of_device_id dptx_phy_match[] = { { .compatible = "apple,t6020-dptx-phy", .data = &apple_dptx_hw_t6020 }, { .compatible = "apple,t8112-dptx-phy", .data = &apple_dptx_hw_t8112 }, + { .compatible = "apple,t6050-dptx-phy", .data = &apple_dptx_hw_t6050 }, {}, }; MODULE_DEVICE_TABLE(of, dptx_phy_match); diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h b/drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h new file mode 100644 index 0000000000000..4af28f8c2ccff --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: MIT */ +/* ACIO80 PHY CIO (USB4/Thunderbolt) mode. + * + * The bring-up runs the mode-independent power-up (the USB recipes' first + * block), the common and CIO/CIO80 tunables, the crossbar, the sleep-control + * release and the firmware start. CIO applies no DP or USB lane masks, + * no DP clock switch and no orientation bit in ATCPHY_CFG +0x8; orientation + * only selects ACIOPHY_TOP +0x78 bit 0. Both lanes are released. + * Spread-spectrum stays on. + */ +#ifndef APPLE_T6050_CIO_PHY_H +#define APPLE_T6050_CIO_PHY_H +#include "apple_t6050_usb_phy.h" + +/* The power-up and reset release, identical to the USB recipes. */ +#define APPLE_T6050_CIO_POWERUP \ + { T6050_MASK, 39, 0x0, 0x1, 0x1, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_POLL, 39, 0x4, 0x1, 0x1, 1000, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x0, 0x2, 0x2, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_POLL, 39, 0x4, 0x2, 0x2, 1000, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x0, 0x4, 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_POWER, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x2c, 0x1, 0x0, 0, 0 }, \ + { T6050_POLL, 39, 0x4, 0x100, 0x0, 1000, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x0, 0x8, 0x8, 0, 0 } + +/* The common tunables, then the CIO set and the CIO80 set. An absent CIO80 + * tunable is skipped. + */ +#define APPLE_T6050_CIO_TUNABLES \ + { T6050_TUNABLE, 41, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF" }, \ + { T6050_TUNABLE, 40, 0x0, 0x0, 0x0, 0, "tunable_ATC_FABRIC" }, \ + { T6050_TUNABLE, 43, 0x0, 0x0, 0x0, 0, "tunable_ATC_COMMON_CFG" }, \ + { T6050_TUNABLE, 42, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF_LIOA" }, \ + { T6050_TUNABLE, 20, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_CFG_TX_OF_RXCLK" }, \ + { T6050_TUNABLE, 29, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_CFG_TX_OF_RXCLK" }, \ + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_DFLT" }, \ + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_DFLT" }, \ + { T6050_TUNABLE, 5, 0x0, 0x0, 0x0, 0, "tunable_AUS40CMN_SHM" }, \ + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, \ + { T6050_TUNABLE, 10, 0x0, 0x0, 0x0, 0, "tunable_CIO4PLL_CORE" }, \ + { T6050_TUNABLE, 3, 0x0, 0x0, 0x0, 0, "tunable_UC_REGS_CIO_DFLT" }, \ + { T6050_TUNABLE, 16, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_TOP_CIO_DFLT" }, \ + { T6050_TUNABLE, 25, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_TOP_CIO_DFLT" }, \ + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_CIO_DFLT" }, \ + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_CIO_DFLT" } + +/* The crossbar: clear the lane selects, set the orientation bit (+0x78) + * and select CIO. + */ +#define APPLE_T6050_CIO_XBAR(reverse) \ + { T6050_MASK, 2, 0x6c, 0xfc, 0x0, 0, 0 }, \ + { T6050_MASK, 2, 0x78, 0x1f, (reverse), 0, 0 }, \ + { T6050_MASK, 2, 0x6c, 0x3, 0x2, 0, 0 } + +/* Power a lane's receiver up, then release its transmitter overrides. */ +#define APPLE_T6050_CIO_LANE(l) \ + { T6050_MASK, 2, 0x10, 0x10 << (l), 0x10 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x40 << (l), 0x40 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x1 << (l), 0x1 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x4 << (l), 0x4 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x400 << (l), 0x400 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x1000 << (l), 0x1000 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x40 << (l), 0x40 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x100 << (l), 0x100 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x4000 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x10000 << (l), 0x10000 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x40 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x4 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x400 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 } + +/* The sleep-control release: the common block's overrides, then both lanes. */ +#define APPLE_T6050_CIO_SLPCTRL \ + { T6050_MASK, 2, 0xc, 0x2a, 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + APPLE_T6050_CIO_LANE(0), \ + APPLE_T6050_CIO_LANE(1) + +/* Start the PHY firmware, wait for UC_TNBL_DONE and + * deassert the ACIOPHY reset. No PIPE BIST and no ATCPHY_CFG +0x8 write. + */ +#define APPLE_T6050_CIO_START \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_MASK, 38, 0x40, 0x1, 0x1, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, \ + { T6050_POLL, 2, 0x260, 0x400, 0x400, 1000000, 0 }, \ + { T6050_MASK, 39, 0x0, 0x10, 0x10, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x64, 0, 0 } + +static const struct apple_t6050_usb_step apple_t6050_cio_init_normal[] = { + APPLE_T6050_CIO_POWERUP, + APPLE_T6050_CIO_TUNABLES, + APPLE_T6050_CIO_XBAR(0x0), + APPLE_T6050_CIO_SLPCTRL, + APPLE_T6050_CIO_START, +}; + +static const struct apple_t6050_usb_step apple_t6050_cio_init_reverse[] = { + APPLE_T6050_CIO_POWERUP, + APPLE_T6050_CIO_TUNABLES, + APPLE_T6050_CIO_XBAR(0x1), + APPLE_T6050_CIO_SLPCTRL, + APPLE_T6050_CIO_START, +}; + +/* Power a lane's transmitter down. */ +#define APPLE_T6050_CIO_TX_DOWN(l) \ + { T6050_MASK, 2, 0x120, 0x100 << (l), 0x100 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x1 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x10 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 } + +/* Power a lane's receiver down; CIO powers the receivers down, USB does not. */ +#define APPLE_T6050_CIO_RX_DOWN(l) \ + { T6050_MASK, 2, 0xc, 0x4000 << (l), 0x4000 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x40 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x400 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x1 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x10 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 } + +/* The shutdown: both lanes down without the PIPE BIST, the firmware + * power-down, then the USB recipes' sleep, clamp and reset sequence. + */ +static const struct apple_t6050_usb_step apple_t6050_cio_shutdown[] = { + APPLE_T6050_CIO_TX_DOWN(0), + APPLE_T6050_CIO_TX_DOWN(1), + { T6050_MASK, 5, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + APPLE_T6050_CIO_RX_DOWN(0), + APPLE_T6050_CIO_RX_DOWN(1), + { T6050_DELAY, 0, 0x0, 0x0, 0xc8, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x100, 1000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x4, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, +}; +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h new file mode 100644 index 0000000000000..503cbb537f865 --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef APPLE_T6050_USB2_RESET_H +#define APPLE_T6050_USB2_RESET_H +#include "apple_t6050_usb3_bist.h" + +/* Native T6050 Type-C PHY USB2 port reset block, native27: + * bank0+4 bit1 assert, sleep 5 ms, release. + * Caller holds exclusive PHY ownership with its USB power domains active. + * sleep_ms is supplied by the OS frontend; there is no packet-path polling. */ +static inline int apple_t6050_usb2_port_reset( + const struct apple_t6050_usb_io *io, void *sleep_context, + void (*sleep_ms)(void *, unsigned int)) +{ + if (!io || !io->read || !io->write || !sleep_ms) + return -1; + apple_t6050_usb_mask(io, 0, 4, 2U, 2U); + if (!(io->read(io->context, 0, 4) & 2U)) + return -1; + sleep_ms(sleep_context, 5); + apple_t6050_usb_mask(io, 0, 4, 2U, 0); + return io->read(io->context, 0, 4) & 2U ? -1 : 0; +} +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h new file mode 100644 index 0000000000000..3af8fcf2a3a13 --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: MIT */ +/* T6050-specific native PHY recipe; no Windows/Linux frontend policy. */ +#ifndef APPLE_T6050_USB3_BIST_H +#define APPLE_T6050_USB3_BIST_H +struct apple_t6050_usb_io { + void *context; + unsigned int (*read)(void *, unsigned int bank, unsigned int offset); + void (*write)(void *, unsigned int bank, unsigned int offset, unsigned int value); + int (*poll)(void *, unsigned int bank, unsigned int offset, unsigned int mask, unsigned int value, unsigned int usec); + void (*delay)(void *, unsigned int usec); +}; +static inline void apple_t6050_usb_mask(const struct apple_t6050_usb_io *io, unsigned int bank, unsigned int offset, unsigned int mask, unsigned int value) +{ io->write(io->context, bank, offset, (io->read(io->context, bank, offset) & ~mask) | value); } +static inline int apple_t6050_usb_bist_off(const struct apple_t6050_usb_io *io) +{ + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x110, 0x660, 0x0); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x8); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x0); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x1, 0x0); + return 0; +} +static inline int apple_t6050_usb_bist_on(const struct apple_t6050_usb_io *io) +{ + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x1, 0x1); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x10c, 0x1e100000, 0x4100000); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x110, 0x660, 0x220); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x8); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x0); + if (io->poll(io->context, 35, 0x104, 0x100, 0x0, 20000)) return -1; + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x20); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x0); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x10c, 0x1e000000, 0x6000000); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x110, 0x600, 0x200); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x8); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x0); + if (io->poll(io->context, 35, 0x104, 0x40, 0x40, 130000)) return -1; + if (io->poll(io->context, 35, 0x104, 0x100, 0x0, 1000)) return -1; + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x20); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x0); + return 0; +} +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h new file mode 100644 index 0000000000000..2504c7db2ad8f --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: MIT */ +/* Platform backend for native direct USB3, not a Type-C policy engine. + * The caller owns the port, has powered the PHY/firmware for native USB3, + * completed DWC3 core init and enabled SUSPHY, and has valid DART mappings. + * Bank 35 is T6050's ADT PHY reg[35]. Bank 44 is usb-drdN reg[3]. + * Only ports explicitly handed over to Linux may use this backend. + */ +#ifndef APPLE_T6050_USB3_PIPE_H +#define APPLE_T6050_USB3_PIPE_H +#include "apple_t6050_usb3_bist.h" + +/* Cold controller setup while RESET_N is clear. Linux's existing ATC + * frontend supplies a dummy PIPE clock before DWC3 reads its core registers. + * T6050 uses the native-RE 6:4 clock field and dummy encoding 0x42. + * This is not an active producer handoff; never use it for USB4 teardown. + */ +static inline int +apple_t6050_usb3_pipe_cold_dummy(const struct apple_t6050_usb_io *io) +{ + unsigned int mux; + if (!io || !io->read || !io->write || !io->delay) + return -1; + if (io->read(io->context, 44, 0x1c) & 1) + return -1; + mux = io->read(io->context, 44, 0x0c) & 0x73; + /* Unset, or already on the dummy producer (1:0 = 2) with any clock: when nothing + * has reset the controller domain since boot, the mux can read 0x22. */ + if (mux != 0 && (mux & 3) != 2) + return -1; + if (mux == 0x42) + return 0; + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 10); + apple_t6050_usb_mask(io, 44, 0x0c, 3, 2); + io->delay(io->context, 10); + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0x40); + io->delay(io->context, 10); + return (io->read(io->context, 44, 0x0c) & 0x73) == 0x42 ? 0 : -1; +} + +/* + * Returns 0, or the failing step: -2 the PIPE was not parked on a known producer, -4..-6 the + * BIST, -7 the mux did not read back USB3, -8 the BIST did not stop. Up to -6 the mux is + * untouched. A lock or unlock that is not acknowledged within 6 ms is not fatal: it is logged + * and the switch carries on. On J716c the lock ACK does not come. + */ +static inline int +apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) +{ + unsigned int aon, mux; + int result = -1, locked = 0, bist = 0; + + if (!io || !io->read || !io->write || !io->poll || !io->delay) + return -1; + aon = io->read(io->context, 44, 0x1c); + mux = io->read(io->context, 44, 0x0c) & 0x73; + /* Never reroute an active USB4 producer or guess an unknown state. */ + if (!(aon & 1) || (mux != 0x42 && mux != 0x10)) + return -2; + + /* Native T8142 xHCI USB3 mode switch, native27 T6050-compatible path. */ + apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); + apple_t6050_usb_mask(io, 44, 0x00, 1, 1); + apple_t6050_usb_mask(io, 44, 0x10, 1, 1); + locked = 1; + io->poll(io->context, 44, 0x14, 1, 1, 6000); + + bist = 1; + result = -4; + if (apple_t6050_usb_bist_on(io)) + goto out; + apple_t6050_usb_mask(io, 44, 0x20, 0x100f, 3); + result = -5; + if (io->poll(io->context, 44, 0x20, 0xf, 3, 1000)) + goto out; + result = -6; + if (apple_t6050_usb_bist_off(io)) + goto out; + bist = 0; + + /* T6050 clock bits are 6:4, not the older 5:3. Native USB3 is 0x10. */ + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 5); + apple_t6050_usb_mask(io, 44, 0x0c, 0x73, 0); + apple_t6050_usb_mask(io, 44, 0x0c, 0x73, 0x10); + result = (io->read(io->context, 44, 0x0c) & 0x73) == 0x10 ? 0 : -7; + +out: + if (bist && apple_t6050_usb_bist_off(io)) + result = -8; + apple_t6050_usb_mask(io, 44, 0x00, 5, 0); + if (locked) { + apple_t6050_usb_mask(io, 44, 0x10, 1, 0); + io->poll(io->context, 44, 0x14, 1, 0, 6000); + } + return result; +} + +/* + * Move a running DWC3 from the dummy producer (0x42) to a USB4 router's USB3 tunnel (0x21: PIPE mode 1, + * clock 2), once the tunnel is active. As for native USB3, the PIPE is locked only when DWC3 is out of + * reset, and a lock that is not acknowledged within 6 ms is not fatal. + * Returns 0, -2 if the PIPE is on another producer, or -7 if the mux did not read back the tunnel. + */ +static inline int +apple_t6050_usb3_pipe_tunnel(const struct apple_t6050_usb_io *io) +{ + unsigned int mux; + int locked; + + if (!io || !io->read || !io->write || !io->poll || !io->delay) + return -1; + mux = io->read(io->context, 44, 0x0c) & 0x73; + if (mux == 0x21) + return 0; + if (mux != 0x42) + return -2; + locked = io->read(io->context, 44, 0x1c) & 1; + + apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); + apple_t6050_usb_mask(io, 44, 0x00, 1, 1); + if (locked) { + apple_t6050_usb_mask(io, 44, 0x10, 1, 1); + io->poll(io->context, 44, 0x14, 1, 1, 6000); + } + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 5); + apple_t6050_usb_mask(io, 44, 0x0c, 3, 1); + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0x20); + apple_t6050_usb_mask(io, 44, 0x00, 1, 0); + apple_t6050_usb_mask(io, 44, 0x00, 4, 0); + if (locked) { + apple_t6050_usb_mask(io, 44, 0x10, 1, 0); + io->poll(io->context, 44, 0x14, 1, 0, 6000); + } + return (io->read(io->context, 44, 0x0c) & 0x73) == 0x21 ? 0 : -7; +} + +/* The native close() call quiesces the ACIO PHY while PIPE is locked, + * before selecting the dummy producer. The caller supplies that lifecycle + * operation; merely changing MUX is not a complete disconnect sequence. + */ +static inline int +apple_t6050_usb3_pipe_stop(const struct apple_t6050_usb_io *io, + int (*quiesce)(void *), void *context) +{ + unsigned int mux; + int result = -1; + + if (!io || !io->read || !io->write || !io->poll || !io->delay || !quiesce) + return -1; + mux = io->read(io->context, 44, 0x0c) & 0x73; + if (mux != 0x10 && mux != 0x21 && mux != 0x42) + return -1; + apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); + apple_t6050_usb_mask(io, 44, 0x00, 1, 1); + apple_t6050_usb_mask(io, 44, 0x00, 4, 4); + apple_t6050_usb_mask(io, 44, 0x10, 1, 1); + /* Native mode0-from2 waits 150 ms, logs a missing lock ACK, and + * continues teardown, as the saved native driver does. + * The caller has already stopped the controller's DMA/PIPE consumer; + * this is not the active-producer handoff used by pipe_switch(). + * The nonselected override, final mux readback and unlock checks below + * remain mandatory, as does successful PHY quiescence. + */ + io->poll(io->context, 44, 0x14, 1, 1, 150000); + if (quiesce(context)) + goto out; + apple_t6050_usb_mask(io, 44, 0x20, 0x100f, 0x1002); + if (io->poll(io->context, 44, 0x20, 0xf, 2, 1000)) + goto out; + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 5); + apple_t6050_usb_mask(io, 44, 0x0c, 3, 2); + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0x40); + result = (io->read(io->context, 44, 0x0c) & 0x73) == 0x42 ? 0 : -1; +out: + apple_t6050_usb_mask(io, 44, 0x00, 1, 0); + apple_t6050_usb_mask(io, 44, 0x00, 4, 0); + apple_t6050_usb_mask(io, 44, 0x10, 1, 0); + if (io->poll(io->context, 44, 0x14, 1, 0, 6000)) + result = -1; + return result; +} +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h new file mode 100644 index 0000000000000..5c96da7d82c2d --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: MIT */ +/* Generated by generate-phy.py; modeled ACKs are not hardware proof. */ +#ifndef APPLE_T6050_USB_PHY_H +#define APPLE_T6050_USB_PHY_H +#include "apple_t6050_usb3_bist.h" +struct apple_t6050_usb_platform { + const struct apple_t6050_usb_io *io; + void *context; + int (*wrapper_power)(void *, unsigned int); + int (*firmware)(void *, unsigned int); + int (*tunable)(void *, unsigned int, const char *); + void (*pause_ns)(void *, unsigned int); +}; +enum apple_t6050_usb_op { T6050_MASK, T6050_POLL, T6050_DELAY, + T6050_POWER, T6050_FIRMWARE, T6050_TUNABLE }; +struct apple_t6050_usb_step { + unsigned int op, bank, offset, mask, value, timeout; + const char *name; +}; +static const struct apple_t6050_usb_step apple_t6050_usb2_host[] = { + { T6050_TUNABLE, 0, 0x0, 0x0, 0x0, 0, "tunable_USB2PHY_DFLT" }, + { T6050_TUNABLE, 0, 0x0, 0x0, 0x0, 0, "tunable_USB2PHY_HOST" }, + { T6050_DELAY, 0, 0x0, 0x0, 0x2710, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x1, 0x0, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x2, 0x0, 0, 0 }, + { T6050_MASK, 1, 0x0, 0x9, 0x9, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x4, 0x4, 0, 0 }, + { T6050_MASK, 0, 0x1c, 0x20000000, 0x0, 0, 0 }, + { T6050_MASK, 0, 0x1c, 0x40000000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1e, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, + { T6050_MASK, 0, 0x0, 0x7, 0x2, 0, 0 }, +}; +static const struct apple_t6050_usb_step apple_t6050_usb3_init_normal[] = { + { T6050_MASK, 39, 0x0, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x1, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x2, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x2, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x0, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x8, 0, 0 }, + { T6050_TUNABLE, 41, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF" }, + { T6050_TUNABLE, 40, 0x0, 0x0, 0x0, 0, "tunable_ATC_FABRIC" }, + { T6050_TUNABLE, 43, 0x0, 0x0, 0x0, 0, "tunable_ATC_COMMON_CFG" }, + { T6050_TUNABLE, 42, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF_LIOA" }, + { T6050_TUNABLE, 20, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 29, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_DFLT" }, + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_DFLT" }, + { T6050_TUNABLE, 5, 0x0, 0x0, 0x0, 0, "tunable_AUS40CMN_SHM" }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_TUNABLE, 10, 0x0, 0x0, 0x0, 0, "tunable_CIO4PLL_CORE" }, + { T6050_MASK, 4, 0x0, 0x3f0, 0x1c0, 0, 0 }, + { T6050_MASK, 4, 0x18, 0xf000, 0x3000, 0, 0 }, + { T6050_MASK, 9, 0x88, 0xff, 0x27, 0, 0 }, + { T6050_MASK, 33, 0x4, 0x38, 0x8, 0, 0 }, + { T6050_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0, 0 }, + { T6050_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 20, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 29, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_MASK, 2, 0x6c, 0x28000fc, 0x44, 0, 0 }, + { T6050_MASK, 2, 0x78, 0x1ffff, 0x110, 0, 0 }, + { T6050_MASK, 2, 0x6c, 0x3, 0x2, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x2a, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x40, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x10000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x100, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x1000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x40, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x400, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_MASK, 38, 0x40, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, + { T6050_POLL, 2, 0x260, 0x400, 0x400, 1000000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x64, 0, 0 }, +}; +static const struct apple_t6050_usb_step apple_t6050_usb3_init_reverse[] = { + { T6050_MASK, 39, 0x0, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x1, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x2, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x2, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x0, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x8, 0, 0 }, + { T6050_TUNABLE, 41, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF" }, + { T6050_TUNABLE, 40, 0x0, 0x0, 0x0, 0, "tunable_ATC_FABRIC" }, + { T6050_TUNABLE, 43, 0x0, 0x0, 0x0, 0, "tunable_ATC_COMMON_CFG" }, + { T6050_TUNABLE, 42, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF_LIOA" }, + { T6050_TUNABLE, 20, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 29, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_DFLT" }, + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_DFLT" }, + { T6050_TUNABLE, 5, 0x0, 0x0, 0x0, 0, "tunable_AUS40CMN_SHM" }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_TUNABLE, 10, 0x0, 0x0, 0x0, 0, "tunable_CIO4PLL_CORE" }, + { T6050_MASK, 4, 0x0, 0x3f0, 0x1c0, 0, 0 }, + { T6050_MASK, 4, 0x18, 0xf000, 0x3000, 0, 0 }, + { T6050_MASK, 9, 0x88, 0xff, 0x27, 0, 0 }, + { T6050_MASK, 33, 0x4, 0x38, 0x8, 0, 0 }, + { T6050_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0, 0 }, + { T6050_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 20, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 29, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_MASK, 2, 0x6c, 0x14000fc, 0x28, 0, 0 }, + { T6050_MASK, 2, 0x78, 0x1ffff, 0x111, 0, 0 }, + { T6050_MASK, 2, 0x6c, 0x3, 0x2, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x2a, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x80, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x20000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x200, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x2000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x80, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x800, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x2, 0x2, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_MASK, 38, 0x40, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, + { T6050_POLL, 2, 0x260, 0x400, 0x400, 1000000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x64, 0, 0 }, +}; +static const struct apple_t6050_usb_step apple_t6050_usb3_shutdown[] = { + { T6050_MASK, 35, 0x4, 0x1, 0x1, 0, 0 }, + { T6050_MASK, 35, 0x10c, 0x100000, 0x0, 0, 0 }, + { T6050_MASK, 35, 0x110, 0x60, 0x20, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x8, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x0, 0, 0 }, + { T6050_POLL, 35, 0x104, 0x100, 0x100, 1000000, 0 }, + { T6050_MASK, 35, 0x10c, 0x1e000000, 0x4000000, 0, 0 }, + { T6050_MASK, 35, 0x110, 0x600, 0x200, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x8, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x0, 0, 0 }, + { T6050_POLL, 35, 0x104, 0x100, 0x100, 1000000, 0 }, + { T6050_MASK, 35, 0x4, 0x1, 0x0, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x100, 0x100, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x10, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x200, 0x200, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x20, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 5, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xc8, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x100, 1000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x4, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, +}; + +static inline int +apple_t6050_usb_phy_run(const struct apple_t6050_usb_platform *p, + const struct apple_t6050_usb_step *steps, unsigned int count) +{ + if (!p || !p->io || !p->io->read || !p->io->write || + !p->io->poll || !p->io->delay || !p->wrapper_power || + !p->firmware || !p->tunable) + return -1; + for (unsigned int i = 0; i < count; i++) { + const struct apple_t6050_usb_step *s = &steps[i]; + int ret = 0; + switch (s->op) { + case T6050_MASK: + apple_t6050_usb_mask(p->io, s->bank, s->offset, s->mask, s->value); + break; + case T6050_POLL: + ret = p->io->poll(p->io->context, s->bank, s->offset, + s->mask, s->value, s->timeout); + break; + case T6050_DELAY: p->io->delay(p->io->context, s->value); break; + case T6050_POWER: ret = p->wrapper_power(p->context, s->value); break; + case T6050_FIRMWARE: ret = p->firmware(p->context, s->value); break; + case T6050_TUNABLE: ret = p->tunable(p->context, s->bank, s->name); break; + default: return -1; + } + if (ret) + return ret; + } + return 0; +} +#define APPLE_T6050_USB_RUN(p, steps) \ + apple_t6050_usb_phy_run((p), (steps), sizeof(steps) / sizeof((steps)[0])) + +/* Native shutdown first asserts PORT_RESET only if it was not already set. + * The 5 ms wait belongs to that transition, not every redundant stop call. + * Preserve IOPause(500 ns), whose unit differs from IODelay(microseconds). + */ +static inline int apple_t6050_usb2_stop(const struct apple_t6050_usb_platform *p) +{ + if (!p || !p->io || !p->io->read || !p->io->write || + !p->io->delay || !p->pause_ns) + return -1; + if (!(p->io->read(p->io->context, 0, 4) & 2)) { + apple_t6050_usb_mask(p->io, 0, 4, 2, 2); + p->io->delay(p->io->context, 5000); + } + apple_t6050_usb_mask(p->io, 0, 0, 7, 4); + apple_t6050_usb_mask(p->io, 0, 4, 8, 8); + apple_t6050_usb_mask(p->io, 0, 4, 1, 1); + apple_t6050_usb_mask(p->io, 0, 0x1c, 0x20000000, 0x20000000); + apple_t6050_usb_mask(p->io, 0, 0x1c, 0x40000000, 0x40000000); + p->pause_ns(p->context, 500); + return 0; +} +#endif diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index 7688cfd4ed60d..a2e1422e05bd8 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -8,10 +8,12 @@ #include #include #include +#include #include #include #include #include +#include struct macsmc_reboot_nvmem { struct nvmem_cell *shutdown_flag; @@ -33,10 +35,39 @@ enum boot_stage { BOOT_STAGE_KERNEL_STARTED = 0x30, /* Normal OS booting */ }; +/* + * The PMU's power-on policy byte: bit 0 set keeps the machine off when power returns after a + * loss, bit 5 set keeps it off when power is connected (bit 5 clear powers it on at connect + * whatever bit 0 says). Changes are read-modify-write: the other bits are the PMU's. + */ +#define PM_SETTING_NO_RESTORE BIT(0) +#define PM_SETTING_NO_POWER_ON_CONNECT BIT(5) +#define PM_SETTING_AC_POWER_MASK (PM_SETTING_NO_RESTORE | PM_SETTING_NO_POWER_ON_CONNECT) + +enum ac_power_mode { + AC_POWER_OFF, /* stay off */ + AC_POWER_RESTORE, /* power on after a power loss if it was on */ + AC_POWER_ALWAYS, /* power on whenever power is connected */ +}; + +static const char * const ac_power_modes[] = { + [AC_POWER_OFF] = "off", + [AC_POWER_RESTORE] = "restore", + [AC_POWER_ALWAYS] = "always", +}; + +static const u8 ac_power_mode_bits[] = { + [AC_POWER_OFF] = PM_SETTING_NO_RESTORE | PM_SETTING_NO_POWER_ON_CONNECT, + [AC_POWER_RESTORE] = PM_SETTING_NO_POWER_ON_CONNECT, + [AC_POWER_ALWAYS] = PM_SETTING_NO_RESTORE, +}; + struct macsmc_reboot { struct device *dev; struct apple_smc *smc; struct notifier_block reboot_notify; + struct nvmem_cell *pm_setting; + struct mutex pm_setting_lock; union { struct macsmc_reboot_nvmem nvm; @@ -198,6 +229,80 @@ static void macsmc_power_init_error_counts(struct macsmc_reboot *reboot) dev_err(reboot->dev, "Failed to reset boot_error_count\n"); } +static enum ac_power_mode macsmc_ac_power_mode(u8 val) +{ + if (!(val & PM_SETTING_NO_POWER_ON_CONNECT)) + return AC_POWER_ALWAYS; + if (val & PM_SETTING_NO_RESTORE) + return AC_POWER_OFF; + return AC_POWER_RESTORE; +} + +static ssize_t ac_power_mode_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(dev); + enum ac_power_mode mode; + int val, i, len = 0; + + val = nvmem_cell_get_u8(reboot->pm_setting); + if (val < 0) + return val; + mode = macsmc_ac_power_mode(val); + + for (i = 0; i < ARRAY_SIZE(ac_power_modes); i++) + len += sysfs_emit_at(buf, len, i == mode ? "[%s] " : "%s ", ac_power_modes[i]); + buf[len - 1] = '\n'; + + return len; +} + +static ssize_t ac_power_mode_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t n) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(dev); + int mode, val, ret; + u8 new; + + mode = sysfs_match_string(ac_power_modes, buf); + if (mode < 0) + return mode; + + mutex_lock(&reboot->pm_setting_lock); + val = nvmem_cell_get_u8(reboot->pm_setting); + if (val < 0) { + ret = val; + goto out; + } + new = (val & ~PM_SETTING_AC_POWER_MASK) | ac_power_mode_bits[mode]; + ret = new == val ? 0 : nvmem_cell_set_u8(reboot->pm_setting, new); + if (!ret && new != val) + dev_info(dev, "PM setting %02x => %02x (ac_power_mode %s)\n", val, new, + ac_power_modes[mode]); +out: + mutex_unlock(&reboot->pm_setting_lock); + + return ret < 0 ? ret : n; +} +static DEVICE_ATTR_RW(ac_power_mode); + +static struct attribute *macsmc_reboot_attrs[] = { + &dev_attr_ac_power_mode.attr, + NULL +}; + +static umode_t macsmc_reboot_attr_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(kobj_to_dev(kobj)); + + return reboot && reboot->pm_setting ? attr->mode : 0; +} + +static const struct attribute_group macsmc_reboot_group = { + .attrs = macsmc_reboot_attrs, + .is_visible = macsmc_reboot_attr_visible, +}; +__ATTRIBUTE_GROUPS(macsmc_reboot); + static int macsmc_reboot_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); @@ -237,6 +342,18 @@ static int macsmc_reboot_probe(struct platform_device *pdev) reboot->nvm_cells[i] = cell; } + /* Optional: the power-on policy byte, for ac_power_mode */ + mutex_init(&reboot->pm_setting_lock); + reboot->pm_setting = devm_nvmem_cell_get(&pdev->dev, "pm_setting"); + if (IS_ERR(reboot->pm_setting)) { + ret = PTR_ERR(reboot->pm_setting); + if (ret == -EPROBE_DEFER) + return ret; + if (ret != -ENOENT) + dev_warn(&pdev->dev, "Missing NVMEM cell pm_setting (%d)\n", ret); + reboot->pm_setting = NULL; + } + /* Set the boot_stage to indicate we're running the OS kernel */ if (reboot->nvm.boot_stage && nvmem_cell_set_u8(reboot->nvm.boot_stage, BOOT_STAGE_KERNEL_STARTED) < 0) @@ -287,6 +404,7 @@ static struct platform_driver macsmc_reboot_driver = { .driver = { .name = "macsmc-reboot", .of_match_table = macsmc_reboot_of_table, + .dev_groups = macsmc_reboot_groups, }, .probe = macsmc_reboot_probe, }; diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 1bc23703654a4..1e306c7bb3f6c 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -960,10 +960,34 @@ static int macsmc_power_event(struct notifier_block *nb, unsigned long event, vo return NOTIFY_DONE; } +static int macsmc_read_identity(struct apple_smc *smc, smc_key key, + char *out, size_t capacity) +{ + struct apple_smc_key_info info; + int ret; + + if (!capacity) + return -EINVAL; + out[0] = '\0'; + ret = apple_smc_get_key_info(smc, key, &info); + if (ret < 0) + return ret; + if (!info.size || info.size >= capacity || !(info.flags & APPLE_SMC_READABLE)) + return -EINVAL; + /* SMC read requests must describe this key, not the destination buffer. */ + ret = apple_smc_read(smc, key, out, info.size); + if (ret != info.size) { + out[0] = '\0'; + return ret < 0 ? ret : -EIO; + } + out[info.size] = '\0'; + return 0; +} + static int macsmc_power_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct apple_smc *smc; struct power_supply_config psy_cfg = {}; struct apple_smc_key_info info; struct macsmc_power *power; @@ -976,6 +1000,9 @@ static int macsmc_power_probe(struct platform_device *pdev) enum power_supply_property *props; size_t nprops; + if (!dev->parent) + return dev_err_probe(dev, -ENODEV, "Missing SMC parent\n"); + smc = dev_get_drvdata(dev->parent); if (!smc) return -ENODEV; @@ -1121,12 +1148,12 @@ static int macsmc_power_probe(struct platform_device *pdev) power->batt_desc.num_properties = nprops; /* Fetch identity strings */ - apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, - sizeof(power->model_name) - 1); - apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, - sizeof(power->serial_number) - 1); - apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, - sizeof(power->mfg_date) - 1); + macsmc_read_identity(smc, SMC_KEY(BMDN), power->model_name, + sizeof(power->model_name)); + macsmc_read_identity(smc, SMC_KEY(BMSN), power->serial_number, + sizeof(power->serial_number)); + macsmc_read_identity(smc, SMC_KEY(BMDT), power->mfg_date, + sizeof(power->mfg_date)); apple_smc_read_u8(power->smc, SMC_KEY(BNCB), &power->num_cells); power->nominal_voltage_mv = MACSMC_NOMINAL_CELL_VOLTAGE_MV * power->num_cells; diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index af299473a6c87..e5755ebb39aff 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -37,7 +37,12 @@ * separate 16 KiB page at 0x350780000, so the register is 0x350780008. The * device tree describes that page as its own syscon. t8122 is unresolved. * - * On t8142, RegMap 0x62 is a separate MMIO region at 0x380780000. + * On t8142, RegMap 0x62 is a separate MMIO region at 0x380780000. On t6050 + * the register is at +0x60 in a page of the die's PMGR block (0x280620000 + * on die 0). There the odd bit is a done flag rather than a busy flag: it + * is set once the request is taken, then request and done clear by + * themselves within a few ms. Only after that is the port's CPU wrapper + * accessible. * The t8132 layout has not been decoded. * * Reference firmware polls the busy bits, stores the request, then polls @@ -56,6 +61,8 @@ #define APPLE_CIO_RESET_POLL_US 100 #define APPLE_CIO_RESET_TIMEOUT_US 100000 +/* A t6050 reconfiguration takes a few ms; allow ample margin. */ +#define APPLE_CIO_RESET_T6050_TIMEOUT_US 192000 #define T8103_CIO_CTRL_STRIDE 0x4000 #define T8103_CIO_CTRL_INIT_REQ BIT(0) @@ -75,6 +82,7 @@ struct apple_cio_reset; struct apple_cio_reset_variant { unsigned int nr_resets; bool pmgr_child; + u32 ctrl; /* request register, t8142 layout */ int (*deassert)(struct apple_cio_reset *priv, unsigned long id); }; @@ -174,19 +182,19 @@ static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) guard(mutex)(&priv->lock); /* Serialize all four ports sharing the request register. */ - ret = regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + ret = regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, !(val & T8142_CIO_CTRL_BUSY_MASK), APPLE_CIO_RESET_POLL_US, APPLE_CIO_RESET_TIMEOUT_US); if (ret) return ret; - ret = regmap_write(priv->regmap, T8142_CIO_CTRL, + ret = regmap_write(priv->regmap, priv->variant->ctrl, T8142_CIO_CTRL_INIT_REQ(id)); if (ret) return ret; - return regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + return regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, !(val & T8142_CIO_CTRL_INIT_BUSY(id)), APPLE_CIO_RESET_POLL_US, APPLE_CIO_RESET_TIMEOUT_US); @@ -194,9 +202,54 @@ static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) static const struct apple_cio_reset_variant apple_t8142_cio_reset = { .nr_resets = 4, + .ctrl = T8142_CIO_CTRL, .deassert = t8142_cio_deassert, }; +static int t6050_cio_deassert(struct apple_cio_reset *priv, unsigned long id) +{ + u32 mask = T8142_CIO_CTRL_INIT_REQ(id) | T8142_CIO_CTRL_INIT_BUSY(id); + u32 val; + int ret; + + guard(mutex)(&priv->lock); + + /* Let a reconfiguration still under way finish first. */ + ret = regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, + !(val & mask), APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_T6050_TIMEOUT_US); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, priv->variant->ctrl, + T8142_CIO_CTRL_INIT_REQ(id)); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, + val & T8142_CIO_CTRL_INIT_BUSY(id), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_T6050_TIMEOUT_US); + if (ret) + return ret; + + /* + * The CPU wrapper is usable only once the reconfiguration is over, when + * the request and done bits clear again, a few ms later. An access + * before that takes an SError. + */ + return regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, + !(val & mask), APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_T6050_TIMEOUT_US); +} + +/* The t8142 bit layout in a register mapped on its own, with a done flag. */ +static const struct apple_cio_reset_variant apple_t6050_cio_reset = { + .nr_resets = 4, + .ctrl = 0x60, + .deassert = t6050_cio_deassert, +}; + static const struct apple_cio_reset_variant apple_t8103_cio_reset = { .nr_resets = 2, .deassert = t8103_cio_deassert, @@ -277,6 +330,10 @@ static int apple_cio_reset_probe(struct platform_device *pdev) } static const struct of_device_id apple_cio_reset_match[] = { + { + .compatible = "apple,t6050-cio-reset", + .data = &apple_t6050_cio_reset, + }, { .compatible = "apple,t8142-cio-reset", .data = &apple_t8142_cio_reset, diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index c13c03f9d92ab..999dec63b3b29 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -144,6 +144,11 @@ static bool ascwrap_v8_rx_irq; module_param(ascwrap_v8_rx_irq, bool, 0444); MODULE_PARM_DESC(ascwrap_v8_rx_irq, "Service T8152 GPU ASCWrap v8 receive with the recv-not-empty interrupt (default: poll, no GPU mailbox interrupt is enabled)"); +/* Physical T6050 GPU windows, shared by J775D and J716c. */ +#define APPLE_J775D_GFX_WRAPPER_BASE 0x2302600000ULL +#define APPLE_J775D_GFX_IOP_VBAR_BASE 0x2302050000ULL +#define APPLE_J775D_GFX1_WRAPPER_BASE 0x2302e00000ULL +#define APPLE_J775D_GFX1_IOP_VBAR_BASE 0x2302850000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -1636,8 +1641,7 @@ static const struct apple_mbox_hw apple_mbox_t8152_gfx_ascwrap_v8_hw = { static_assert(0x20 + sizeof(u32) <= APPLE_ASCWRAP_V8_MBOX_SIZE); -/* T6050's native ASCWrap v6 uses the same RUN/stop sequence. Aura grants - * these complete wrapper/VBAR windows in the retained GPU aperture. */ +/* Retained T6050 wrapper/VBAR alias profile. */ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, APPLE_T8140_GFX_WRAPPER_BASE + APPLE_T6050_ASCWRAP_BASE_DELTA, @@ -1647,6 +1651,17 @@ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_ascwrap_v6_ APPLE_T8140_GFX1_IOP_VBAR_BASE + APPLE_T6050_ASCWRAP_BASE_DELTA }, }; +/* + * T6050 at its physical addresses, with both roles in the GPU aperture at + * 0x2300000000 on die 0. The other T6050 profiles retain their alias windows. + */ +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_j775d_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_J775D_GFX_WRAPPER_BASE, APPLE_J775D_GFX_IOP_VBAR_BASE }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_J775D_GFX1_WRAPPER_BASE, APPLE_J775D_GFX1_IOP_VBAR_BASE }, +}; + static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_apif_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, APPLE_T8140_GFX_WRAPPER_BASE, @@ -1672,6 +1687,22 @@ static const struct apple_mbox_hw apple_mbox_t6050_ascwrap_v6_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t6050_j775d_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t6050_j775d_ascwrap_v6_roles), + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + .has_irq_controls = false, +}; + /* Retained compatibility profile: unshifted wrapper aliases, native VBAR. */ static const struct apple_mbox_hw apple_mbox_t6050_apif_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, @@ -1717,6 +1748,8 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, + { .compatible = "apple,t6050-j775d-ascwrap-v6", + .data = &apple_mbox_t6050_j775d_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c index 5defa12862b15..2ec4bd9569e46 100644 --- a/drivers/soc/apple/rtkit-crashlog.c +++ b/drivers/soc/apple/rtkit-crashlog.c @@ -3,6 +3,8 @@ * Apple RTKit IPC library * Copyright (C) The Asahi Linux Contributors */ +#include + #include "rtkit-internal.h" #define FOURCC(a, b, c, d) \ @@ -13,6 +15,51 @@ #define APPLE_RTKIT_CRASHLOG_MBOX FOURCC('C', 'm', 'b', 'x') #define APPLE_RTKIT_CRASHLOG_TIME FOURCC('C', 't', 'i', 'm') #define APPLE_RTKIT_CRASHLOG_REGS FOURCC('C', 'r', 'g', '8') +#define APPLE_RTKIT_CRASHLOG_REGS_CL4 FOURCC('C', 'r', 'l', '4') + +/* + * Newer firmware writes a container instead of a single crashlog: a + * 'CLGH' header and a table of records, each with an ID and an offset from + * the container. The record whose ID is the container's current ID holds + * the log of this crash, and may itself be a container (on T6050's DCP: + * per-coprocessor, then per-task). The log at the end is an ordinary + * 'CLHE' crashlog with the same sections. + */ +struct apple_rtkit_crashlog_container { + u32 fourcc; + u32 version; + u32 size; + u32 current_id; + u32 count; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_container) == 0x14); + +struct apple_rtkit_crashlog_record { + u64 id; + u64 _unk0; + u32 offset; + u32 _unk1; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_record) == 0x18); + +#define APPLE_RTKIT_CRASHLOG_CONTAINER_VERSION 2 +#define APPLE_RTKIT_CRASHLOG_CONTAINER_MAX_DEPTH 8 +/* Set in a log's flags while the co-processor is still writing it. */ +#define APPLE_RTKIT_CRASHLOG_INCOMPLETE BIT(8) + +/* Register frame of the 'Crl4' section, offsets from the section start. */ +#define APPLE_RTKIT_CRASHLOG_CL4_SECTION_SIZE 0x378 +#define APPLE_RTKIT_CRASHLOG_CL4_FLAGS 0x10 +#define APPLE_RTKIT_CRASHLOG_CL4_SIZE 0x14 +#define APPLE_RTKIT_CRASHLOG_CL4_NOMINAL_SIZE 0xd8 +#define APPLE_RTKIT_CRASHLOG_CL4_REGS 0x18 +#define APPLE_RTKIT_CRASHLOG_CL4_LR 0x108 +#define APPLE_RTKIT_CRASHLOG_CL4_PC 0x110 +#define APPLE_RTKIT_CRASHLOG_CL4_SP 0x118 +#define APPLE_RTKIT_CRASHLOG_CL4_PSR 0x120 +#define APPLE_RTKIT_CRASHLOG_CL4_ESR 0x358 +#define APPLE_RTKIT_CRASHLOG_CL4_FAR 0x360 +#define APPLE_RTKIT_CRASHLOG_CL4_UNDECODABLE BIT(16) /* For COMPILE_TEST on non-ARM64 architectures */ #ifndef PSR_MODE_EL0t @@ -71,8 +118,8 @@ static void apple_rtkit_crashlog_dump_str(struct apple_rtkit *rtk, u8 *bfr, *newline = tmp; ptr = newline + 1; } else { - dev_warn(rtk->dev, "RTKit: Message (id=%x): %s", idx, - ptr); + dev_warn(rtk->dev, "RTKit: Message (id=%x): %.*s", idx, + (int)(end - ptr), ptr); break; } } @@ -81,7 +128,8 @@ static void apple_rtkit_crashlog_dump_str(struct apple_rtkit *rtk, u8 *bfr, static void apple_rtkit_crashlog_dump_version(struct apple_rtkit *rtk, u8 *bfr, size_t size) { - dev_warn(rtk->dev, "RTKit: Version: %s", bfr + 16); + /* The string need not be terminated inside the section. */ + dev_warn(rtk->dev, "RTKit: Version: %.*s", (int)(size - 16), bfr + 16); } static void apple_rtkit_crashlog_dump_time(struct apple_rtkit *rtk, u8 *bfr, @@ -100,6 +148,10 @@ static void apple_rtkit_crashlog_dump_mailbox(struct apple_rtkit *rtk, u8 *bfr, size_t n_messages; struct apple_rtkit_crashlog_mbox_entry entry; + if (size < 28) { + dev_warn(rtk->dev, "RTKit: Mailbox section too small: 0x%zx", size); + return; + } memcpy(&type, bfr + 16, 4); memcpy(&index, bfr + 24, 4); n_messages = (size - 28) / sizeof(entry); @@ -173,13 +225,150 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr, dev_warn(rtk->dev, "\n"); } +static void apple_rtkit_crashlog_dump_regs_cl4(struct apple_rtkit *rtk, + u8 *section, size_t size) +{ + u32 flags, regs_size; + int i; + + if (size < APPLE_RTKIT_CRASHLOG_CL4_SECTION_SIZE) { + dev_warn(rtk->dev, "RTKit: Crl4 section too small: 0x%zx", size); + return; + } + flags = get_unaligned_le32(section + APPLE_RTKIT_CRASHLOG_CL4_FLAGS); + regs_size = get_unaligned_le32(section + APPLE_RTKIT_CRASHLOG_CL4_SIZE); + if (regs_size != APPLE_RTKIT_CRASHLOG_CL4_NOMINAL_SIZE || + (flags & APPLE_RTKIT_CRASHLOG_CL4_UNDECODABLE)) { + dev_warn(rtk->dev, "RTKit: Crl4 frame not decodable (flags %#x, size %#x)", + flags, regs_size); + return; + } + + dev_warn(rtk->dev, "RTKit: Exception dump (Crl4):"); + dev_warn(rtk->dev, " PSR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_PSR)); + dev_warn(rtk->dev, " PC = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_PC)); + dev_warn(rtk->dev, " LR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_LR)); + dev_warn(rtk->dev, " ESR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_ESR)); + dev_warn(rtk->dev, " FAR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_FAR)); + dev_warn(rtk->dev, " SP = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_SP)); + for (i = 0; i < 28; i += 4) { + u8 *r = section + APPLE_RTKIT_CRASHLOG_CL4_REGS + i * 8; + + dev_warn(rtk->dev, " x%02d-x%02d = %016llx %016llx %016llx %016llx", + i, i + 3, get_unaligned_le64(r), get_unaligned_le64(r + 8), + get_unaligned_le64(r + 16), get_unaligned_le64(r + 24)); + } +} + +/** + * apple_rtkit_crashlog_locate() - find this crash's log in a crashlog buffer + * @bfr: a copy of the buffer + * @size: its size + * @log_offset: set to the offset of the 'CLHE' log + * @log_id: set to the ID of the record that held it (0 for a bare log) + * @depth: set to the number of containers followed + * + * A bare 'CLHE' log at offset 0 is returned as it is. In a 'CLGH' container, + * the record with the current ID is followed, through nested containers, + * to its log. Offsets only move forward, so a corrupt table cannot loop. + * + * Return: 0 if a complete log was found, -EAGAIN if the selected log is + * still being written, -EINVAL otherwise. + */ +int apple_rtkit_crashlog_locate(const u8 *bfr, size_t size, size_t *log_offset, + u32 *log_id, unsigned int *depth) +{ + struct apple_rtkit_crashlog_container container; + struct apple_rtkit_crashlog_header header; + struct apple_rtkit_crashlog_record record; + size_t offset = 0, avail, next; + unsigned int level; + u32 id = 0, i; + + for (level = 0; level <= APPLE_RTKIT_CRASHLOG_CONTAINER_MAX_DEPTH; level++) { + avail = size - offset; + if (avail < sizeof(header)) + return -EINVAL; + memcpy(&header, bfr + offset, sizeof(header)); + + if (header.fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { + if (header.size < sizeof(header) || header.size > avail) + return -EINVAL; + if (level && (header.flags & APPLE_RTKIT_CRASHLOG_INCOMPLETE)) + return -EAGAIN; + *log_offset = offset; + *log_id = id; + *depth = level; + return 0; + } + if (header.fourcc != APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC) + return -EINVAL; + + memcpy(&container, bfr + offset, sizeof(container)); + if (((container.version >> 8) & 0xff) != + APPLE_RTKIT_CRASHLOG_CONTAINER_VERSION || + container.size > avail || !container.count || + container.count > (avail - sizeof(container)) / sizeof(record)) + return -EINVAL; + + next = 0; + for (i = 0; i < container.count; i++) { + memcpy(&record, bfr + offset + sizeof(container) + + i * sizeof(record), sizeof(record)); + if (record.id == container.current_id) { + next = record.offset; + break; + } + } + if (next < sizeof(container) || next >= avail) + return -EINVAL; + + id = container.current_id; + offset += next; + } + + return -EINVAL; +} + void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) { size_t offset; u32 section_fourcc, section_size; struct apple_rtkit_crashlog_header header; + if (size < sizeof(header)) { + dev_warn(rtk->dev, "RTKit: Crashlog buffer too small: 0x%zx", + size); + return; + } memcpy(&header, bfr, sizeof(header)); + if (header.fourcc == APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC) { + unsigned int depth; + u32 id; + int ret; + + ret = apple_rtkit_crashlog_locate(bfr, size, &offset, &id, &depth); + if (ret) { + dev_warn(rtk->dev, + "RTKit: Crashlog container holds no complete log (%d)", + ret); + return; + } + dev_warn(rtk->dev, + "RTKit: Crashlog container: log '%c%c%c%c' at 0x%zx (depth %u)", + (id >> 24) & 0xff, (id >> 16) & 0xff, (id >> 8) & 0xff, + id & 0xff, offset, depth); + bfr += offset; + size -= offset; + memcpy(&header, bfr, sizeof(header)); + } + if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { dev_warn(rtk->dev, "RTKit: Expected crashlog header but got %x", header.fourcc); @@ -196,13 +385,35 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) offset = sizeof(header); while (offset < size) { + if (size - offset < 16) { + dev_warn(rtk->dev, + "RTKit: Crashlog section header at 0x%zx is truncated", + offset); + return; + } memcpy(§ion_fourcc, bfr + offset, 4); memcpy(§ion_size, bfr + offset + 12, 4); - switch (section_fourcc) { - case APPLE_RTKIT_CRASHLOG_HEADER_FOURCC: + if (section_fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { dev_dbg(rtk->dev, "RTKit: End of crashlog reached"); return; + } + /* A zero size would never advance. */ + if (section_size < 16 || section_size > size - offset) { + dev_warn(rtk->dev, + "RTKit: Crashlog section %x at 0x%zx has a bad size 0x%x", + section_fourcc, offset, section_size); + return; + } + /* + * The decoders read up to section_size bytes from 16 bytes into + * the section, so a section is only decoded with at least the + * footer's 16-byte header after it. + */ + if (section_size > size - offset - 16) + break; + + switch (section_fourcc) { case APPLE_RTKIT_CRASHLOG_STR: apple_rtkit_crashlog_dump_str(rtk, bfr + offset + 16, section_size); @@ -223,6 +434,10 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) apple_rtkit_crashlog_dump_regs(rtk, bfr + offset + 16, section_size); break; + case APPLE_RTKIT_CRASHLOG_REGS_CL4: + apple_rtkit_crashlog_dump_regs_cl4(rtk, bfr + offset, + section_size); + break; default: dev_warn(rtk->dev, "RTKit: Unknown crashlog section: %x", diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 2ad4d96bec768..b67ef42a2b6d0 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -23,6 +23,8 @@ #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 #define APPLE_RTKIT_CRASHLOG_HEADER_FOURCC 0x434c4845 +/* A container of crashlogs (one per task or core): 'CLGH'. */ +#define APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC 0x434c4748 struct apple_rtkit_crashlog_header { u32 fourcc; @@ -84,9 +86,16 @@ struct apple_rtkit { bool crashlog_inherited; bool oslog_inherited; + /* A crash reported before the crashlog buffer was mapped: the log is + * read once the co-processor has written it. */ + struct delayed_work crashlog_late_work; + unsigned int crashlog_late_polls; + struct workqueue_struct *wq; }; void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size); +int apple_rtkit_crashlog_locate(const u8 *bfr, size_t size, size_t *log_offset, + u32 *log_id, unsigned int *depth); #endif diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3b821415d54d8..8307b9f3505bb 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -272,6 +272,9 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) + dev_info(rtk->dev, "RTKit: crashlog endpoint not in the endpoint map\n"); + for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) { switch (ep) { /* the management endpoint is started by default */ @@ -720,11 +723,82 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, len, DMA_FROM_DEVICE); } +#define APPLE_RTKIT_CRASHLOG_LATE_POLL_MS 250 +#define APPLE_RTKIT_CRASHLOG_LATE_POLLS 20 + +/* + * Whether the crashlog buffer holds a log: an ordinary 'CLHE' log at its + * start, or a 'CLGH' container whose current record leads to a complete one. + */ +static bool apple_rtkit_crashlog_header_valid(struct apple_rtkit *rtk) +{ + struct apple_rtkit_crashlog_header header; + size_t size = rtk->crashlog_buffer.size, offset; + unsigned int depth; + u32 id; + + if (size < sizeof(header)) + return false; + apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, + sizeof(header)); + if (header.fourcc == APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC) { + u8 *bfr __free(kfree) = kmalloc(size, GFP_KERNEL); + + if (!bfr) + return false; + apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0, size); + return !apple_rtkit_crashlog_locate(bfr, size, &offset, &id, + &depth); + } + return header.fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC && + header.size >= sizeof(header) && + header.size <= size; +} + +static void apple_rtkit_crashlog_late(struct work_struct *work) +{ + struct apple_rtkit *rtk = container_of(to_delayed_work(work), + struct apple_rtkit, + crashlog_late_work); + u8 *bfr __free(kfree) = NULL; + + if (!apple_rtkit_crashlog_header_valid(rtk)) { + if (++rtk->crashlog_late_polls < APPLE_RTKIT_CRASHLOG_LATE_POLLS) + queue_delayed_work(rtk->wq, &rtk->crashlog_late_work, + msecs_to_jiffies(APPLE_RTKIT_CRASHLOG_LATE_POLL_MS)); + else + dev_err(rtk->dev, + "RTKit: crash reported, but no crashlog was written in %u ms\n", + APPLE_RTKIT_CRASHLOG_LATE_POLLS * + APPLE_RTKIT_CRASHLOG_LATE_POLL_MS); + return; + } + + /* The crash was reported and the client told; only dump the log. */ + dev_err(rtk->dev, "RTKit: co-processor crashed; crashlog written after %u ms\n", + (rtk->crashlog_late_polls + 1) * APPLE_RTKIT_CRASHLOG_LATE_POLL_MS); + bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL); + if (!bfr) + return; + apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0, + rtk->crashlog_buffer.size); + apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size); +} + static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); u8 *bfr __free(kfree) = NULL; - struct apple_rtkit_crashlog_header header; + /* + * An undiscovered crashlog endpoint was never started, so the + * co-processor was never asked for a buffer and, as in an inherited + * session, a message there can only be a crash report. Firmware that + * leaves this endpoint out of its map still reports crashes on it, + * with a buffer of its own. + */ + bool reports_crash = rtk->crashlog_inherited || + !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints); + bool await_log = false; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -735,26 +809,23 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) - goto unmapped; + goto no_buffer; /* * In a session that went through the handshake, the first - * message on this endpoint is the buffer request. Firmware may - * place that buffer in memory which still holds the log of an - * earlier crash, so only look for one in an inherited session. + * message on a started crashlog endpoint is the buffer request. + * Firmware may place that buffer in memory which still holds + * the log of an earlier crash, so only look for one when the + * message can only be a crash report. */ - if (!rtk->crashlog_inherited) + if (!reports_crash) return; } - if (rtk->crashlog_inherited) { - if (rtk->crashlog_buffer.size < sizeof(header)) - goto unmapped; - - apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, - sizeof(header)); - if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC || - header.size < sizeof(header) || - header.size > rtk->crashlog_buffer.size) - goto unmapped; + if (reports_crash) { + if (rtk->crashlog_buffer.size < + sizeof(struct apple_rtkit_crashlog_header)) + goto no_buffer; + if (!apple_rtkit_crashlog_header_valid(rtk)) + goto no_log; } dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); @@ -791,14 +862,34 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); return; -unmapped: - /* An inherited session already negotiated its crash buffer. */ - if (!rtk->crashlog_inherited) +no_buffer: + /* Otherwise this was a buffer request that could not be served. */ + if (!reports_crash) return; - dev_err(rtk->dev, "RTKit: adopted co-processor crashed without a readable log\n"); + dev_err(rtk->dev, "RTKit: co-processor crashed without a usable crashlog buffer\n"); + goto crashed; + +no_log: + /* The buffer is mapped, but holds no valid log (yet). */ + dev_err(rtk->dev, "RTKit: co-processor crashed without a readable log\n"); + /* + * A co-processor whose crashlog endpoint was never discovered or + * started (the external DCP on T6050) requests the buffer only when + * it crashes, and writes the log after receiving it. Read the log once + * it is there, rather than dropping it. + */ + await_log = !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints); + +crashed: rtk->crashed = true; if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, NULL, 0); + + if (await_log) { + rtk->crashlog_late_polls = 0; + queue_delayed_work(rtk->wq, &rtk->crashlog_late_work, + msecs_to_jiffies(APPLE_RTKIT_CRASHLOG_LATE_POLL_MS)); + } } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) @@ -1316,6 +1407,9 @@ static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) static void apple_rtkit_stop_rx(struct apple_rtkit *rtk) { apple_rtkit_detach_rx(rtk); + /* A crashlog message still being handled may queue the late read. */ + flush_workqueue(rtk->wq); + cancel_delayed_work_sync(&rtk->crashlog_late_work); destroy_workqueue(rtk->wq); } @@ -1420,6 +1514,7 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, init_completion(&rtk->coreanalytics_completion); init_completion(&rtk->iop_pwr_ack_completion); init_completion(&rtk->ap_pwr_ack_completion); + INIT_DELAYED_WORK(&rtk->crashlog_late_work, apple_rtkit_crashlog_late); bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); @@ -1559,6 +1654,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) /* make sure we don't handle any messages while reinitializing */ apple_rtkit_detach_rx(rtk); flush_workqueue(rtk->wq); + cancel_delayed_work_sync(&rtk->crashlog_late_work); apple_rtkit_free_buffer(rtk, &rtk->coreanalytics_buffer); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 6cedcb867c61b..8270a5a731f2c 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -54,15 +55,19 @@ /* SARTv3 registers */ #define APPLE_SART3_CONFIG(idx) (0x00 + 4 * (idx)) -#define APPLE_SART3_PADDR(idx) (0x40 + 4 * (idx)) +#define APPLE_SART3_PADDR 0x40 #define APPLE_SART3_PADDR_SHIFT 12 -#define APPLE_SART3_SIZE(idx) (0x80 + 4 * (idx)) +#define APPLE_SART3_SIZE 0x80 #define APPLE_SART3_SIZE_SHIFT 12 #define APPLE_SART3_SIZE_MAX GENMASK(29, 0) #define APPLE_SART3_FLAGS_ALLOW 0xff +/* SARTv4 (M5): SARTv3's entries, with the address and size arrays moved up */ +#define APPLE_SART4_PADDR 0x60 +#define APPLE_SART4_SIZE 0xc0 + struct apple_sart_ops { void (*get_entry)(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size); @@ -70,6 +75,9 @@ struct apple_sart_ops { phys_addr_t paddr_shifted, size_t size_shifted); /* This is probably a bitfield but the exact meaning of each bit is unknown. */ unsigned int flags_allow; + /* SARTv3/v4: where the address and size arrays start */ + unsigned int paddr_reg; + unsigned int size_reg; unsigned int size_shift; unsigned int paddr_shift; size_t size_max; @@ -179,8 +187,8 @@ static struct apple_sart_ops sart_ops_v2 = { static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size) { - phys_addr_t paddr_ = readl(sart->regs + APPLE_SART3_PADDR(index)); - size_t size_ = readl(sart->regs + APPLE_SART3_SIZE(index)); + phys_addr_t paddr_ = readl(sart->regs + sart->ops->paddr_reg + 4 * index); + size_t size_ = readl(sart->regs + sart->ops->size_reg + 4 * index); *flags = readl(sart->regs + APPLE_SART3_CONFIG(index)); *size = size_ << APPLE_SART3_SIZE_SHIFT; @@ -190,8 +198,8 @@ static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, static void sart3_set_entry(struct apple_sart *sart, int index, u8 flags, phys_addr_t paddr_shifted, size_t size_shifted) { - writel(paddr_shifted, sart->regs + APPLE_SART3_PADDR(index)); - writel(size_shifted, sart->regs + APPLE_SART3_SIZE(index)); + writel(paddr_shifted, sart->regs + sart->ops->paddr_reg + 4 * index); + writel(size_shifted, sart->regs + sart->ops->size_reg + 4 * index); writel(flags, sart->regs + APPLE_SART3_CONFIG(index)); } @@ -199,6 +207,19 @@ static struct apple_sart_ops sart_ops_v3 = { .get_entry = sart3_get_entry, .set_entry = sart3_set_entry, .flags_allow = APPLE_SART3_FLAGS_ALLOW, + .paddr_reg = APPLE_SART3_PADDR, + .size_reg = APPLE_SART3_SIZE, + .size_shift = APPLE_SART3_SIZE_SHIFT, + .paddr_shift = APPLE_SART3_PADDR_SHIFT, + .size_max = APPLE_SART3_SIZE_MAX, +}; + +static struct apple_sart_ops sart_ops_v4 = { + .get_entry = sart3_get_entry, + .set_entry = sart3_set_entry, + .flags_allow = APPLE_SART3_FLAGS_ALLOW, + .paddr_reg = APPLE_SART4_PADDR, + .size_reg = APPLE_SART4_SIZE, .size_shift = APPLE_SART3_SIZE_SHIFT, .paddr_shift = APPLE_SART3_PADDR_SHIFT, .size_max = APPLE_SART3_SIZE_MAX, @@ -236,6 +257,7 @@ static int apple_sart_probe(struct platform_device *pdev) i, flags, &paddr, size); set_bit(i, &sart->protected_entries); } + dev_dbg(dev, "%lu bootloader entries protected\n", hweight_long(sart->protected_entries)); platform_set_drvdata(pdev, sart); return 0; @@ -409,6 +431,10 @@ static void apple_sart_shutdown(struct platform_device *pdev) } static const struct of_device_id apple_sart_of_match[] = { + { + .compatible = "apple,t6050-sart", + .data = &sart_ops_v4, + }, { .compatible = "apple,t6000-sart", .data = &sart_ops_v3, diff --git a/drivers/thunderbolt/apple-dpin-state.c b/drivers/thunderbolt/apple-dpin-state.c index 1a2721d6e84a4..afaa64e3f6ba6 100644 --- a/drivers/thunderbolt/apple-dpin-state.c +++ b/drivers/thunderbolt/apple-dpin-state.c @@ -49,6 +49,27 @@ const struct apple_dpin_policy apple_dpin_m4 = { .lazy_idle = true, }; +/* + * T6050: the M3 flow (tunnel changed notifications, with the 10 ms HPD + * Output Set pulse and HPD wait before them; capacity retry, deferred + * bringup). Once a tunnel uses a DP IN block, its pending events are + * acknowledged (+0x04 written back, and +0x00 after a plug event) and its + * two events enabled (+0x08), as the block is prepared for a display. The + * DP IN handshake writes only DPTX_INACTIVE (+0x0c bit 0) and polls its ACK + * (+0x10 bit 0), gated on HPD. A tunnel whose DP IN block DCP never + * activated is torn down without the inactive handshake. A DPRX that is not + * done in time keeps the tunnel: the display may still train. + */ +const struct apple_dpin_policy apple_dpin_m5 = { + .flow = APPLE_DPIN_CHANGED, + .settle_ms = 100, + .capacity_retry = true, + .setup_irqs = true, + .defer_new_bringup = true, + .host_policy = TB_HOST_DP_NOTIFY | TB_HOST_DP_KEEP_DPRX_TIMEOUT, + .lazy_idle = true, +}; + enum apple_dpin_flow apple_dpin_hooks(const struct apple_dpin_policy *policy, bool queue_present, bool display_enabled) { diff --git a/drivers/thunderbolt/apple-dpin-state.h b/drivers/thunderbolt/apple-dpin-state.h index dcf3e53c0a1d6..38b8ec7eb439e 100644 --- a/drivers/thunderbolt/apple-dpin-state.h +++ b/drivers/thunderbolt/apple-dpin-state.h @@ -25,6 +25,7 @@ struct apple_dpin_policy { unsigned long host_policy; /* teardown touches the DP IN block only after DCP's Activate/Deactivate did */ bool lazy_idle; + unsigned int settle_ms; }; extern const struct apple_dpin_policy apple_dpin_disabled; @@ -32,6 +33,7 @@ extern const struct apple_dpin_policy apple_dpin_m1; extern const struct apple_dpin_policy apple_dpin_m2; extern const struct apple_dpin_policy apple_dpin_m3; extern const struct apple_dpin_policy apple_dpin_m4; +extern const struct apple_dpin_policy apple_dpin_m5; enum apple_dpin_phase { APPLE_DPIN_IDLE, diff --git a/drivers/thunderbolt/apple-dpin-test.c b/drivers/thunderbolt/apple-dpin-test.c index 0d4891ad74e14..dd532a2a051b5 100644 --- a/drivers/thunderbolt/apple-dpin-test.c +++ b/drivers/thunderbolt/apple-dpin-test.c @@ -360,6 +360,8 @@ static void apple_dpin_qualification_profiles(struct kunit *test) { "apple,t6031", &apple_dpin_m3, true, &apple_dpin_disabled }, { "apple,t8132", &apple_dpin_m4, true, &apple_dpin_m4 }, { "apple,t8132", &apple_dpin_m4, false, &apple_dpin_m4 }, + { "apple,t6050", &apple_dpin_m5, true, &apple_dpin_m5 }, + { "apple,t6050", &apple_dpin_m5, false, &apple_dpin_m5 }, { "apple,unknown", &apple_dpin_m1, true, &apple_dpin_disabled }, }; unsigned int i; diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 3374e0d3e794f..4aec7f9d77679 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -131,7 +131,12 @@ #define APPLE_MXWRAP_CPU_CTRL_RUN BIT(4) #define APPLE_MXWRAP_CPU_STATUS 0x44 +/* DP IN adapter blocks a host router may have (T6050 has four) */ +#define APPLE_DPIN_MAX 4 + struct apple_cio_hw { + /* DP IN adapter blocks of the router; 0 means the two of t8103 */ + unsigned int dpin_count; bool mxwrap; bool lstx_explicit; /* Leave MxWrap RUN alone: the CIO reconfig restarts the CPU. */ @@ -141,6 +146,23 @@ struct apple_cio_hw { bool strict_fw_ready; /* Keep the links of the host router out of CL states. */ bool no_host_clx; + /* Keep the domain's links symmetric. */ + bool no_asym; + /* Type 7 router: Enter_USB cable words, bootloader RC tuning, Type 7 NHI. */ + bool type7; + /* + * Release the CPU with a plain store of the RUN value rather than a + * read-modify-write, and halt it again when the router stops. + */ + bool run_store; + /* Hold LSTX low as a GPIO from probe until the router first takes it. */ + bool lstx_hold_from_probe; + /* The router trains Gen4 only on a cable word that says Gen4. */ + bool gen4_cable; + /* Run Gen4 links with link recovery enabled at both ends. */ + bool gen4_link_recovery; + /* Leave the host router's TMU to its firmware; host TMU constants. */ + bool host_tmu; const struct apple_dpin_policy *dp; }; @@ -178,10 +200,37 @@ static const struct apple_cio_hw apple_cio_t8132_hw = { /* Type 7 uses its existing ready-state check and has no qualified DP route. */ static const struct apple_cio_hw apple_cio_t8142_hw = { + .type7 = true, .dp = &apple_dpin_disabled, }; +/* + * T6050 is Type 7 with an MxWrap CPU wrapper that is only clocked once the + * CIO reset (the reconfig request) has run. Its CPU is released with a plain + * store of RUN; as on T8132 it takes LSTX over only once its RTKit is up, and + * the AP holds the pin while the router is off. Its four DP IN adapters take + * the M5 policy, see apple_dpin_m5. A Gen4 link times out CL training once CL + * states are enabled, then drops; it also dropped in the middle of a + * transition to an asymmetric width. Its links stay in CL0 and symmetric. + */ +static const struct apple_cio_hw apple_cio_t6050_hw = { + .dpin_count = 4, + .mxwrap = true, + .type7 = true, + .run_store = true, + .lstx_after_boot = true, + .lstx_hold_from_probe = true, + .strict_fw_ready = true, + .gen4_cable = true, + .no_host_clx = true, + .no_asym = true, + .gen4_link_recovery = true, + .host_tmu = true, + .dp = &apple_dpin_m5, +}; + static const struct apple_cio_hw apple_cio_t6040_hw = { + .type7 = true, .mxwrap = true, .lstx_explicit = true, .skip_run = true, @@ -256,12 +305,19 @@ MODULE_PARM_DESC(s2idle_teardown, #define TB_VSE_CAP_APPLE_CABLE_INFO_20_GBPS BIT(4) #define TB_VSE_CAP_APPLE_CABLE_INFO_LEGACY_ADAPTER BIT(9) #define TB_VSE_CAP_APPLE_CABLE_INFO_TBT2_3 BIT(10) +#define TB_VSE_CAP_APPLE_CABLE_INFO_GEN4 BIT(11) + +/* Enter_USB cable speed: USB PD 3.2 adds USB4 Gen4 after Gen3. */ +#define APPLE_EUDO_CABLE_SPEED_USB4_GEN4 4 static unsigned int dpin_mode_value = 8; module_param(dpin_mode_value, uint, 0644); MODULE_PARM_DESC(dpin_mode_value, "DPIN0 MODE_A/MODE_B lab value (0-" __stringify(APPLE_DPIN_MODE_VALUE_MAX) "); read at each activate"); +/* DP IN adapters without an interrupt are polled this often (ms). */ +#define APPLE_DPIN_POLL_MS 20 + struct apple_dpin_binding; /** @@ -306,6 +362,10 @@ struct apple_cio { struct apple_tunable *rc_tunable; void __iomem *pcie_adapter_base; struct apple_tunable *pcie_adapter_tunable; + void __iomem *router_ap_fw_base; + struct apple_tunable *router_ap_fw_tunable; + /* A run_store router CPU that apple_cio_start() released */ + bool cpu_running; struct resource *sram_res; void __iomem *sram_base; @@ -357,6 +417,7 @@ struct apple_cio { /* Type-C connector this router is wired to, for DP tunnel routing */ struct device_node *connector_np; bool dp_dual_stream; + unsigned int dpin_count; struct workqueue_struct *dp_wq; struct apple_dpin_ctx { struct apple_cio *acio; @@ -367,9 +428,19 @@ struct apple_cio { unsigned int idx; struct apple_dpin_state state; struct apple_dpin_tokens tokens; + unsigned long settle_until; /* new tunnel's admission deadline */ struct apple_dpin_binding *binding; /* worker-owned, handed to DCP */ bool dcp_touched; /* DCP ran the handshake on this mapping */ - } dpin[2]; + bool dcp_active; /* DCP activated the link (DPTX_INACTIVE clear) */ + /* DP IN interrupts: the line, or a poll where there is none */ + int irq; + bool irq_enabled; + struct delayed_work poll; + struct work_struct event_work; /* passes events on, on dp_wq */ + unsigned int events; /* APPLE_DP_TUNNEL_*, under lock */ + bool hpd; /* HPD level last seen, under lock */ + unsigned int irq_count; + } dpin[APPLE_DPIN_MAX]; }; /* Immutable identity for one callback handoff; freed only after provider drain. */ @@ -380,21 +451,27 @@ struct apple_dpin_binding { }; /* - * The ATC's DP IN adapter blocks ("atcN-dpin0/1" in the Apple device tree). + * The ATC's DP IN adapter blocks ("atcN-dpin0..3" in the Apple device tree). * Registers on t8103: * 0x00 status, bit 2 = HPD level * 0x04 interrupt status (write 1 to clear) * 0x08 interrupt enable [1:0] * 0x0c bit 0 DPTX_INACTIVE * 0x10 bit 0 DPTX_INACTIVE_ACK - * The block sits 0x390000 after the ACIO "rc" region; dpin1 is 0x8000 after dpin0. + * The blocks sit 0x390000 after the ACIO "rc" region: dpin1 0x8000 after + * dpin0, and on routers with four, dpin2 and dpin3 0x18000 and 0x20000 after. */ #define APPLE_DPIN_OFFSET 0x390000 -#define APPLE_DPIN_STRIDE 0x8000 #define APPLE_DPIN_SIZE 0x4000 + +static const u32 apple_dpin_offsets[APPLE_DPIN_MAX] = { + 0x0, 0x8000, 0x18000, 0x20000, +}; #define APPLE_DPIN_STATUS 0x00 #define APPLE_DPIN_STATUS_HPD BIT(2) #define APPLE_DPIN_IRQ_STATUS 0x04 +#define APPLE_DPIN_IRQ_HPD_CHANGE BIT(0) +#define APPLE_DPIN_IRQ_SINK BIT(1) #define APPLE_DPIN_IRQ_ENABLE 0x08 #define APPLE_DPIN_CTRL 0x0c #define APPLE_DPIN_CTRL_INACTIVE BIT(0) @@ -418,6 +495,8 @@ static const struct of_device_id apple_dpin_qualified_soc[] = { { .compatible = "apple,t6021" }, { .compatible = "apple,t6030" }, { .compatible = "apple,t8132" }, + /* T6050: t8103 DP IN handshake, M3 flow, four DP IN adapters */ + { .compatible = "apple,t6050" }, {}, }; @@ -545,6 +624,146 @@ static void apple_dpin_request_inactive(void *data) apple_dpin_set_active(c->acio, c->regs, c->idx, false); } +/* + * DP IN interrupts. +0x4 holds the events, write-1-to-clear: bit 0 the HPD + * level in +0x0 changed (+0x0 latches status bits that are cleared by + * writing it back), bit 1 the sink raised IRQ_HPD. Read +0x4 until it is + * stable, clear it, and hand what happened to appledrm: the display's DPTX + * port has no HPD input of its own on a tunnel. c->lock held. + */ +static unsigned int apple_dpin_irq_collect(struct apple_dpin_ctx *c, bool *seen) +{ + unsigned int events = 0, tries; + u32 irq, again, st; + bool hpd; + + lockdep_assert_held(&c->lock); + *seen = false; + /* A handler or poll may have waited on the lock while teardown stopped it. */ + if (!c->irq_enabled || !c->state.alive || !c->regs) + return 0; + irq = readl(c->regs + APPLE_DPIN_IRQ_STATUS); + for (tries = 0; tries < 8; tries++) { + again = readl(c->regs + APPLE_DPIN_IRQ_STATUS); + if (again == irq) + break; + irq = again; + } + if (!irq) + return 0; + *seen = true; + writel(irq, c->regs + APPLE_DPIN_IRQ_STATUS); + st = readl(c->regs + APPLE_DPIN_STATUS); + if (irq & APPLE_DPIN_IRQ_HPD_CHANGE) + writel(st, c->regs + APPLE_DPIN_STATUS); + hpd = st & APPLE_DPIN_STATUS_HPD; + if (hpd != c->hpd) + events |= APPLE_DP_TUNNEL_HPD_CHANGED; + c->hpd = hpd; + if (irq & APPLE_DPIN_IRQ_SINK) + events |= APPLE_DP_TUNNEL_IRQ_HPD; + c->irq_count++; + dev_dbg(c->acio->dev, "dpin%u: interrupt %#x (#%u), status %#x: HPD %d%s%s\n", + c->idx, irq, c->irq_count, st, hpd, + events & APPLE_DP_TUNNEL_IRQ_HPD ? ", IRQ_HPD" : "", + events & APPLE_DP_TUNNEL_HPD_CHANGED ? ", HPD changed" : ""); + events &= APPLE_DP_TUNNEL_IRQ_HPD | APPLE_DP_TUNNEL_HPD_CHANGED; + c->events |= events; + return events; +} + +/* Returns false if the adapter had nothing pending. */ +static bool apple_dpin_irq_dispatch(struct apple_dpin_ctx *c) +{ + unsigned int events; + bool seen; + + scoped_guard(mutex, &c->lock) + events = apple_dpin_irq_collect(c, &seen); + if (events) + queue_work(c->acio->dp_wq, &c->event_work); + return seen; +} + +static irqreturn_t apple_dpin_irq_thread(int irq, void *data) +{ + return apple_dpin_irq_dispatch(data) ? IRQ_HANDLED : IRQ_NONE; +} + +static void apple_dpin_poll_fn(struct work_struct *work) +{ + struct apple_dpin_ctx *c = container_of(to_delayed_work(work), + struct apple_dpin_ctx, poll); + bool mapped; + + apple_dpin_irq_dispatch(c); + scoped_guard(mutex, &c->lock) + mapped = c->irq_enabled && c->regs; + if (mapped) + queue_delayed_work(system_dfl_wq, &c->poll, + msecs_to_jiffies(APPLE_DPIN_POLL_MS)); +} + +/* Start taking the interrupts of a mapped adapter. c->lock held. */ +static void apple_dpin_irq_start(struct apple_dpin_ctx *c) +{ + lockdep_assert_held(&c->lock); + if (c->irq_enabled || !c->regs) + return; + c->hpd = readl(c->regs + APPLE_DPIN_STATUS) & APPLE_DPIN_STATUS_HPD; + c->events = 0; + c->irq_enabled = true; + if (c->irq > 0) + enable_irq(c->irq); + else + queue_delayed_work(system_dfl_wq, &c->poll, + msecs_to_jiffies(APPLE_DPIN_POLL_MS)); +} + +/* The handler takes c->lock: never wait for it here. c->lock held. */ +static void apple_dpin_irq_stop(struct apple_dpin_ctx *c) +{ + lockdep_assert_held(&c->lock); + if (!c->irq_enabled) + return; + c->irq_enabled = false; + c->events = 0; + if (c->irq > 0) + disable_irq_nosync(c->irq); + else + cancel_delayed_work(&c->poll); +} + +/* On dp_wq, ordered with bring-up and teardown: the binding is stable. */ +static void apple_dpin_event_fn(struct work_struct *work) +{ + struct apple_dpin_ctx *c = container_of(work, struct apple_dpin_ctx, event_work); + typeof(apple_dcp_tb_dp_event) *event; + unsigned int events; + bool hpd, active; + int ret; + + scoped_guard(mutex, &c->lock) { + events = c->events; + c->events = 0; + hpd = c->hpd; + active = c->dcp_active && c->state.alive; + } + /* The sink's requests matter once the display has activated the link. */ + if (!active) + events &= ~APPLE_DP_TUNNEL_IRQ_HPD; + if (!events || !c->binding) + return; + event = symbol_get(apple_dcp_tb_dp_event); + if (!event) + return; + ret = event(c->acio->connector_np, c->idx, c->binding->generation, events, hpd); + symbol_put(apple_dcp_tb_dp_event); + if (ret) + dev_info(c->acio->dev, "dpin%u: display did not take events %#x: %d\n", + c->idx, events, ret); +} + static void apple_dpin_request_mask_irqs(void *data) { struct apple_dpin_ctx *c = data; @@ -555,6 +774,7 @@ static void apple_dpin_request_mask_irqs(void *data) writel(readl(c->regs + APPLE_DPIN_IRQ_ENABLE) & ~3, c->regs + APPLE_DPIN_IRQ_ENABLE); + apple_dpin_irq_stop(c); } static bool apple_dpin_request(struct apple_dpin_ctx *c, u64 generation, bool active) @@ -564,10 +784,15 @@ static bool apple_dpin_request(struct apple_dpin_ctx *c, u64 generation, bool ac .mask_irqs = apple_dpin_request_mask_irqs, .ctx = c, }; + bool replacement = active && c->tokens.requested != generation; lockdep_assert_held(&c->lock); - return apple_dpin_powered_request(&c->state, &c->tokens, c->acio->dp, - generation, active, !!c->regs, &ops); + if (!apple_dpin_powered_request(&c->state, &c->tokens, c->acio->dp, + generation, active, !!c->regs, &ops)) + return false; + if (replacement) + c->settle_until = jiffies + msecs_to_jiffies(c->acio->dp->settle_ms); + return true; } /* Called by appledrm from DCP's Activate/Deactivate calls. */ @@ -575,6 +800,7 @@ static int apple_dpin_dcp_set_active(void *data, bool active) { const struct apple_dpin_binding *binding = data; struct apple_dpin_ctx *c = binding->ctx; + int ret; guard(mutex)(&c->lock); if (!apple_dpin_token_access(&c->tokens, binding->generation)) @@ -583,7 +809,10 @@ static int apple_dpin_dcp_set_active(void *data, bool active) if (!c->state.alive || !c->regs) return -ENODEV; c->dcp_touched = true; - return apple_dpin_set_active(c->acio, c->regs, c->idx, active); + ret = apple_dpin_set_active(c->acio, c->regs, c->idx, active); + if (!ret) + c->dcp_active = active; + return ret; } static bool apple_dpin_provider_get(void *data) @@ -683,14 +912,45 @@ static int apple_dpin_connect(struct apple_dpin_ctx *c, } } +/* + * A USB4 dock can withdraw and re-advertise its DP OUT adapters while the + * initial capability exchange completes. On T6050, starting DCP on the + * temporary tunnel can leave its AUX request stuck after that tunnel goes + * away. Let the request settle before admitting any DCP callback or touching + * the adapter. Teardown still idles immediately in apple_dpin_request(). + * A replacement owns a new deadline; this worker never admits the old token. + */ +static int apple_dpin_wait_settled(struct apple_dpin_ctx *c, u64 generation) +{ + unsigned long delay, now; + + if (!c->acio->dp->settle_ms) + return 0; + for (;;) { + scoped_guard(mutex, &c->lock) { + if (!c->state.alive || c->tokens.requested != generation) + return -ESTALE; + now = jiffies; + if (time_after_eq(now, c->settle_until)) + return 0; + delay = c->settle_until - now; + } + msleep(jiffies_to_msecs(delay) + 1); + } +} + static int apple_dpin_up(struct apple_dpin_ctx *c, struct apple_dpin_binding *binding, bool *deferred) { struct apple_cio *acio = c->acio; void __iomem *regs = c->regs; + int ret; u32 irq; *deferred = false; + ret = apple_dpin_wait_settled(c, binding->generation); + if (ret) + return ret; scoped_guard(mutex, &c->lock) { if (!c->state.alive || c->tokens.requested != binding->generation) return -ESTALE; @@ -701,7 +961,7 @@ static int apple_dpin_up(struct apple_dpin_ctx *c, /* This admission also owns the first DCP call after dropping the lock. */ if (!regs) { regs = ioremap_np(acio->rc_res->start + APPLE_DPIN_OFFSET + - c->idx * APPLE_DPIN_STRIDE, APPLE_DPIN_SIZE); + apple_dpin_offsets[c->idx], APPLE_DPIN_SIZE); if (!regs) return -ENOMEM; c->regs = regs; @@ -710,8 +970,7 @@ static int apple_dpin_up(struct apple_dpin_ctx *c, * Acknowledge what is pending and enable the DP IN interrupts: * +0x4 is write-1-to-clear, and a plug event (bit 0) also * latches status bits in +0x0 that are cleared by writing it - * back. Nothing handles these interrupts yet; they are enabled - * as part of bringing the adapter up. + * back. See apple_dpin_irq_collect() for what they report. */ if (acio->dp->setup_irqs) { irq = readl(regs + APPLE_DPIN_IRQ_STATUS); @@ -719,6 +978,7 @@ static int apple_dpin_up(struct apple_dpin_ctx *c, if (irq & BIT(0)) writel(readl(regs + APPLE_DPIN_STATUS), regs + APPLE_DPIN_STATUS); writel(readl(regs + APPLE_DPIN_IRQ_ENABLE) | 3, regs + APPLE_DPIN_IRQ_ENABLE); + apple_dpin_irq_start(c); } /* Activate may run before attach returns, so admit its immutable cookie now. */ apple_dpin_token_admit(&c->tokens, binding->generation); @@ -753,9 +1013,11 @@ static void apple_dpin_down(struct apple_dpin_ctx *c) } /* The powered request hook idled the adapter before queued teardown. */ scoped_guard(mutex, &c->lock) { + apple_dpin_irq_stop(c); regs = c->regs; c->regs = NULL; c->dcp_touched = false; + c->dcp_active = false; apple_dpin_step(&c->state, c->acio->dp, APPLE_DPIN_DROPPED, false, 0); } if (regs) @@ -883,8 +1145,10 @@ static void apple_cio_destroy_wq(void *data) struct apple_cio *acio = data; /* a pending retry would queue onto the destroyed queue */ - for (unsigned int i = 0; i < ARRAY_SIZE(acio->dpin); i++) + for (unsigned int i = 0; i < ARRAY_SIZE(acio->dpin); i++) { cancel_delayed_work_sync(&acio->dpin[i].retry); + cancel_delayed_work_sync(&acio->dpin[i].poll); + } destroy_workqueue(acio->dp_wq); } @@ -895,7 +1159,8 @@ static void apple_cio_of_node_put(void *data) static bool apple_cio_pciec_eligible(const struct device_node *np) { - return of_device_is_compatible(np, "apple,t6040-pciec") || + return of_device_is_compatible(np, "apple,t6050-pciec") || + of_device_is_compatible(np, "apple,t6040-pciec") || of_device_is_compatible(np, "apple,t8103-pciec") || of_device_is_compatible(np, "apple,t6000-pciec"); } @@ -1108,6 +1373,10 @@ static int apple_nhi_probe_irqs(struct apple_nhi *anhi) n_irqs = platform_irq_count(anhi->pdev); if (n_irqs < 0) return dev_err_probe(anhi->dev, n_irqs, "platform_irq_count failed\n"); + /* T6050 adds a named SRAM parity interrupt after the ring interrupts. */ + if (n_irqs > 0 && + platform_get_irq_byname_optional(anhi->pdev, "parity") > 0) + n_irqs--; if (n_irqs == 0) { dev_err(anhi->dev, "No interrupts found\n"); return -EINVAL; @@ -1867,7 +2136,7 @@ static void apple_dp_aux_work(struct work_struct *work) } /* - * Which DP IN adapter (0/1) on the host router this port is -- same + * Which DP IN adapter (0..3) on the host router this port is -- same * counting order as apple_dp_in_analog_base() above and the port-to-dpin * mapping in apple_nhi_dp_tunnel_changed(). */ @@ -1889,6 +2158,12 @@ static int apple_dpin_index_for_port(struct apple_nhi *anhi, struct tb_port *in) return -1; } +/* A DP IN adapter index with a block of its own on this router */ +static bool apple_dpin_valid(const struct apple_cio *acio, int idx) +{ + return acio && idx >= 0 && idx < acio->dpin_count; +} + static int apple_nhi_dp_tunnel_pre_activate(struct tb_nhi *nhi, struct tb_port *in, struct tb_port *out) @@ -1908,7 +2183,7 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, /* Validate the token before changing analog/AUX or display state. */ idx = apple_dpin_index_for_port(anhi, in); - if (idx >= 0 && idx <= 1 && anhi->acio && anhi->acio->connector_np && + if (apple_dpin_valid(anhi->acio, idx) && anhi->acio->connector_np && anhi->acio->dp_wq) { c = &anhi->acio->dpin[idx]; scoped_guard(mutex, &c->lock) { @@ -1935,7 +2210,7 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, apple_dp_dump_vse(in->sw); /* Route the display pipeline through the selected host DP IN adapter. */ - if (idx < 0 || idx > 1 || !anhi->acio) { + if (!apple_dpin_valid(anhi->acio, idx)) { dev_warn(anhi->dev, "DP IN tunnel: could not map port %u to a dpin index\n", in->port); @@ -1975,7 +2250,7 @@ static void apple_nhi_dp_tunnel_deactivate(struct tb_nhi *nhi, /* Sleep the adapter before tunnel teardown can remove its power. */ idx = apple_dpin_index_for_port(anhi, in); - if (idx >= 0 && idx <= 1 && anhi->acio && anhi->acio->dp_wq) { + if (apple_dpin_valid(anhi->acio, idx) && anhi->acio->dp_wq) { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; scoped_guard(mutex, &c->lock) { @@ -1998,7 +2273,7 @@ static void apple_nhi_dp_tunnel_changed(struct tb_nhi *nhi, u8 in_port, unsigned int dpin = 0; bool found = false; - /* The crossbar's dpin0/dpin1 follow the host router's DP IN adapters in order. */ + /* The crossbar's dpin0..3 follow the host router's DP IN adapters in order. */ tb_switch_for_each_port(anhi->tb->root_switch, port) { if (!tb_port_is_dpin(port)) continue; @@ -2008,7 +2283,7 @@ static void apple_nhi_dp_tunnel_changed(struct tb_nhi *nhi, u8 in_port, } dpin++; } - if (!found || dpin > 1) { + if (!found || dpin >= acio->dpin_count) { dev_warn(acio->dev, "DP tunnel from unexpected adapter %u\n", in_port); return; } @@ -2036,7 +2311,7 @@ static bool apple_nhi_dp_tunnel_awaits_display(struct tb_nhi *nhi, int idx; idx = apple_dpin_index_for_port(anhi, in); - if (idx < 0 || idx > 1 || !anhi->acio) + if (!apple_dpin_valid(anhi->acio, idx)) return false; c = &anhi->acio->dpin[idx]; guard(mutex)(&c->lock); @@ -2084,9 +2359,13 @@ static int apple_nhi_probe(struct platform_device *pdev) int cap_apple; int ret = 0; + /* The T6050 router is the Type 7 one released by a RUN store. */ if (acio->type7 != (of_device_is_compatible(pdev->dev.of_node, "apple,t8142-usb4-nhi") || - of_device_is_compatible(pdev->dev.of_node, "apple,t6040-usb4-nhi"))) { + of_device_is_compatible(pdev->dev.of_node, "apple,t6040-usb4-nhi") || + of_device_is_compatible(pdev->dev.of_node, "apple,t6050-usb4-nhi")) || + acio->hw->run_store != + !!of_device_is_compatible(pdev->dev.of_node, "apple,t6050-usb4-nhi")) { ret = dev_err_probe(&pdev->dev, -EINVAL, "ACIO/NHI generation mismatch\n"); goto err; } @@ -2173,6 +2452,12 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->nhi.quirks |= QUIRK_HOST_DP_NFC_CREDITS; if (acio->hw->no_host_clx) anhi->nhi.quirks |= QUIRK_NO_HOST_CLX; + if (acio->hw->no_asym) + anhi->nhi.quirks |= QUIRK_NO_ASYM; + if (acio->hw->gen4_link_recovery) + anhi->nhi.quirks |= QUIRK_GEN4_LINK_RECOVERY; + if (acio->hw->host_tmu) + anhi->nhi.quirks |= QUIRK_HOST_TMU; anhi->nhi.hop_count = readl(anhi->nhi_base + APPLE_CIO_NHI_HOP_COUNT) & APPLE_CIO_NHI_HOP_COUNT_MASK; @@ -2515,6 +2800,9 @@ static const struct of_device_id apple_nhi_match[] = { { .compatible = "apple,t6040-usb4-nhi", }, + { + .compatible = "apple,t6050-usb4-nhi", + }, { .compatible = "apple,t8142-usb4-nhi", }, @@ -2648,6 +2936,22 @@ static int apple_cio_lstx_release(struct apple_cio *acio) return pinctrl_select_state(acio->pinctrl, acio->lstx_pins); } +/* + * Halt a run_store router CPU that apple_cio_start() released. Its wrapper is + * only clocked after the CIO reset, so never touch it otherwise. + */ +static void apple_cio_cpu_halt(struct apple_cio *acio) +{ + u32 val; + + if (!acio->hw->run_store || !acio->cpu_running) + return; + acio->cpu_running = false; + val = readl(acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + writel(val & ~APPLE_MXWRAP_CPU_CTRL_RUN, + acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); +} + /** * apple_cio_stop - Stop and power down the ACIO complex * @acio: ACIO complex to stop @@ -2672,6 +2976,7 @@ static void apple_cio_stop(struct apple_cio *acio) "Failed to shutdown M3 RTKit, continuing ACIO shutdown anyway\n"); apple_rtkit_free(acio->rtk); acio->rtk = NULL; + apple_cio_cpu_halt(acio); /* High only if the router left it driven high with the cable still in */ if (acio->hw->lstx_after_boot) @@ -2691,9 +2996,15 @@ static void apple_cio_stop(struct apple_cio *acio) acio->resume_link_failed = false; } -/* Only called while the ACIO power domains are on. */ +/* + * Only called while the ACIO power domains are on. Neither a T6050 router's rc + * block nor its CPU wrapper is known to be readable while it starts (see + * apple_cio_start()), so log nothing for it. + */ static void apple_cio_log_state(struct apple_cio *acio, const char *when) { + if (acio->hw->run_store) + return; if (acio->cpu_base) dev_info(acio->dev, "%s: LSTX 0x%08x, FW state 0x%08x, CPU ctrl 0x%08x, status 0x%08x\n", when, readl(acio->rc_base + APPLE_CIO_M3_CTRL), @@ -2809,7 +3120,11 @@ static int apple_cio_start(struct apple_cio *acio) * them with a bare store. */ if (acio->hw->lstx_after_boot) { - /* Written below, once RTKit is up. */ + /* + * Written below, once RTKit is up. A T6050 CPU is started through + * its wrapper below, and its rc block takes an external abort until + * the CPU runs, so M3_CTRL is left alone here. + */ } else if (acio->hw->lstx_explicit) { writel(APPLE_CIO_LSTX_CTRL_ENABLE | APPLE_CIO_LSTX_CTRL_LEVEL, acio->rc_base + APPLE_CIO_M3_CTRL); @@ -2820,9 +3135,21 @@ static int apple_cio_start(struct apple_cio *acio) if (acio->hw->mxwrap && !acio->hw->skip_run) { apple_cio_log_state(acio, "before RUN"); - val = readl(acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); - writel(val | APPLE_MXWRAP_CPU_CTRL_RUN, - acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + if (acio->hw->run_store) { + /* + * The CIO reset above clocks the wrapper; the CPU runs the + * image the boot loader left in its SRAM once RUN is set. + * The run value is stored outright, not merged. + */ + writel(APPLE_MXWRAP_CPU_CTRL_RUN, + acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + acio->cpu_running = true; + apple_cio_log_state(acio, "after RUN"); + } else { + val = readl(acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + writel(val | APPLE_MXWRAP_CPU_CTRL_RUN, + acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + } } acio->rtk = apple_rtkit_init(acio->dev, acio, NULL, 0, &apple_cio_rtkit_ops); @@ -2881,6 +3208,8 @@ static int apple_cio_start(struct apple_cio *acio) apple_tunable_apply(acio->pcie_adapter_base, acio->pcie_adapter_tunable); } + /* The router AP firmware tunable goes with the others once the firmware is ready. */ + apple_tunable_apply(acio->router_ap_fw_base, acio->router_ap_fw_tunable); /* * Bring up devices which are part of ACIO and are now accessible by the main SoC @@ -2918,6 +3247,7 @@ static int apple_cio_start(struct apple_cio *acio) apple_rtkit_free(acio->rtk); remove_links: acio->rtk = NULL; + apple_cio_cpu_halt(acio); /* Cut power to reset the entire block */ for (i = 0; i < acio->pd_list->num_pds; i++) { if (acio->pd_list->pd_links[i]) @@ -3131,7 +3461,16 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, acio->target_cable_info = TB_VSE_CAP_APPLE_CABLE_INFO_PRESENT; if (FIELD_GET(EUDO_CABLE_TYPE_MASK, data->usb4.eudo) != EUDO_CABLE_TYPE_PASSIVE) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_ACTIVE_CABLE; - if (FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= EUDO_CABLE_SPEED_USB4_GEN3) + /* + * The T6050 router firmware trains Gen4 only on a Gen4 cable, and + * takes Gen4 alone: no 20 Gb/s bit beside it. + */ + if (acio->hw->gen4_cable && + FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= + APPLE_EUDO_CABLE_SPEED_USB4_GEN4) + acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_GEN4; + else if (FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= + EUDO_CABLE_SPEED_USB4_GEN3) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_20_GBPS; if (data->orientation == TYPEC_ORIENTATION_REVERSE) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_ORIENTATION_REVERSE; @@ -3335,16 +3674,15 @@ static int apple_cio_probe(struct platform_device *pdev) init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; - acio->type7 = of_device_is_compatible(acio->np, "apple,t8142-usb4-acio") || - of_device_is_compatible(acio->np, "apple,t6040-usb4-acio"); - if (acio->type7 && - of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) - return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); - acio->hw = of_device_get_match_data(dev); if (!acio->hw) return -EINVAL; + acio->type7 = acio->hw->type7; + if (acio->type7 && + of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) + return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); + if (acio->hw->mxwrap) { struct resource *cpu_res; @@ -3367,6 +3705,9 @@ static int apple_cio_probe(struct platform_device *pdev) acio->lstx_pins = NULL; acio->lstx_hold = acio->lstx_pins && of_property_present(dev->of_node, "lstx-gpios"); + /* Hold the line low until the router first takes it. */ + if (acio->hw->lstx_hold_from_probe) + apple_cio_lstx_hold(acio, false); } acio->pcie_tunnel_np = of_parse_phandle(dev->of_node, "apple,pcie-tunnel", 0); @@ -3435,6 +3776,32 @@ static int apple_cio_probe(struct platform_device *pdev) } } + { + struct resource *ap_fw; + + ap_fw = platform_get_resource_byname(pdev, IORESOURCE_MEM, "router-ap-fw"); + if (ap_fw) { + acio->router_ap_fw_base = devm_ioremap_resource(dev, ap_fw); + if (IS_ERR(acio->router_ap_fw_base)) + return dev_err_probe(dev, PTR_ERR(acio->router_ap_fw_base), + "Unable to map router AP firmware regs\n"); + acio->router_ap_fw_tunable = + devm_apple_tunable_parse(dev, acio->np, + "apple,tunable-router-ap-fw", ap_fw); + if (IS_ERR(acio->router_ap_fw_tunable)) { + if (PTR_ERR(acio->router_ap_fw_tunable) != -ENOENT) + return dev_err_probe(dev, + PTR_ERR(acio->router_ap_fw_tunable), + "Unable to load router AP firmware tunable\n"); + acio->router_ap_fw_tunable = NULL; + } + } + } + + /* + * On T6050 this is the CIO reconfig request, which also clocks the + * router CPU's wrapper: nothing may touch the wrapper before it. + */ acio->reset = devm_reset_control_get_exclusive(dev, NULL); if (IS_ERR(acio->reset)) return dev_err_probe(dev, PTR_ERR(acio->reset), "Unable to get CIO reset\n"); @@ -3463,6 +3830,7 @@ static int apple_cio_probe(struct platform_device *pdev) else if (ret < 3) return dev_err_probe(dev, -EINVAL, "Not enough PM domains\n"); + acio->dpin_count = acio->hw->dpin_count ?: 2; acio->connector_np = of_graph_get_remote_node(dev->of_node, 1, -1); acio->dp_dual_stream = apple_dp_tunnel_dual_stream(acio->connector_np); /* Register layout, root qualification and connector wiring are separate. */ @@ -3477,6 +3845,8 @@ static int apple_cio_probe(struct platform_device *pdev) acio->dpin[i].idx = i; INIT_WORK(&acio->dpin[i].work, apple_dpin_work_fn); INIT_DELAYED_WORK(&acio->dpin[i].retry, apple_dpin_retry_fn); + INIT_DELAYED_WORK(&acio->dpin[i].poll, apple_dpin_poll_fn); + INIT_WORK(&acio->dpin[i].event_work, apple_dpin_event_fn); ret = devm_mutex_init(dev, &acio->dpin[i].lock); if (ret) return ret; @@ -3488,6 +3858,25 @@ static int apple_cio_probe(struct platform_device *pdev) ret = devm_add_action_or_reset(dev, apple_cio_destroy_wq, acio); if (ret) return ret; + /* Freed before the queue: started and stopped with the mapping. */ + for (unsigned int i = 0; i < acio->dpin_count && i < APPLE_DPIN_MAX; i++) { + struct apple_dpin_ctx *c = &acio->dpin[i]; + char name[8]; + + snprintf(name, sizeof(name), "dpin%u", i); + c->irq = platform_get_irq_byname_optional(to_platform_device(dev), name); + if (c->irq <= 0) + continue; + ret = devm_request_threaded_irq(dev, c->irq, NULL, + apple_dpin_irq_thread, + IRQF_ONESHOT | IRQF_NO_AUTOEN, + devm_kasprintf(dev, GFP_KERNEL, "%s-%s", + dev_name(dev), name), + c); + if (ret) + return dev_err_probe(dev, ret, + "cannot request %s interrupt\n", name); + } } /* And finally register the OOB notification for Thunderbolt/USB4 cables */ @@ -3548,6 +3937,10 @@ static const struct of_device_id apple_acio_match[] = { .compatible = "apple,t6040-usb4-acio", .data = &apple_cio_t6040_hw, }, + { + .compatible = "apple,t6050-usb4-acio", + .data = &apple_cio_t6050_hw, + }, { .compatible = "apple,t8142-usb4-acio", .data = &apple_cio_t8142_hw, diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c index 389de7df84f65..f2d56acb21a10 100644 --- a/drivers/thunderbolt/eeprom.c +++ b/drivers/thunderbolt/eeprom.c @@ -381,6 +381,12 @@ static int tb_drom_parse_entry_port(struct tb_switch *sw, return 0; res = tb_port_read(port, &type, TB_CFG_PORT, 2, 1); + /* Listed, but not implemented by this router: skip it as well */ + if (res == -ENODEV) { + tb_sw_dbg(sw, "DROM entry for unimplemented port %u ignored\n", + header->index); + return 0; + } if (res) return res; type &= 0xffffff; diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 75795fbf865a1..6ec6d61f02759 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -222,6 +222,15 @@ struct tb_nhi_ops { #define QUIRK_KEEP_TUNNELS BIT(7) /* The links of the host router must not enter CL states. */ #define QUIRK_NO_HOST_CLX BIT(8) +/* The links of the domain must stay symmetric. */ +#define QUIRK_NO_ASYM BIT(9) +/* Gen4 links of the domain run with link recovery enabled on both ends. */ +#define QUIRK_GEN4_LINK_RECOVERY BIT(10) +/* + * The host router's firmware runs its TMU, and enhanced uni-directional + * mode takes the host's own constants, see tmu.c. + */ +#define QUIRK_HOST_TMU BIT(11) /* Host display policies are independent of which optional hooks are installed. */ #define TB_HOST_DP_NOTIFY BIT(0) diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index a83bbcbca0194..1bc7165678e9d 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -3206,6 +3206,40 @@ int tb_switch_set_link_width(struct tb_switch *sw, enum tb_link_width width) return ret; } +/* + * Some hosts (Apple silicon) run their Gen4 links with link recovery + * enabled at both ends. Without it, the router firmware has no way back + * from a Gen4 link error but to disconnect the link. + */ +static void tb_switch_gen4_link_recovery(struct tb_switch *sw, + struct tb_port *up, struct tb_port *down) +{ + struct tb_port *ports[] = { down, up }; + unsigned int i; + u32 val; + + if (!(sw->tb->nhi->quirks & QUIRK_GEN4_LINK_RECOVERY) || + tb_port_get_link_generation(up) < 4) + return; + + for (i = 0; i < ARRAY_SIZE(ports); i++) { + struct tb_port *port = ports[i]; + + if (!port->cap_usb4 || + tb_port_read(port, &val, TB_CFG_PORT, + port->cap_usb4 + PORT_CS_19, 1)) + continue; + if (val & PORT_CS_19_GEN4_LINK_RECOVERY) + continue; + val |= PORT_CS_19_GEN4_LINK_RECOVERY; + if (tb_port_write(port, &val, TB_CFG_PORT, + port->cap_usb4 + PORT_CS_19, 1)) + tb_port_warn(port, "cannot enable Gen4 link recovery\n"); + else + tb_port_dbg(port, "Gen4 link recovery enabled\n"); + } +} + /** * tb_switch_configure_link() - Set link configured * @sw: Switch whose link is configured @@ -3236,8 +3270,15 @@ int tb_switch_configure_link(struct tb_switch *sw) down = up->remote; if (tb_switch_is_usb4(down->sw)) - return usb4_port_configure(down); - return tb_lc_configure_port(down); + ret = usb4_port_configure(down); + else + ret = tb_lc_configure_port(down); + if (ret) + return ret; + + if (tb_switch_is_usb4(up->sw) && tb_switch_is_usb4(down->sw)) + tb_switch_gen4_link_recovery(sw, up, down); + return 0; } /** diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index a4d439f2394df..f6e750743206b 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -355,6 +355,10 @@ static int tb_enable_tmu(struct tb_switch *sw) { int ret; + /* The host router's firmware runs its TMU on these hosts. */ + if (!tb_route(sw) && tb_switch_tmu_host_managed(sw)) + return 0; + /* * If both routers at the end of the link are v2 we simply * enable the enhanced uni-directional mode. That covers all @@ -746,9 +750,10 @@ static int tb_maximum_bandwidth(struct tb *tb, struct tb_port *src_port, /* * Can include asymmetric, only if it is actually supported by - * the lane adapter. + * the lane adapter and allowed on the host. */ - if (!tb_asym_supported(src_port, dst_port, port)) + if (!tb_asym_supported(src_port, dst_port, port) || + (tb->nhi->quirks & QUIRK_NO_ASYM)) include_asym = false; if (tb_is_upstream_port(port)) { @@ -1078,7 +1083,7 @@ static int tb_configure_asym(struct tb *tb, struct tb_port *src_port, struct tb_port *up; int ret = 0; - if (!asym_threshold) + if (!asym_threshold || (tb->nhi->quirks & QUIRK_NO_ASYM)) return 0; downstream = tb_port_path_direction_downstream(src_port, dst_port); @@ -3943,9 +3948,11 @@ static int tb_start(struct tb *tb, bool reset) * To support highest CLx state, we set host router's TMU to * Normal mode. */ - tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_MODE_LOWRES); - /* Enable TMU if it is off */ - tb_switch_tmu_enable(tb->root_switch); + if (!tb_switch_tmu_host_managed(tb->root_switch)) { + tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_MODE_LOWRES); + /* Enable TMU if it is off */ + tb_switch_tmu_enable(tb->root_switch); + } /* * Boot firmware might have created tunnels of its own. Since we diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 841d1ccb27042..a10d8d12f3d5d 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1061,6 +1061,7 @@ int tb_switch_tmu_post_time(struct tb_switch *sw); int tb_switch_tmu_disable(struct tb_switch *sw); int tb_switch_tmu_enable(struct tb_switch *sw); int tb_switch_tmu_configure(struct tb_switch *sw, enum tb_switch_tmu_mode mode); +bool tb_switch_tmu_host_managed(const struct tb_switch *sw); /** * tb_switch_tmu_is_configured() - Is given TMU mode configured diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 54057a331a596..827cbb739fb7e 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -398,6 +398,7 @@ struct tb_regs_port_header { #define PORT_CS_19_WOD BIT(17) #define PORT_CS_19_WOU4 BIT(18) #define PORT_CS_19_START_ASYM BIT(24) +#define PORT_CS_19_GEN4_LINK_RECOVERY BIT(31) /* Display Port adapter registers */ #define ADP_DP_CS_0 0x00 diff --git a/drivers/thunderbolt/tmu.c b/drivers/thunderbolt/tmu.c index cf779874c6756..ce494a044e4f0 100644 --- a/drivers/thunderbolt/tmu.c +++ b/drivers/thunderbolt/tmu.c @@ -19,7 +19,7 @@ static const unsigned int tmu_rates[] = { [TB_SWITCH_TMU_MODE_MEDRES_ENHANCED_UNI] = 16, }; -static const struct { +static const struct tmu_mode_params { unsigned int freq_meas_window; unsigned int avg_const; unsigned int delta_avg_const; @@ -37,6 +37,36 @@ static const struct { }, }; +/* + * Enhanced uni-directional mode as Apple silicon hosts run it: averaging + * and delta averaging constants 8, and replenish timeout 1875, threshold 0 + * and count 222, written to the parent side of the link only. + */ +static const struct tmu_mode_params tmu_params_host_enhanced = { + 800, 8, 8, 1875, 0, 222, 255, +}; + +/** + * tb_switch_tmu_host_managed() - Is the TMU set up the host's way + * @sw: Any router of the domain + * + * Return: %true if the host router's TMU is left to its firmware and + * enhanced uni-directional mode uses the host's constants. + */ +bool tb_switch_tmu_host_managed(const struct tb_switch *sw) +{ + return sw->tb->nhi->quirks & QUIRK_HOST_TMU; +} + +static const struct tmu_mode_params * +tb_tmu_params(const struct tb_switch *sw, enum tb_switch_tmu_mode mode) +{ + if (mode == TB_SWITCH_TMU_MODE_MEDRES_ENHANCED_UNI && + tb_switch_tmu_host_managed(sw)) + return &tmu_params_host_enhanced; + return &tmu_params[mode]; +} + static const char *tmu_mode_name(enum tb_switch_tmu_mode mode) { switch (mode) { @@ -66,8 +96,8 @@ static int tb_switch_set_tmu_mode_params(struct tb_switch *sw, u32 freq, avg, val; int ret; - freq = tmu_params[mode].freq_meas_window; - avg = tmu_params[mode].avg_const; + freq = tb_tmu_params(sw, mode)->freq_meas_window; + avg = tb_tmu_params(sw, mode)->avg_const; ret = tb_sw_read(sw, &val, TB_CFG_SWITCH, sw->tmu.cap + TMU_RTR_CS_0, 1); @@ -102,7 +132,7 @@ static int tb_switch_set_tmu_mode_params(struct tb_switch *sw, return ret; if (tb_switch_tmu_enhanced_is_supported(sw)) { - u32 delta_avg = tmu_params[mode].delta_avg_const; + u32 delta_avg = tb_tmu_params(sw, mode)->delta_avg_const; ret = tb_sw_read(sw, &val, TB_CFG_SWITCH, sw->tmu.cap + TMU_RTR_CS_18, 1); @@ -257,10 +287,10 @@ static int tb_port_set_tmu_mode_params(struct tb_port *port, u32 repl_timeout, repl_threshold, repl_n, dirswitch_n, val; int ret; - repl_timeout = tmu_params[mode].repl_timeout; - repl_threshold = tmu_params[mode].repl_threshold; - repl_n = tmu_params[mode].repl_n; - dirswitch_n = tmu_params[mode].dirswitch_n; + repl_timeout = tb_tmu_params(port->sw, mode)->repl_timeout; + repl_threshold = tb_tmu_params(port->sw, mode)->repl_threshold; + repl_n = tb_tmu_params(port->sw, mode)->repl_n; + dirswitch_n = tb_tmu_params(port->sw, mode)->dirswitch_n; ret = tb_port_read(port, &val, TB_CFG_PORT, port->cap_tmu + TMU_ADP_CS_8, 1); @@ -291,6 +321,23 @@ static int tb_port_set_tmu_mode_params(struct tb_port *port, port->cap_tmu + TMU_ADP_CS_9, 1); } +static int tb_port_tmu_dirswitch_write(struct tb_port *port, u32 dirswitch_n) +{ + int ret; + u32 val; + + ret = tb_port_read(port, &val, TB_CFG_PORT, + port->cap_tmu + TMU_ADP_CS_9, 1); + if (ret) + return ret; + + val &= ~TMU_ADP_CS_9_DIRSWITCH_N_MASK; + val |= FIELD_PREP(TMU_ADP_CS_9_DIRSWITCH_N_MASK, dirswitch_n); + + return tb_port_write(port, &val, TB_CFG_PORT, + port->cap_tmu + TMU_ADP_CS_9, 1); +} + /* Can be called to non-v2 lane adapters too */ static int tb_port_tmu_rate_write(struct tb_port *port, int rate) { @@ -786,7 +833,29 @@ static int tb_switch_tmu_enable_enhanced(struct tb_switch *sw) up = tb_upstream_port(sw); down = tb_switch_downstream_port(sw); - ret = tb_port_set_tmu_mode_params(up, sw->tmu.mode_request); + /* + * The host's way: time sync off on both lane adapters first, then + * on again once the mode is programmed (below). The host router + * applies its TMU configuration only when its lane adapter's time + * sync goes from off to on, and it comes out of reset on. + */ + if (tb_switch_tmu_host_managed(sw)) { + ret = tb_port_tmu_time_sync_disable(up); + if (!ret) + ret = tb_port_tmu_time_sync_disable(down); + if (ret) + goto out; + } + + /* + * The host's way: replenish parameters on the parent side only, + * but the direction switch count on both. + */ + if (tb_switch_tmu_host_managed(sw)) + ret = tb_port_tmu_dirswitch_write(up, + tb_tmu_params(sw, sw->tmu.mode_request)->dirswitch_n); + else + ret = tb_port_set_tmu_mode_params(up, sw->tmu.mode_request); if (ret) goto out; @@ -810,6 +879,16 @@ static int tb_switch_tmu_enable_enhanced(struct tb_switch *sw) if (ret) goto out; + /* Time sync on again: device side first, then the host router's. */ + if (tb_switch_tmu_host_managed(sw)) { + ret = tb_port_tmu_time_sync_enable(up); + if (ret) + goto out; + ret = tb_port_tmu_time_sync_enable(down); + if (ret) + goto out; + } + return 0; out: @@ -962,9 +1041,12 @@ int tb_switch_tmu_enable(struct tb_switch *sw) return ret; } - ret = tb_switch_tmu_set_time_disruption(sw, true); - if (ret) - return ret; + /* The host's way leaves the router's time disruption bit alone. */ + if (!tb_switch_tmu_host_managed(sw)) { + ret = tb_switch_tmu_set_time_disruption(sw, true); + if (ret) + return ret; + } if (tb_route(sw)) { /* @@ -1014,6 +1096,8 @@ int tb_switch_tmu_enable(struct tb_switch *sw) tb_sw_dbg(sw, "TMU: mode set to: %s\n", tmu_mode_name(sw->tmu.mode)); } + if (tb_switch_tmu_host_managed(sw)) + return ret; return tb_switch_tmu_set_time_disruption(sw, false); } diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 947a297f4d09e..94b1cac077e97 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1319,6 +1319,37 @@ static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) return 0; } +/* + * An Apple silicon host sets the SSC offset that the DP OUT adapter's + * stream clock recovery allows, the low half of TMU capability word 6, to + * 1500 before it touches the adapter otherwise. Docks come up with their + * own value (2500 on a Barlow Ridge dock). + */ +#define TB_DP_APPLE_SSC_OFFSET 1500 + +static void tb_dp_apple_ssc_offset(struct tb_tunnel *tunnel) +{ + struct tb_port *out = tunnel->dst_port; + u32 v, old; + + if (!tb_port_is_apple_host_dpin(tunnel->src_port) || + !tb_port_is_dpout(out) || !out->cap_tmu) + return; + /* Only inside the TMU capability: before its next-capability pointer. */ + if (tb_port_read(out, &v, TB_CFG_PORT, out->cap_tmu, 1) || + ((v & 0xff) && (v & 0xff) <= out->cap_tmu + 6) || + tb_port_read(out, &v, TB_CFG_PORT, out->cap_tmu + 6, 1)) + return; + old = v; + v = (v & ~GENMASK(15, 0)) | TB_DP_APPLE_SSC_OFFSET; + if (v != old && + tb_port_write(out, &v, TB_CFG_PORT, out->cap_tmu + 6, 1)) { + tb_port_warn(out, "cannot set the DP SSC offset\n"); + return; + } + tb_port_dbg(out, "DP SSC offset %#x (was %#x)\n", v & 0xffff, old & 0xffff); +} + static int tb_dp_pre_activate(struct tb_tunnel *tunnel) { struct tb_port *in = tunnel->src_port; @@ -1326,6 +1357,8 @@ static int tb_dp_pre_activate(struct tb_tunnel *tunnel) struct tb *tb = in->sw->tb; int ret; + tb_dp_apple_ssc_offset(tunnel); + ret = tb_dp_xchg_caps(tunnel); if (ret) return ret; @@ -1609,8 +1642,9 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) /* * Apple silicon host: the sink link is trained by the * host's DPTX through the tunnel; keep this DP OUT adapter - * from starting link training on its own, and hand it back - * once the tunnel is gone. + * from starting link training on its own. The bit is left + * as it is when the tunnel goes: the next tunnel to this + * adapter sets it again before enabling it. */ if (active && apple_dpin && !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { @@ -1619,13 +1653,6 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) tb_port_warn(out, "Apple: cannot hold off DP link training\n"); } ret = tb_dp_port_enable(out, active); - /* hand link training back even if the disable failed */ - if (!active && apple_dpin && - !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { - v &= ~ADP_DP_CS_3_NO_AUTO_LT; - if (tb_port_write(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) - tb_port_warn(out, "Apple: cannot restore DP link training\n"); - } if (ret && active) return ret; } diff --git a/drivers/tty/serial/apple_dockchannel.h b/drivers/tty/serial/apple_dockchannel.h new file mode 100644 index 0000000000000..97fcccc6a5a57 --- /dev/null +++ b/drivers/tty/serial/apple_dockchannel.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple DockChannel UART: console output shared by the earlycon and the driver. + */ +#ifndef _APPLE_DOCKCHANNEL_H +#define _APPLE_DOCKCHANNEL_H + +#include +#include +#include + +#define DOCKCHANNEL_TX8 0x04 +#define DOCKCHANNEL_TX_FREE 0x14 +#define DOCKCHANNEL_TX_RETRIES 10000 + +/* + * Whether console and tty output never wait for a reader: the DT property + * apple,tx-never-wait on the UART node, which only J775D (t6050-j775d.dts) + * sets, where unattended boots run with nothing reading the channel. Every + * other DockChannel Mac keeps waiting for room as before. Before widening + * it (or making it the default), check on one other DockChannel Mac with a + * slow attached reader (m1n1 proxy, USB debug) that long printk lines and tty + * output still arrive whole. + */ +#define DOCKCHANNEL_NEVER_WAIT "apple,tx-never-wait" + +/* Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. */ +static inline void dockchannel_wait_putc(void __iomem *data, unsigned char c) +{ + unsigned int retries = DOCKCHANNEL_TX_RETRIES; + + while (readl(data + DOCKCHANNEL_TX_FREE) == 0 && --retries) + cpu_relax(); + if (!retries) + return; + + writel(c, data + DOCKCHANNEL_TX8); +} + +/* + * With DOCKCHANNEL_NEVER_WAIT: the channel drains only while a debug USB host + * reads it; with none, the TX FIFO stays full for good. So console output + * never waits for room: each + * write sends what TX_FREE says fits and drops the rest. One write reads + * TX_FREE again when the room it read runs out, and stops reading once it + * reads 0. Callers serialize writes, so the room read stays true. + */ +struct dockchannel_tx { + u32 room; + bool full; +}; + +static inline void dockchannel_tx_begin(struct dockchannel_tx *tx) +{ + tx->room = 0; + tx->full = false; +} + +static inline void dockchannel_console_putc(void __iomem *data, + struct dockchannel_tx *tx, + unsigned char c) +{ + if (!tx->room && !tx->full) { + tx->room = readl(data + DOCKCHANNEL_TX_FREE); + tx->full = !tx->room; + } + if (!tx->room) + return; + tx->room--; + writel(c, data + DOCKCHANNEL_TX8); +} + +#endif diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 1048dde41d794..6c427189d34be 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -8,22 +8,33 @@ #include #include +#include +#include #include -#define DOCKCHANNEL_TX8 0x04 -#define DOCKCHANNEL_TX_FREE 0x14 -#define DOCKCHANNEL_TX_RETRIES 10000 +#include "apple_dockchannel.h" + +static struct dockchannel_tx dockchannel_early_tx; +static bool dockchannel_early_never_wait; static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) { - unsigned int retries = DOCKCHANNEL_TX_RETRIES; + if (dockchannel_early_never_wait) + dockchannel_console_putc(port->membase, &dockchannel_early_tx, c); + else + dockchannel_wait_putc(port->membase, c); +} - while (readl(port->membase + DOCKCHANNEL_TX_FREE) == 0 && --retries) - cpu_relax(); - if (!retries) - return; +/* The DT's DockChannel UART node asks for DOCKCHANNEL_NEVER_WAIT (flat DT). */ +static bool __init dockchannel_uart_early_never_wait(void) +{ + const void *fdt = initial_boot_params; + int node; - writel(c, port->membase + DOCKCHANNEL_TX8); + if (!fdt) + return false; + node = fdt_node_offset_by_compatible(fdt, -1, "apple,dockchannel-uart"); + return node >= 0 && fdt_getprop(fdt, node, DOCKCHANNEL_NEVER_WAIT, NULL); } static void dockchannel_uart_early_write(struct console *con, const char *s, @@ -31,6 +42,7 @@ static void dockchannel_uart_early_write(struct console *con, const char *s, { struct earlycon_device *dev = con->data; + dockchannel_tx_begin(&dockchannel_early_tx); uart_console_write(&dev->port, s, n, dockchannel_uart_putc); } @@ -42,6 +54,7 @@ static int __init dockchannel_uart_early_setup(struct earlycon_device *device, if (!port->mapbase) return -ENODEV; + dockchannel_early_never_wait = dockchannel_uart_early_never_wait(); device->con->write = dockchannel_uart_early_write; return 0; } diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index c015d916ac432..d4255cb949848 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -16,6 +16,8 @@ #include #include +#include "apple_dockchannel.h" + #define FIFO_SIZE 0x800 #define IRQ_MASK 0x0 @@ -44,7 +46,13 @@ * that is about 115 bytes per poll, far below the FIFO size. */ #define SW_DRAIN_MS 10 -#define TX_RETRIES 10000 +/* + * TX never waits for FIFO room: what does not fit stays in the xmit buffer + * and the timer sends it as room appears. After this many ticks with no room + * at all, nothing is reading the channel: drop what is queued, so writers do + * not block on a reader that may never come. + */ +#define TX_STUCK_TICKS 2 struct dcuart_port { struct uart_port port; @@ -57,6 +65,12 @@ struct dcuart_port { struct timer_list drain_timer; /* Serializes IRQ mask updates and RX FIFO draining. */ spinlock_t lock; + /* DOCKCHANNEL_NEVER_WAIT: TX never waits for room (J775D only). */ + bool never_wait; + /* One console write's view of the TX FIFO (dockchannel_console_putc). */ + struct dockchannel_tx console_tx; + /* Timer ticks in a row that found TX output queued and no FIFO room. */ + unsigned int tx_stuck_ticks; }; static struct uart_driver dcuart_drv; @@ -97,22 +111,38 @@ static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) static void dcuart_putc(struct uart_port *port, unsigned char c) { - unsigned int retries = TX_RETRIES; + struct dcuart_port *dcuart = container_of(port, struct dcuart_port, port); - while (readl(port->membase + DATA_TX_FREE) == 0 && --retries) - cpu_relax(); - if (!retries) - return; - - writel(c, port->membase + DATA_TX8); + if (dcuart->never_wait) + dockchannel_console_putc(port->membase, &dcuart->console_tx, c); + else + dockchannel_wait_putc(port->membase, c); } static void dcuart_write(struct console *co, const char *s, unsigned int n) { struct dcuart_port *dcuart = dcuart_ports[co->index]; + struct uart_port *port = &dcuart->port; + unsigned long flags; + bool locked = true; + + if (!dcuart->never_wait) { + uart_console_write(port, s, n, dcuart_putc); + return; + } + + /* The TX path writes the same FIFO, and the room read must stay true. */ + if (oops_in_progress) + locked = uart_port_trylock_irqsave(port, &flags); + else + uart_port_lock_irqsave(port, &flags); + + dockchannel_tx_begin(&dcuart->console_tx); + uart_console_write(port, s, n, dcuart_putc); - uart_console_write(&dcuart->port, s, n, dcuart_putc); + if (locked) + uart_port_unlock_irqrestore(port, flags); } static int dcuart_console_setup(struct console *co, char *options) @@ -198,39 +228,106 @@ static irqreturn_t dcuart_interrupt(int irq, void *data) return IRQ_HANDLED; } +/* + * Send what the TX FIFO has room for, without waiting. From the timer (tick), + * also count ticks without room and drop the queue after TX_STUCK_TICKS. + * Called with the port lock held. Returns the bytes left queued. + */ +static unsigned int dcuart_tx_chars(struct dcuart_port *dcuart, bool tick) +{ + struct uart_port *port = &dcuart->port; + struct tty_port *tport = &port->state->port; + unsigned int queued = kfifo_len(&tport->xmit_fifo), pending; + u8 ch; + + if (!queued && !port->x_char) + return 0; + + pending = uart_port_tx_limited(port, ch, readl(port->membase + DATA_TX_FREE), + true, writel(ch, port->membase + DATA_TX8), ({})); + if (!pending || pending < queued) { + dcuart->tx_stuck_ticks = 0; + return pending; + } + if (!tick || ++dcuart->tx_stuck_ticks < TX_STUCK_TICKS) + return pending; + + kfifo_reset_out(&tport->xmit_fifo); + dcuart->tx_stuck_ticks = 0; + uart_write_wakeup(port); + return 0; +} + static void dcuart_drain_timer(struct timer_list *t) { struct dcuart_port *dcuart = timer_container_of(dcuart, t, drain_timer); + struct uart_port *port = &dcuart->port; unsigned long flags; - bool got; + unsigned int pending; - spin_lock_irqsave(&dcuart->lock, flags); - got = dcuart_drain_rx(dcuart); - spin_unlock_irqrestore(&dcuart->lock, flags); + if (!dcuart->never_wait) { + bool got; - if (got) - tty_flip_buffer_push(&dcuart->port.state->port); - mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); + spin_lock_irqsave(&dcuart->lock, flags); + got = dcuart_drain_rx(dcuart); + spin_unlock_irqrestore(&dcuart->lock, flags); + + if (got) + tty_flip_buffer_push(&port->state->port); + mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); + return; + } + + if (dcuart->sw_drain) { + bool got; + + spin_lock_irqsave(&dcuart->lock, flags); + got = dcuart_drain_rx(dcuart); + spin_unlock_irqrestore(&dcuart->lock, flags); + + if (got) + tty_flip_buffer_push(&port->state->port); + } + + uart_port_lock_irqsave(port, &flags); + pending = dcuart_tx_chars(dcuart, true); + uart_port_unlock_irqrestore(port, flags); + + if (dcuart->sw_drain || pending) + mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); } +/* + * Bytes in the FIFO stay there for a host to read whenever it comes; a full + * FIFO means none is reading, not that a transmission is in progress. So + * report empty: a close or tcdrain must not wait for a reader (it waited + * twice the port timeout, 0.4 s, on every close of an unread channel). + */ static unsigned int dcuart_tx_empty(struct uart_port *port) { - return (readl(port->membase + DATA_TX_FREE) != 0) ? - TIOCSER_TEMT : - 0; + if (!port_to_dc(port)->never_wait) + return (readl(port->membase + DATA_TX_FREE) != 0) ? TIOCSER_TEMT : 0; + return TIOCSER_TEMT; } static void dcuart_start_tx(struct uart_port *port) { - char ch; - struct tty_port *tport = &port->state->port; + struct dcuart_port *dcuart = port_to_dc(port); + + if (!dcuart->never_wait) { + struct tty_port *tport = &port->state->port; + char ch; - if (!kfifo_is_empty(&tport->xmit_fifo)) { while (kfifo_get(&tport->xmit_fifo, &ch)) { dcuart_putc(port, ch); port->icount.tx++; } + return; } + + /* The rest goes from the timer, which the RX poll keeps running anyway. */ + if (dcuart_tx_chars(dcuart, false) && !timer_pending(&dcuart->drain_timer)) + mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); } static void dcuart_stop_tx(struct uart_port *port) @@ -258,6 +355,14 @@ static int dcuart_startup(struct uart_port *port) struct dcuart_port *dcuart = port_to_dc(port); unsigned long flags; + /* + * A close waits for the xmit buffer to empty, which on an unread + * channel takes until the timer drops it (systemd closes the console + * after every status line: 30 ms each). Never wait for a reader. + */ + if (dcuart->never_wait) + port->state->port.closing_wait = ASYNC_CLOSING_WAIT_NONE; + spin_lock_irqsave(&dcuart->lock, flags); dcuart_irq_disable(dcuart, dcuart->irq_tx | dcuart->irq_rx); writel_relaxed(1, dcuart->config_base + CONFIG_RX_THRESH); @@ -278,9 +383,10 @@ static void dcuart_shutdown(struct uart_port *port) struct dcuart_port *dcuart = port_to_dc(port); unsigned long flags; - if (dcuart->sw_drain) + /* With never_wait the timer also sends TX in IRQ mode. */ + if (dcuart->sw_drain || dcuart->never_wait) timer_delete_sync(&dcuart->drain_timer); - else + if (!dcuart->sw_drain) disable_irq(port->irq); spin_lock_irqsave(&dcuart->lock, flags); @@ -376,6 +482,7 @@ static int dcuart_probe(struct platform_device *pdev) dcuart->irq_tx = IRQ_TX(channel); dcuart->irq_rx = IRQ_RX(channel); dcuart->sw_drain = of_property_read_bool(pdev->dev.of_node, "apple,sw-drain"); + dcuart->never_wait = of_property_read_bool(pdev->dev.of_node, DOCKCHANNEL_NEVER_WAIT); timer_setup(&dcuart->drain_timer, dcuart_drain_timer, 0); dcuart->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f16aa6f21014b..f66370e6b5f88 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -100,7 +100,8 @@ * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ -#define DWC3_APPLE_NR_TUNABLES 5 +#define DWC3_APPLE_NR_TUNABLES 8 +#define DWC3_APPLE_NR_DRD_REGS 3 /** * struct dwc3_apple - Apple-specific DWC3 USB controller @@ -127,7 +128,8 @@ * @rpm_held: The glue holds a runtime PM reference for the running core * @pre_core_rpm: Runtime PM is the glue's own until the first core bring-up * @dt_dis_u3_susphy: The device tree keeps USB3 PHY suspend off itself - * @drd_cfg: The controller's configuration block (optional, for its tunable) + * @drd_regs: The controller's optional blocks outside dwc3-core (drd-cfg, drd-bulk, drd-ausbc), + * mapped for their tunables * @tunables: Controller tunables from the device tree, see dwc3_apple_tunable_list */ struct dwc3_apple { @@ -158,7 +160,7 @@ struct dwc3_apple { bool pre_core_rpm; bool dt_dis_u3_susphy; - void __iomem *drd_cfg; + void __iomem *drd_regs[DWC3_APPLE_NR_DRD_REGS]; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; }; @@ -232,22 +234,43 @@ enum dwc3_apple_tunable_block { DWC3_APPLE_TUNABLE_GLOBALS, /* DWC3_GLOBALS_REGS_START */ DWC3_APPLE_TUNABLE_LINK, /* dwc3-core + 0xd000 */ DWC3_APPLE_TUNABLE_DBG, /* dwc3-core + 0xd800 */ + DWC3_APPLE_TUNABLE_CIO, /* dwc3-core + 0xcd20, in dwc3-apple */ + /* The optional blocks outside dwc3-core, in drd_regs[] order */ DWC3_APPLE_TUNABLE_DRD_CFG, /* drd-cfg */ + DWC3_APPLE_TUNABLE_DRD_BULK, /* drd-bulk */ + DWC3_APPLE_TUNABLE_DRD_AUSBC, /* drd-ausbc */ + DWC3_APPLE_NR_TUNABLE_BLOCKS }; +static const char * const dwc3_apple_drd_reg_names[DWC3_APPLE_NR_DRD_REGS] = { + "drd-cfg", "drd-bulk", "drd-ausbc", +}; + +static_assert(DWC3_APPLE_NR_TUNABLE_BLOCKS - DWC3_APPLE_TUNABLE_DRD_CFG == DWC3_APPLE_NR_DRD_REGS); + #define DWC3_APPLE_LINK_REGS_START 0xd000 #define DWC3_APPLE_DBG_REGS_START 0xd800 #define DWC3_APPLE_LINK_DBG_SIZE 0x800 +/* The CIO registers, the fixed ones of dwc3_apple_setup_cio() included */ +#define DWC3_APPLE_CIO_REGS_START 0xcd20 +/* + * In the order they are applied. A table is optional, and one for a block that the device tree + * does not describe is refused. The CIO table follows dwc3_apple_setup_cio() and overrides its + * fixed values (T6050). + */ static const struct { const char *prop; enum dwc3_apple_tunable_block block; } dwc3_apple_tunable_list[] = { + { "apple,tunable-bulk", DWC3_APPLE_TUNABLE_DRD_BULK }, { "apple,tunable-gbl-default", DWC3_APPLE_TUNABLE_GLOBALS }, { "apple,tunable-gbl-host", DWC3_APPLE_TUNABLE_GLOBALS }, + { "apple,tunable-cio", DWC3_APPLE_TUNABLE_CIO }, { "apple,tunable-dbg", DWC3_APPLE_TUNABLE_DBG }, { "apple,tunable-cfg-host", DWC3_APPLE_TUNABLE_DRD_CFG }, { "apple,tunable-link", DWC3_APPLE_TUNABLE_LINK }, + { "apple,tunable-ausbc", DWC3_APPLE_TUNABLE_DRD_AUSBC }, }; static_assert(ARRAY_SIZE(dwc3_apple_tunable_list) == DWC3_APPLE_NR_TUNABLES); @@ -262,8 +285,14 @@ static void __iomem *dwc3_apple_tunable_base(struct dwc3_apple *appledwc, return appledwc->apple_regs + DWC3_APPLE_LINK_REGS_START - APPLE_DWC3_REGS_START; case DWC3_APPLE_TUNABLE_DBG: return appledwc->apple_regs + DWC3_APPLE_DBG_REGS_START - APPLE_DWC3_REGS_START; + case DWC3_APPLE_TUNABLE_CIO: + return appledwc->apple_regs + DWC3_APPLE_CIO_REGS_START - APPLE_DWC3_REGS_START; case DWC3_APPLE_TUNABLE_DRD_CFG: - return appledwc->drd_cfg; + case DWC3_APPLE_TUNABLE_DRD_BULK: + case DWC3_APPLE_TUNABLE_DRD_AUSBC: + return appledwc->drd_regs[block - DWC3_APPLE_TUNABLE_DRD_CFG]; + case DWC3_APPLE_NR_TUNABLE_BLOCKS: + break; } return NULL; } @@ -281,8 +310,9 @@ static void dwc3_apple_apply_tunables(struct dwc3_apple *appledwc) /* The tunables are optional; a malformed one fails the probe. */ static int dwc3_apple_parse_tunables(struct dwc3_apple *appledwc, struct platform_device *pdev) { - struct resource *core = appledwc->mmio_resource, *drd_cfg; - struct resource blocks[] = { + struct resource *core = appledwc->mmio_resource; + /* A block the device tree does not describe stays empty, and its table is refused. */ + struct resource blocks[DWC3_APPLE_NR_TUNABLE_BLOCKS] = { [DWC3_APPLE_TUNABLE_GLOBALS] = DEFINE_RES_MEM(core->start + DWC3_GLOBALS_REGS_START, DWC3_GLOBALS_REGS_END - DWC3_GLOBALS_REGS_START + 1), @@ -290,17 +320,24 @@ static int dwc3_apple_parse_tunables(struct dwc3_apple *appledwc, struct platfor DWC3_APPLE_LINK_DBG_SIZE), [DWC3_APPLE_TUNABLE_DBG] = DEFINE_RES_MEM(core->start + DWC3_APPLE_DBG_REGS_START, DWC3_APPLE_LINK_DBG_SIZE), - [DWC3_APPLE_TUNABLE_DRD_CFG] = DEFINE_RES_MEM(0, 0), + [DWC3_APPLE_TUNABLE_CIO] = DEFINE_RES_MEM(core->start + DWC3_APPLE_CIO_REGS_START, + APPLE_DWC3_REGS_END - + DWC3_APPLE_CIO_REGS_START + 1), }; struct device *dev = appledwc->dev; int i; - drd_cfg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "drd-cfg"); - if (drd_cfg) { - appledwc->drd_cfg = devm_ioremap_resource(dev, drd_cfg); - if (IS_ERR(appledwc->drd_cfg)) - return PTR_ERR(appledwc->drd_cfg); - blocks[DWC3_APPLE_TUNABLE_DRD_CFG] = *drd_cfg; + for (i = 0; i < DWC3_APPLE_NR_DRD_REGS; i++) { + struct resource *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + dwc3_apple_drd_reg_names[i]); + if (!res) + continue; + appledwc->drd_regs[i] = devm_ioremap_resource(dev, res); + if (IS_ERR(appledwc->drd_regs[i])) + return PTR_ERR(appledwc->drd_regs[i]); + blocks[DWC3_APPLE_TUNABLE_DRD_CFG + i] = *res; } for (i = 0; i < ARRAY_SIZE(dwc3_apple_tunable_list); i++) { @@ -628,9 +665,16 @@ static int dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_stat dwc3_enable_susphy(&appledwc->dwc, true); ret = phy_set_mode(appledwc->usb3_phy, host ? PHY_MODE_USB_HOST : PHY_MODE_USB_DEVICE); if (ret) { - /* A failed T8132 BIST or PIPE handoff must not start xHCI. */ - if (of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3")) + /* + * A failed T8132 BIST or PIPE handoff must not start xHCI. Neither must a failed + * T6050 one: that PHY reports a port left on USB2 (four DisplayPort lanes) as + * success, and its errors leave the PIPE without a known producer. + */ + if (of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3") || + of_device_is_compatible(appledwc->dev->of_node, "apple,t6050-dwc3")) { + dev_err(appledwc->dev, "USB3 PHY setup failed, err=%d\n", ret); return ret; + } dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); } else if (host && appledwc->usb3_phy && of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3")) { @@ -648,18 +692,19 @@ static int dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_stat } /* - * No device mode: the T6040 USB2 PHY implements host mode only (phy-apple-t6040-usb2), and a - * kernel built without gadget support turns dwc3_gadget_init() into a stub that succeeds. Such a - * controller must not follow a USB_ROLE_DEVICE request from the Type-C port controller (a charger - * or a host on the other end), or the core would end up in device mode on top of a PHY powered - * in host mode. + * No device mode: the T6040 USB2 PHY (phy-apple-t6040-usb2) and the T6050 Type-C PHY implement + * host mode only, and a kernel built without gadget support turns dwc3_gadget_init() into a stub + * that succeeds. Such a controller must not follow a USB_ROLE_DEVICE request from the Type-C port + * controller (a charger or a host on the other end), or the core would end up in device mode on + * top of a PHY powered in host mode. */ static bool dwc3_apple_host_only(struct device *dev) { if (!IS_ENABLED(CONFIG_USB_DWC3_GADGET) && !IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE)) return true; - return of_device_is_compatible(dev->of_node, "apple,t6040-dwc3"); + return of_device_is_compatible(dev->of_node, "apple,t6040-dwc3") || + of_device_is_compatible(dev->of_node, "apple,t6050-dwc3"); } static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) @@ -691,6 +736,16 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st dev_err(appledwc->dev, "Failed to initialize host, ret=%d\n", ret); goto core_exit; } + /* + * A USB2 PHY with a port reset (the T6050 Type-C PHY) takes it once the root hub is + * up; for the other PHYs, which have none, this is a no-op. + */ + ret = phy_reset(appledwc->usb2_phy); + if (ret) { + dev_err(appledwc->dev, "Failed to reset the USB2 port, ret=%d\n", ret); + dwc3_host_exit(&appledwc->dwc); + goto core_exit; + } break; case DWC3_APPLE_DEVICE: ret = dwc3_gadget_init(&appledwc->dwc); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 74c820dd0be24..25325dde72a8c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -2017,7 +2017,14 @@ static int cd321x_switch_power_state(struct tps6598x *tps, u8 target_state) if (state == target_state) return 0; - ret = tps6598x_exec_cmd(tps, "SSPS", sizeof(u8), &target_state, 0, NULL); + /* SN201202x can still be negotiating an attached dock while waking. */ + if (target_state == TPS_SYSTEM_POWER_STATE_S0 && + device_is_compatible(tps->dev, "apple,sn201202x")) + ret = tps6598x_exec_cmd_tmo(tps, "SSPS", sizeof(target_state), + &target_state, 0, NULL, 5000, 0); + else + ret = tps6598x_exec_cmd(tps, "SSPS", sizeof(target_state), + &target_state, 0, NULL); if (ret) return ret; @@ -2905,6 +2912,66 @@ static int cd321x_remove(struct tps6598x *tps) return 0; } +static int cd321x_reset_inherited_connection(struct tps6598x *tps, u32 *status) +{ + /* J716c's port-controller key is the reversed target identifier J716. */ + static const u8 key[] = { '6', '1', '7', 'J' }; + static const u8 locked[4]; + u8 delay = 5, result; + unsigned long timeout; + int ret, lock_ret, mode; + + if (!device_property_read_bool(tps->dev, "apple,reset-inherited-usb2-connection") || + !of_machine_is_compatible("apple,j716c") || + !device_is_compatible(tps->dev, "apple,sn201202x") || + tps->dbma_at_probe || !(*status & TPS_STATUS_PLUG_PRESENT) || + !TPS_STATUS_TO_TYPEC_DATAROLE(*status)) + return 0; + mode = tps6598x_check_mode(tps); + if (mode != TPS_MODE_APP && mode != TPS_MODE_APP1) + return 0; + + /* All suppliers are available, but no USB or router session has started. */ + ret = tps6598x_exec_cmd(tps, "LOCK", sizeof(key), key, sizeof(result), &result); + if (ret) + return ret; + if (result) { + ret = -EIO; + goto relock; + } + + /* + * Nonzero DISC delay restores normal CC detection automatically. The + * task can take more than a second to acknowledge at boot; wait for it + * before issuing LOCK, which a still-busy controller would reject. + */ + ret = tps6598x_exec_cmd_tmo(tps, "DISC", sizeof(delay), &delay, 0, NULL, + 5000, 0); + if (!ret) { + msleep(5100); + timeout = jiffies + msecs_to_jiffies(2000); + do { + mode = tps6598x_check_mode(tps); + if (mode == TPS_MODE_APP || mode == TPS_MODE_APP1) + break; + msleep(20); + } while (time_before(jiffies, timeout)); + if (mode != TPS_MODE_APP && mode != TPS_MODE_APP1) + ret = mode < 0 ? mode : -ETIMEDOUT; + } +relock: + /* CC/PD negotiation may still be finishing when application mode returns. */ + lock_ret = tps6598x_exec_cmd_tmo(tps, "LOCK", sizeof(locked), locked, 0, NULL, + 5000, 0); + if (ret || lock_ret) + return ret ?: lock_ret; + if (!tps6598x_read_status(tps, status)) + return -EIO; + tps->status = *status; + dev_info(tps->dev, "reinitialized inherited USB2 connection\n"); + return 0; +} + int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; @@ -3011,12 +3078,17 @@ int tipd_init(struct tps6598x *tps) if (tps->contaminant_ops) { ret = tps->contaminant_ops->update(tps); if (ret) { - dev_warn(tps->dev, - "connector contaminant state unavailable: %d\n", ret); + if (ret != -EINPROGRESS) + dev_warn(tps->dev, + "connector contaminant state unavailable: %d\n", ret); ret = 0; } } + ret = cd321x_reset_inherited_connection(tps, &status); + if (ret) + goto err_disconnect; + if (status & TPS_STATUS_PLUG_PRESENT) { ret = -EINVAL; if (!tps->data->read_power_status(tps)) diff --git a/drivers/usb/typec/tipd/sn201202x-pm-test.c b/drivers/usb/typec/tipd/sn201202x-pm-test.c index 68e36f6d951c8..7fab562ea81d8 100644 --- a/drivers/usb/typec/tipd/sn201202x-pm-test.c +++ b/drivers/usb/typec/tipd/sn201202x-pm-test.c @@ -826,6 +826,57 @@ static void sn201202x_ldcm_dry_order_test(struct kunit *test) sn201202x_ldcm_test_actions(test, &t, actions, ARRAY_SIZE(actions)); } +static void sn201202x_ldcm_dry_then_busy_test(struct kunit *test) +{ + static const enum typec_contaminant last[] = { + TYPEC_CONTAMINANT_UNKNOWN, + TYPEC_CONTAMINANT_NONE, + TYPEC_CONTAMINANT_LIQUID, + }; + const enum sn201202x_ldcm_test_action actions[] = { + LDCM_TEST_READ, + LDCM_TEST_MITIGATE_OFF, + LDCM_TEST_READ, + LDCM_TEST_PUBLISH_NONE, + }; + unsigned int i; + + /* Removing mitigation can start a measurement after a fresh dry result. */ + for (i = 0; i < ARRAY_SIZE(last); i++) { + struct sn201202x_ldcm_test t = { + .reads = { TYPEC_CONTAMINANT_NONE, TYPEC_CONTAMINANT_UNKNOWN }, + }; + + KUNIT_EXPECT_EQ(test, sn201202x_ldcm_update_state(&t, + &sn201202x_ldcm_test_ops, + last[i]), + -EINPROGRESS); + sn201202x_ldcm_test_actions(test, &t, actions, + ARRAY_SIZE(actions)); + } +} + +static void sn201202x_ldcm_dry_then_read_failure_test(struct kunit *test) +{ + const enum sn201202x_ldcm_test_action actions[] = { + LDCM_TEST_READ, + LDCM_TEST_MITIGATE_OFF, + LDCM_TEST_READ, + LDCM_TEST_MITIGATE_ON, + LDCM_TEST_PUBLISH_UNKNOWN, + }; + struct sn201202x_ldcm_test t = { + .reads = { TYPEC_CONTAMINANT_NONE, TYPEC_CONTAMINANT_UNKNOWN }, + .read_rets = { 0, -EIO }, + }; + + /* A failed second read must still mitigate, even after a dry result. */ + KUNIT_EXPECT_EQ(test, sn201202x_ldcm_update_state(&t, + &sn201202x_ldcm_test_ops, + TYPEC_CONTAMINANT_UNKNOWN), -EIO); + sn201202x_ldcm_test_actions(test, &t, actions, ARRAY_SIZE(actions)); +} + static void sn201202x_ldcm_dry_race_test(struct kunit *test) { const enum sn201202x_ldcm_test_action actions[] = { @@ -931,6 +982,55 @@ static void sn201202x_ldcm_read_failure_with_result_test(struct kunit *test) sn201202x_ldcm_test_actions(test, &t, actions, ARRAY_SIZE(actions)); } +struct sn201202x_ldcm_stop_test { + struct sn201202x sn; + atomic_t runs; +}; + +static void sn201202x_ldcm_test_work(struct work_struct *work) +{ + struct sn201202x *sn = container_of(to_delayed_work(work), + struct sn201202x, ldcm_work); + struct sn201202x_ldcm_stop_test *t = container_of(sn, + struct sn201202x_ldcm_stop_test, sn); + + atomic_inc(&t->runs); +} + +static void sn201202x_ldcm_test_cancel(void *data) +{ + cancel_delayed_work_sync(data); +} + +static void sn201202x_ldcm_stop_requeue_test(struct kunit *test) +{ + struct sn201202x_ldcm_stop_test *t; + struct tps6598x *tps; + int ret; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + tps = &t->sn.cd.tps; + mutex_init(&tps->lock); + atomic_set(&t->runs, 0); + INIT_DELAYED_WORK(&t->sn.ldcm_work, sn201202x_ldcm_test_work); + ret = kunit_add_action_or_reset(test, sn201202x_ldcm_test_cancel, &t->sn.ldcm_work); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* Removal must cancel a pending retry and prevent later producers. */ + mod_delayed_work(system_freezable_wq, &t->sn.ldcm_work, 60 * HZ); + sn201202x_contaminant_ops.stop(tps); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->sn.ldcm_work)); + + /* Reset recovery can still invalidate the detector before it is drained. */ + mutex_lock(&tps->lock); + sn201202x_contaminant_ops.invalidate(tps); + mutex_unlock(&tps->lock); + flush_delayed_work(&t->sn.ldcm_work); + KUNIT_EXPECT_EQ(test, atomic_read(&t->runs), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->sn.ldcm_work)); +} + static struct kunit_case sn201202x_pm_cases[] = { KUNIT_CASE(sn201202x_sleep_cycle_test), KUNIT_CASE(sn201202x_failed_wakeup_test), @@ -945,11 +1045,14 @@ static struct kunit_case sn201202x_pm_cases[] = { KUNIT_CASE(sn201202x_ldcm_read_failure_order_test), KUNIT_CASE(sn201202x_ldcm_mitigation_failure_order_test), KUNIT_CASE(sn201202x_ldcm_dry_order_test), + KUNIT_CASE(sn201202x_ldcm_dry_then_busy_test), + KUNIT_CASE(sn201202x_ldcm_dry_then_read_failure_test), KUNIT_CASE(sn201202x_ldcm_dry_race_test), KUNIT_CASE(sn201202x_ldcm_dry_race_mitigation_failure_test), KUNIT_CASE(sn201202x_ldcm_busy_keeps_result_test), KUNIT_CASE(sn201202x_ldcm_busy_without_result_test), KUNIT_CASE(sn201202x_ldcm_read_failure_with_result_test), + KUNIT_CASE(sn201202x_ldcm_stop_requeue_test), KUNIT_CASE(sn201202x_idle_sleep_test), KUNIT_CASE(sn201202x_atomic_access_test), KUNIT_CASE(sn201202x_idle_sleep_retry_test), diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 32c3778871013..30d472c3d83a4 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -300,9 +300,13 @@ static int sn201202x_ldcm_set_mitigation(void *ctx, bool enable) if (sn->ldcm_mitigation_known && sn->ldcm_mitigated == enable) return 0; - /* Successful task completion is the controller's mitigation ACK. */ + /* + * Successful task completion is the controller's mitigation ACK. + * Startup commands can take more than a second on SN201202x. Wait for + * completion before a retry can request the opposite mitigation state. + */ ret = tps6598x_exec_cmd_tmo(tps, "LDCM", sizeof(command), &command, - 0, NULL, 1000, 0); + 0, NULL, 5000, 0); sn->ldcm_mitigation_known = !ret; if (!ret) sn->ldcm_mitigated = enable; @@ -354,6 +358,16 @@ sn201202x_ldcm_update_state(void *ctx, const struct sn201202x_ldcm_ops *ops, /* Do not report dry if liquid appeared while mitigation was removed. */ read_ret = ops->read(ctx, &contaminant); + if (!read_ret && contaminant == TYPEC_CONTAMINANT_UNKNOWN) { + /* + * Removing mitigation can start another measurement. The dry + * snapshot above is still the latest result; preserve it just + * as we preserve a previously published result at entry. Forcing + * mitigation back on here can prevent the measurement finishing. + */ + ops->publish(ctx, TYPEC_CONTAMINANT_NONE); + return -EINPROGRESS; + } if (read_ret || contaminant != TYPEC_CONTAMINANT_NONE) { ret = ops->set_mitigation(ctx, true); if (ret) { @@ -453,16 +467,18 @@ static void sn201202x_ldcm_invalidate(struct tps6598x *tps) static void sn201202x_ldcm_stop(struct tps6598x *tps) { - cancel_delayed_work_sync(&tps_to_sn(tps)->ldcm_work); + /* Reset recovery is drained later and may still try to queue an update. */ + disable_delayed_work_sync(&tps_to_sn(tps)->ldcm_work); } -static const struct tipd_contaminant_ops sn201202x_contaminant_ops = { +VISIBLE_IF_KUNIT const struct tipd_contaminant_ops sn201202x_contaminant_ops = { .update = sn201202x_ldcm_update, .refresh = sn201202x_ldcm_refresh, .event = sn201202x_ldcm_event, .invalidate = sn201202x_ldcm_invalidate, .stop = sn201202x_ldcm_stop, }; +EXPORT_SYMBOL_IF_KUNIT(sn201202x_contaminant_ops); static struct regmap *__devm_regmap_init_sn201202x(struct spmi_device *sdev, const struct regmap_config *config, diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c39feab87c20f..20b6b73436a29 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -564,6 +564,7 @@ int sn201202x_runtime_resume(struct device *dev); void sn201202x_idle_sleep_start(struct tps6598x *tps); void sn201202x_idle_sleep_stop(struct tps6598x *tps); extern const struct regmap_bus regmap_sn201202x; +extern const struct tipd_contaminant_ops sn201202x_contaminant_ops; u32 sn201202x_sleep_links(const struct cd321x_sleep_state *st); enum typec_contaminant sn201202x_ldcm_decode(const u8 status[SN201202X_LDCM_STATUS_LEN]); diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index fbc047b00afd8..604c54b0b3274 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -267,6 +267,23 @@ APPLE_SMC_TYPE_OPS(s32) APPLE_SMC_TYPE_OPS(s16) APPLE_SMC_TYPE_OPS(s8) +struct device; + +/* + * The SMC hwmon cell's fan control for a kernel thermal policy (a GPU + * governor): see drivers/hwmon/macsmc-hwmon.c. + */ +#if IS_REACHABLE(CONFIG_SENSORS_MACSMC_HWMON) +int macsmc_hwmon_fans_request(struct device *dev, unsigned int permille, + unsigned int min_step_rpm); +#else +static inline int macsmc_hwmon_fans_request(struct device *dev, unsigned int permille, + unsigned int min_step_rpm) +{ + return -ENODEV; +} +#endif + static inline int apple_smc_read_flag(struct apple_smc *smc, smc_key key, bool *flag) { u8 val; diff --git a/include/linux/pci.h b/include/linux/pci.h index 5cec6d838b3f7..c0d76d93a78f5 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1559,6 +1559,7 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev); /* Functions for PCI Hotplug drivers to use */ unsigned int pci_rescan_bus(struct pci_bus *bus); void pci_lock_rescan_remove(void); +bool pci_trylock_rescan_remove(void); void pci_unlock_rescan_remove(void); /* Vital Product Data routines */ diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index ce81d6ec5bcd3..2c166c70dfc68 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -141,12 +141,19 @@ int apple_dart_save_tunnel_state(struct device *dev); int apple_dart_restore_tunnel_state(struct device *dev); /* Registered J713 managed radio provider, with no owned valid root or fault. */ int apple_dart_radio_check_idle(struct device *dev); +/* A firmware-owned buffer's address for a host copy; Linux's own mappings refused. */ +int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, phys_addr_t *phys); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_radio_check_idle(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, + phys_addr_t *phys) +{ + return -EOPNOTSUPP; +} #endif #endif diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 1597e338eca45..034a74b852ebc 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -19,7 +19,7 @@ struct phy; /* * appledrm: route a display pipeline to (active) or away from (!active) the - * crossbar output of DP IN adapter @dpin (0/1) of the router wired to + * crossbar output of DP IN adapter @dpin (0-3) of the router wired to * @connector_np. @set_active(@binding, active) runs the DP IN adapter's * DPTX_INACTIVE handshake and is called back from DCP's link activation. * Activation returns -EBUSY while no pipeline is free for the stream (or a @@ -38,6 +38,21 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, u64 generation, bool active, int (*set_active)(void *binding, bool active), void *binding); +/* The sink raised IRQ_HPD on the tunnel's DP IN adapter. */ +#define APPLE_DP_TUNNEL_IRQ_HPD BIT(0) +/* The HPD level of the tunnel's DP IN adapter changed to @hpd. */ +#define APPLE_DP_TUNNEL_HPD_CHANGED BIT(1) + +/* + * appledrm: pass interrupt @events (APPLE_DP_TUNNEL_*) of DP IN adapter + * @dpin of the router wired to @connector_np to the display pipeline that + * holds the tunnel handed over with @generation. The pipeline's DPTX port + * gets them as a DP source's HPD input does. Returns -ESTALE if no + * pipeline holds that tunnel. Does not wait for DCP; may sleep. + */ +int apple_dcp_tb_dp_event(struct device_node *connector_np, unsigned int dpin, + u64 generation, unsigned int events, bool hpd); + /* * ATC PHY: start the pixel clock for a DP tunnel at DP link rate code @rate, * or stop it (@rate == 0). The PHY must be in Thunderbolt/USB4 mode. @@ -90,6 +105,14 @@ static inline int apple_dp_tunnel_t8132_rate_ok(u8 rate, u8 other) int apple_atc_dp_tunnel_rate_source(struct phy *phy, unsigned int source, unsigned int dpin, u8 rate); +/* + * T6050 ATC PHY in USB4/TBT mode: run tunnel pixel clock @clock (1-4, which + * feeds the display crossbar's pixel clock @clock) at DP link rate code @rate + * (0x06 RBR, 0x0a HBR, 0x14 HBR2, 0x1e HBR3, 0x81 UHBR10, 0x82 UHBR20), or + * stop it (@rate == 0). A running clock holds its clock gate. + */ +int apple_t6050_dp_tunnel_clock(struct phy *phy, unsigned int clock, u8 rate); + /* * ATC PHY: wake the DP clock path of a PHY in Thunderbolt/USB4 mode ahead of * apple_atc_dp_tunnel_rate(), before DCP is told about the display. @@ -214,6 +237,32 @@ static inline bool apple_dp_tunnel_route_wired(struct device_node *route, return wired && apple_dp_tunnel_wiring_dual(endpoints, named[0], named[1], legacy); } +/* + * The graph port of USB-C connector @connector_np that carries its display + * routes: the first port with an endpoint leading to a route of a display + * pipeline's "typec-routes" node. Most boards put them on port 3; a board + * whose port 3 is taken by the USB4 router uses another one. Without such an + * endpoint, port 3, if present. Returns a reference to the port node, or NULL. + */ +static inline struct device_node * +apple_dp_tunnel_routes_port(struct device_node *connector_np) +{ + if (!connector_np) + return NULL; + for_each_of_graph_port(connector_np, port) { + for_each_of_graph_port_endpoint(port, endpoint) { + struct device_node *route __free(device_node) = + of_graph_get_remote_port_parent(endpoint); + struct device_node *routes __free(device_node) = + route ? of_get_parent(route) : NULL; + + if (routes && of_node_name_eq(routes, "typec-routes")) + return of_node_get(port); + } + } + return of_graph_get_port_by_id(connector_np, 3); +} + /* * DT wiring only: no DCP registration, firmware or display status. * Two candidates alone (e.g. j314) are single-stream. Named DPIN0/1 controls, @@ -228,7 +277,7 @@ static inline bool apple_dp_tunnel_dual_stream(struct device_node *connector_np) if (!connector_np) return false; - port = of_graph_get_port_by_id(connector_np, 3); + port = apple_dp_tunnel_routes_port(connector_np); for_each_child_of_node_scoped(port, endpoint) endpoints++; if (endpoints < 2) diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 68ff959be8cab..0502328425df6 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -666,6 +666,8 @@ struct hci_dev { struct bt_codec *codec, __u8 *vnd_len, __u8 **vnd_data); u8 (*classify_pkt_type)(struct hci_dev *hdev, struct sk_buff *skb); + /* Called in command-sync context before enabling duplicate-filtered scanning. */ + int (*reset_dup_filter)(struct hci_dev *hdev); }; #define hci_set_quirk(hdev, nr) set_bit((nr), (hdev)->quirk_flags) diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 55092da885071..4519da91ac0ca 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -371,11 +371,11 @@ enum drm_asahi_feature { DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, /* - * Bits 17 to 28 belong to the M5 (G17G/G17S) runtime, which reports - * them; this kernel never does. They are listed so that the numbers are - * not given to anything else, and userspace reads every kernel's - * features the same way. Bits 25 and 27 are also reserved for that - * runtime. + * Bits 17 to 28 belong to the M5 (G17G/G17S) command set. This kernel + * reports them only on M5 GPUs, and never bits 24 and 25 (see + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES). They are listed so that the + * numbers are not given to anything else, and userspace reads every + * kernel's features the same way. */ /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: M5: per-program render storage. */ @@ -404,16 +404,33 @@ enum drm_asahi_feature { /** * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: M5: a compute prefix for a - * render's tiling half. + * render's tiling half, in a shorter command form. Retired with + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES, for the same reason. */ DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, + /** + * @DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: M5: render commands accept + * DRM_ASAHI_RENDER_STAGE_PREFIXES in a shorter command form. Retired: no + * kernel may report this bit or bit 24 again. Userspace that sees either + * sends staged renders of 352 or 368 bytes with prefix fields at bytes + * 336 to 367, which struct drm_asahi_cmd_render gives to its ray-query + * stages. Staged renders use DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2. + */ + DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES = (1UL) << 25, + /** * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed * multisample stencil. */ DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, + /** + * @DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: M5: render commands accept + * DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS. + */ + DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS = (1UL) << 27, + /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, @@ -424,11 +441,19 @@ enum drm_asahi_feature { */ DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, + /** + * @DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS: Compute commands accept + * DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as an ordering hint. + */ + DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS = (1UL) << 30, + /* - * Bit 30 is used by the M5 runtime's userspace. Bits 31 and 32 belong - * to the G18 (M6, T8152) backend, which alone reports them. Bit 32 is + * Bit 30 above is used by the M5 runtime's userspace; the kernel does + * not report it. Bits 31 and 32 belong to the G18 (M6, T8152) backend, + * which alone reports them. Bit 32 is * DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES, defined after this enum so - * that the enum keeps its 32-bit type. + * that the enum keeps its 32-bit type. Bit 33, + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2, is defined there too. */ /** @@ -450,6 +475,14 @@ enum drm_asahi_feature { */ #define DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES ((1ULL) << 32) +/* + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: M5: render commands accept + * DRM_ASAHI_RENDER_STAGE_PREFIXES as the complete struct + * drm_asahi_cmd_render_m5. A feature bit like those of enum + * drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 ((1ULL) << 33) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -1044,6 +1077,87 @@ struct drm_asahi_sync { */ #define DRM_ASAHI_BARRIER_NONE (0xFFFFu) +/** + * struct drm_asahi_compute_program - Storage of one linked compute dispatch. + * + * SET_COMPUTE_RESOURCES replaces the list for subsequent compute commands. + * Include each distinct dispatch profile in the command stream. The kernel + * combines their requirements before publishing work. All sizes are bytes. + */ +struct drm_asahi_compute_program { + /** @primary_bytes: Per-thread primary storage. */ + __u32 primary_bytes; + /** @secondary_bytes: Per-thread secondary storage. */ + __u32 secondary_bytes; + /** @linked_bytes: Additional primary storage after linking. */ + __u32 linked_bytes; + /** @minimum_primary_bytes: Minimum primary storage across variants. */ + __u32 minimum_primary_bytes; + /** @threadgroup_bytes: Storage shared by a workgroup. */ + __u32 threadgroup_bytes; + /** @threads: Number of invocations in a workgroup. */ + __u32 threads; + /** @flags: Reserved; must be zero. */ + __u32 flags; +}; + +/** + * struct drm_asahi_render_program - Storage of one linked render program. + * + * SET_RENDER_RESOURCES replaces the list for subsequent renders in this submit. + * Include every vertex and fragment program, including background and end-of-tile + * programs. Sizes describe the final linked program, in bytes. Each entry is + * evaluated separately before taking the maximum demand across the pass. + */ +struct drm_asahi_render_program { + /** @stage: 0 for vertex, 1 for fragment (including background/end-of-tile). */ + __u32 stage; + /** @primary_bytes: Primary per-thread storage after linking. */ + __u32 primary_bytes; + /** @secondary_bytes: Secondary per-thread storage after linking. */ + __u32 secondary_bytes; + /** @linked_bytes: Additional primary storage from linked programs. */ + __u32 linked_bytes; + /** @minimum_primary_bytes: Minimum primary storage across linked variants. */ + __u32 minimum_primary_bytes; + /** @auxiliary_count: Fragment auxiliary entries; zero for vertex programs. */ + __u32 auxiliary_count; + /** @threadgroup_bytes: Fragment threadgroup storage; zero for vertex. */ + __u32 threadgroup_bytes; + /** @imageblock_bytes: Fragment imageblock storage; zero for vertex. */ + __u32 imageblock_bytes; + /** @multisample_bytes: Fragment multisample storage; zero for vertex. */ + __u32 multisample_bytes; + /** @flags: MBZ. */ + __u32 flags; +}; +/** Version 2 linked render storage. Version 1 remains exactly 40 bytes. */ +#define DRM_ASAHI_RENDER_PROGRAM_SCS (1U << 0) +#define DRM_ASAHI_RENDER_PROGRAM_MESH (1U << 1) +#define DRM_ASAHI_RENDER_PROGRAM_OBJECT (1U << 2) +/* With OBJECT: the object program is an API task stage. The kernel binds the + * TA mesh rendering buffer (the backing of the object payload address) to + * such kicks only. Requires DRM_ASAHI_FEATURE_TASK_MESH_STORAGE. */ +#define DRM_ASAHI_RENDER_PROGRAM_TASK (1U << 3) +struct drm_asahi_render_program_v2 { + struct drm_asahi_render_program legacy; + /** @threads: Mesh/object workgroup invocation count; zero otherwise. */ + __u32 threads; + /** @usage_flags: DRM_ASAHI_RENDER_PROGRAM_* bits. */ + __u32 usage_flags; + /** @scs_layout: Packed depth/levels/cliques/user/driver SCS layout. */ + __u64 scs_layout; + /** @auxiliary_allocation: The stage's auxiliary allocation word. */ + __u32 auxiliary_allocation; + __u32 reserved; + __u64 intersection_pool; + __u64 intersection_pool_bytes; + __u32 intersection_first; + __u32 intersection_count; + __u64 intersection_default_program; + __u64 intersection_default_control; +}; + /** * struct drm_asahi_cmd_header - Top level command structure * @@ -1283,6 +1397,38 @@ enum drm_asahi_render_flags { * Bits 21 to 24 are used by the M5 runtime's userspace. */ + /** + * @DRM_ASAHI_RENDER_SHADER_WRITES: A graphics program in this command + * writes external memory, including buffer/image stores or atomic RMW. + * Ordinary raster outputs and compiler-private transport do not count. + * Requires DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES. Omission retains the + * existing encoding, allowing older userspace to submit unchanged. + */ + DRM_ASAHI_RENDER_SHADER_WRITES = (1U << 21), + + /** + * @DRM_ASAHI_RENDER_STAGE_PREFIXES: The tiling half waits only on the + * render and compute prefixes in struct drm_asahi_render_stage_prefixes + * (ta_vdm_barrier, ta_prior_vdm, ta_cdm_barrier and ta_prior_cdm), each + * no later than the command's own prefix of that subqueue; the fragment + * half waits on the whole ones. Earlier renders in the tiling prefix are + * waited on to completion. The payload must be the complete struct + * drm_asahi_cmd_render_m5. Not combined with + * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. Requires + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2. + */ + DRM_ASAHI_RENDER_STAGE_PREFIXES = (1U << 23), + + /** + * @DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS: The render area covers the full + * extent of every attachment, so the render's tile stores write no pixel + * outside it. The kernel need not order such a render against another + * one that writes the same attachments without a dependency between + * them: any overlap is the application's own race. Requires + * DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS. + */ + DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS = (1U << 24), + /** * @DRM_ASAHI_RENDER_G18_RESOURCES: G18 only; requires * DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH. The payload is a complete @@ -1672,6 +1818,66 @@ struct drm_asahi_cmd_render { struct drm_asahi_render_ray_query_stage ray_query[2]; }; +/** + * struct drm_asahi_render_stage_prefixes - Tiling-half prefixes of an M5 render + * + * Carried by render commands with DRM_ASAHI_RENDER_STAGE_PREFIXES, after the + * complete struct drm_asahi_cmd_render (see struct drm_asahi_cmd_render_m5). + * The kernel reads it only with that flag; a render without the flag may + * carry it only as zeroes. + */ +struct drm_asahi_render_stage_prefixes { + /** + * @ta_cdm_barrier: Compute prefix of the tiling half, a command index like + * drm_asahi_cmd_header::cdm_barrier. + */ + __u16 ta_cdm_barrier; + + /** @reserved0: MBZ. */ + __u16 reserved0; + + /** @reserved1: MBZ. */ + __u32 reserved1; + + /** + * @ta_prior_cdm: Exact compute prefix of the tiling half, an ordinal like + * drm_asahi_cmd_render::prior_cdm. + */ + __u64 ta_prior_cdm; + + /** + * @ta_vdm_barrier: Render prefix of the tiling half, a command index like + * drm_asahi_cmd_header::vdm_barrier. + */ + __u16 ta_vdm_barrier; + + /** @reserved2: MBZ. */ + __u16 reserved2; + + /** @reserved3: MBZ. */ + __u32 reserved3; + + /** + * @ta_prior_vdm: Exact render prefix of the tiling half, an ordinal like + * drm_asahi_cmd_render::prior_vdm. + */ + __u64 ta_prior_vdm; +}; + +/** + * struct drm_asahi_cmd_render_m5 - M5 render payload with stage prefixes + * + * The payload of a render command with DRM_ASAHI_RENDER_STAGE_PREFIXES: the + * render command followed by its tiling-half prefixes. Every field of @legacy + * keeps its meaning. + */ +struct drm_asahi_cmd_render_m5 { + /** @legacy: The render command. */ + struct drm_asahi_cmd_render legacy; + /** @stage_prefixes: The tiling half's render and compute prefixes. */ + struct drm_asahi_render_stage_prefixes stage_prefixes; +}; + /** * struct drm_asahi_cmd_compute - Command to submit compute * @@ -1712,9 +1918,10 @@ struct drm_asahi_cmd_compute { struct drm_asahi_timestamp ts_end_copies[8]; /** - * @scs_layout: Packed G17P Gen1 shader-context scratch layout, or zero. - * Bits 0:15 contain stack depth, 16:23 levels, 24:31 cliques, - * 32:47 user bytes and 48:63 driver bytes. Non-G17P paths require zero. + * @scs_layout: Packed shader-context scratch layout on G17P (Gen1) and + * M5, or zero. Bits 0:15 contain stack depth, 16:23 levels, 24:31 + * cliques, 32:47 user bytes and 48:63 driver bytes. Other GPUs require + * zero. */ __u64 scs_layout; @@ -1736,7 +1943,13 @@ struct drm_asahi_cmd_compute { */ __u64 prior_cdm; - /** @cdm_auxiliary_allocation: G17P auxiliary allocation, zero or 8..32. */ + /** + * @cdm_auxiliary_allocation: M5: the per-core page count of the + * dispatch's shader-context scratch storage, which the hardware is + * given as is and the scratch pool covers. Zero, or 8..32 with + * @intersection_pool and shader-context scratch (@scs_layout). Other + * GPUs require zero. + */ __u32 cdm_auxiliary_allocation; /** @reserved: MBZ. */ __u32 reserved; @@ -1781,6 +1994,20 @@ enum drm_asahi_compute_flags { * Bits 16 and 17 are used by the M5 runtime's userspace. */ + /** + * @DRM_ASAHI_COMPUTE_USES_MATRIX: The command's dispatches use the + * hardware's matrix instructions. Requires DRM_ASAHI_FEATURE_COMPUTE_MATRIX. + */ + DRM_ASAHI_COMPUTE_USES_MATRIX = (1L << 16), + + /** + * @DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS: The command neither copies nor + * resets queries, so it cannot observe timestamps the kernel publishes + * for earlier commands and need not wait for that publication. Requires + * DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS. + */ + DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS = (1L << 17), + /** * @DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE: G18 only. The command's * dispatch binds texture or image state (a texture table, a sampled diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 0b4dfda2bbed8..34027b50941ab 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -2306,6 +2306,15 @@ static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, { struct hci_cp_le_set_scan_enable cp; + /* Firmware duplicate caches may need an explicit reset between scans. */ + if (val && filter_dup != LE_SCAN_FILTER_DUP_DISABLE && + !hci_dev_test_flag(hdev, HCI_MESH) && hdev->reset_dup_filter) { + int err = hdev->reset_dup_filter(hdev); + + if (err) + return err; + } + if (use_ext_scan(hdev)) return hci_le_set_ext_scan_enable_sync(hdev, val, filter_dup); diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c index 5deced5b975e0..c3366c96822b4 100644 --- a/rust/helpers/cpumask.c +++ b/rust/helpers/cpumask.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include __rust_helper void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) @@ -81,3 +82,9 @@ void rust_helper_free_cpumask_var(cpumask_var_t mask) free_cpumask_var(mask); } #endif + +__rust_helper +const struct cpumask *rust_helper_topology_cluster_cpumask(int cpu) +{ + return topology_cluster_cpumask(cpu); +} diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index 7cd3b000a5b6b..c79f38f98f9c2 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include __rust_helper void rust_helper_init_work_with_key(struct work_struct *work, @@ -20,3 +21,26 @@ rust_helper_alloc_ordered_workqueue(const char *name, unsigned int flags) { return alloc_ordered_workqueue("%s", flags, name); } + +__rust_helper struct workqueue_struct * +rust_helper_alloc_workqueue_flags(const char *name, unsigned int flags) +{ + return alloc_workqueue("%s", flags, 0, name); +} + +__rust_helper int rust_helper_workqueue_restrict(struct workqueue_struct *wq, + const struct cpumask *cpus, + bool ordered, bool highpri) +{ + struct workqueue_attrs *attrs = alloc_workqueue_attrs(); + int ret; + + if (!attrs) + return -ENOMEM; + cpumask_copy(attrs->cpumask, cpus); + attrs->nice = highpri ? MIN_NICE : 0; + attrs->ordered = ordered; + ret = apply_workqueue_attrs(wq, attrs); + free_workqueue_attrs(attrs); + return ret; +} diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index 44bb36636ee32..3b82790eb0340 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -77,6 +77,14 @@ impl Cpumask { unsafe { &*ptr.cast() } } + /// The CPUs of `cpu`'s cluster as the architecture's topology describes it: `cpu` alone when + /// there is no cluster information. + pub fn cluster_of(cpu: CpuId) -> &'static Self { + // SAFETY: The topology keeps one mask per possible CPU for the kernel's lifetime, and `cpu` + // is a valid CPU ID by its type invariant. + unsafe { Self::from_raw(bindings::topology_cluster_cpumask(cpu.as_u32() as i32)) } + } + /// Obtain the raw `struct cpumask` pointer. pub fn as_raw(&self) -> *mut bindings::cpumask { let this: *const Self = self; diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 83eca852139be..095f798e77d31 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -604,8 +604,19 @@ impl driver::AllocImpl for Object { prime_handle_to_fd: None, prime_fd_to_handle: None, gem_prime_import: None, - gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), - dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + // The core creates imported and dumb objects through `gem_create_object`. Without it they + // would be bare `drm_gem_shmem_object`s, which every handle lookup still treats as an + // `Object`: offer them only to drivers that can allocate the whole object. + gem_prime_import_sg_table: if T::HAS_CORE_OBJECT_ARGS { + Some(bindings::drm_gem_shmem_prime_import_sg_table) + } else { + None + }, + dumb_create: if T::HAS_CORE_OBJECT_ARGS { + Some(bindings::drm_gem_shmem_dumb_create) + } else { + None + }, dumb_map_offset: None, }; } diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 7250c61eccd38..8194c22003c1f 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -471,6 +471,12 @@ pub struct SchedulerInner { // SchedulerInner instead, so outstanding work cannot prevent finalization. struct SchedulerOwner { allocation: Arc>, + /// The driver's submission queue, if it supplied one. Fields drop after `Drop::drop`, so the + /// queue is destroyed only after `drm_sched_fini` has stopped the scheduler's work on it. + _submit_wq: Option, + /// The driver's timeout queue, if it supplied one; possibly shared with other schedulers. This + /// reference keeps it alive until `drm_sched_fini` has synchronized the timeout work. + _timeout_wq: Option>, } impl Drop for SchedulerOwner { @@ -527,6 +533,46 @@ impl Scheduler { hang_limit, timeout_ms, name, + None, + None, + ) + } + + /// Creates a new DRM Scheduler object whose run and free work goes to `submit_wq` and whose + /// timeout work goes to `timeout_wq`, each where given. + /// + /// Without `submit_wq`, drm_sched allocates an ordered reclaim-safe queue itself. With one, + /// the scheduler owns it and destroys it after it has been finalized; like drm_sched's own it + /// must be ordered and usable for memory reclaim (see [`OwnedQueue::new_ordered_reclaim`]), so + /// that one scheduler's jobs run and are freed one at a time. + /// + /// Without `timeout_wq`, the timeout work runs on `system_percpu_wq`, which memory reclaim + /// cannot rely on. A driver whose timeout handler signals job fences passes a queue usable for + /// memory reclaim (such as [`OwnedQueue::new_percpu_reclaim`]); several schedulers may share + /// it, and each keeps a reference until it has been finalized. + /// + /// [`OwnedQueue::new_ordered_reclaim`]: crate::workqueue::OwnedQueue::new_ordered_reclaim + /// [`OwnedQueue::new_percpu_reclaim`]: crate::workqueue::OwnedQueue::new_percpu_reclaim + pub fn new_with_queues( + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + submit_wq: Option, + timeout_wq: Option>, + ) -> Result> { + Self::init( + UniqueArc::new_uninit(GFP_KERNEL)?, + device, + num_rqs, + credit_limit, + hang_limit, + timeout_ms, + name, + submit_wq, + timeout_wq, ) } @@ -539,6 +585,8 @@ impl Scheduler { hang_limit: u32, timeout_ms: time::Msecs, name: &'static CStr, + submit_wq: Option, + timeout_wq: Option>, ) -> Result> { // Allocate the owner before starting C workers: no fallible allocation // may abandon an initialized scheduler without running drm_sched_fini. @@ -556,8 +604,8 @@ impl Scheduler { let init_ops = bindings::drm_sched_init_args { ops: &Self::OPS, - submit_wq: core::ptr::null_mut(), - timeout_wq: core::ptr::null_mut(), + submit_wq: submit_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), + timeout_wq: timeout_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), num_rqs, credit_limit, hang_limit, @@ -579,7 +627,15 @@ impl Scheduler { // SAFETY: C initialized sched and PhantomData has no stored state. // No fallible operation follows successful initialization. let allocation = unsafe { sched.assume_init() }.into(); - Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) + Ok(Scheduler( + owner + .write(SchedulerOwner { + allocation, + _submit_wq: submit_wq, + _timeout_wq: timeout_wq, + }) + .into(), + )) } } @@ -684,6 +740,8 @@ mod tests { 0, TIMEOUT_MS, c"rust-drm-sched-kunit", + None, + None, ) } diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 3124062c3cad9..3e4d97583073a 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -298,6 +298,33 @@ impl Queue { } } + /// Returns the raw `struct workqueue_struct` pointer, for C interfaces that take a queue. + pub fn as_raw(&self) -> *mut bindings::workqueue_struct { + self.0.get() + } + + /// Enqueues a work item on `cpu`. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// On a per-CPU queue the work item runs on `cpu`. On an unbound queue `cpu` only selects the + /// affinity pod (for example the CPU's last-level cache) whose worker runs it, so the + /// scheduler may pick another CPU of that pod. + pub fn enqueue_on(&self, cpu: crate::cpu::CpuId, w: W) -> W::EnqueueOutput + where + W: RawWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: As for `enqueue`. `cpu` is a valid CPU ID by its type invariant, which is what + // `queue_work_on` requires in place of `WORK_CPU_UNBOUND`. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on(cpu.as_u32() as ffi::c_int, queue_ptr, work_ptr) + }) + } + } + /// Enqueues a delayed work item. /// /// This may fail if the work item is already enqueued in a workqueue. @@ -374,17 +401,29 @@ impl WorkItem for ClosureWork { /// A work queue owned by the caller. /// -/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()`. It dereferences -/// to a [`Queue`], so work items are enqueued on it exactly like on the system queues. Dropping it -/// calls `destroy_workqueue()`, which waits for every pending work item to finish first. +/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()` or, for an +/// unbound or per-CPU queue, `alloc_workqueue()`. It dereferences to a [`Queue`], so work items +/// are enqueued on it exactly like on the system queues. Dropping it calls `destroy_workqueue()`, +/// which waits for every pending work item to finish first. +/// +/// An ordered owned queue gives a driver two things the system queues do not: its work items run +/// one at a time in queueing order, and the driver can wait for all of them at once (see +/// [`drain`]) before it tears down the state they use. An unbound owned queue runs its work items +/// in the affinity pod of the CPU they are queued for (see [`Queue::enqueue_on`]), and either kind +/// can be restricted to a set of CPUs (see [`restrict`]). A per-CPU owned queue runs each work item +/// on the CPU it is queued on, as the per-CPU system queues do. /// -/// An owned queue gives a driver two things the system queues do not: its work items run one at a -/// time in queueing order, and the driver can wait for all of them at once (see [`drain`]) before -/// it tears down the state they use. +/// A queue whose work may signal a DMA fence, or that memory reclaim otherwise waits on, must be +/// allocated by a `_reclaim` constructor (`WQ_MEM_RECLAIM`): its rescuer thread runs that work +/// when no new worker can be created because memory is short. /// /// # Invariants /// -/// The pointer is valid and points at a workqueue allocated by `alloc_ordered_workqueue()`. +/// `queue` is valid and points at a workqueue allocated by `alloc_ordered_workqueue()` when +/// `ordered` is set, or else by `alloc_workqueue()`, with `WQ_UNBOUND` when `unbound` is set +/// (`WQ_PERCPU` otherwise) and `WQ_HIGHPRI` when `highpri` is set. Ordered queues are unbound. +/// +/// [`restrict`]: OwnedQueue::restrict /// /// # Examples /// @@ -422,7 +461,12 @@ impl WorkItem for ClosureWork { /// ``` /// /// [`drain`]: OwnedQueue::drain -pub struct OwnedQueue(NonNull); +pub struct OwnedQueue { + queue: NonNull, + ordered: bool, + unbound: bool, + highpri: bool, +} impl OwnedQueue { /// Allocates an ordered workqueue named `name`. @@ -431,11 +475,78 @@ impl OwnedQueue { /// they were queued, by an unbound worker. The name is copied by the kernel and appears in the /// worker's thread name. pub fn new_ordered(name: &CStr) -> Result { + Self::ordered(name, 0) + } + + /// Allocates an ordered workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), such as a GPU scheduler's submission queue. + pub fn new_ordered_reclaim(name: &CStr) -> Result { + Self::ordered(name, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + fn ordered(name: &CStr, flags: u32) -> Result { // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. - let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), 0) }; + let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), flags) }; // INVARIANT: `alloc_ordered_workqueue()` returns a valid workqueue or NULL. - Ok(Self(NonNull::new(ptr).ok_or(ENOMEM)?)) + Ok(Self { + queue: NonNull::new(ptr).ok_or(ENOMEM)?, + ordered: true, + unbound: true, + highpri: false, + }) + } + + /// Allocates an unbound workqueue named `name`, with high-priority workers if `highpri`. + /// + /// Its work items run concurrently, each in the affinity pod of the CPU it is queued for. + pub fn new_unbound(name: &CStr, highpri: bool) -> Result { + Self::flags(name, true, highpri, 0) + } + + /// Allocates an unbound workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), with high-priority workers if `highpri`. + pub fn new_unbound_reclaim(name: &CStr, highpri: bool) -> Result { + Self::flags(name, true, highpri, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + /// Allocates a per-CPU workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), with high-priority workers if `highpri`: a driver's own counterpart of + /// `system_highpri_wq` (or `system_percpu_wq`) for work that memory reclaim may wait on. + pub fn new_percpu_reclaim(name: &CStr, highpri: bool) -> Result { + Self::flags(name, false, highpri, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + fn flags(name: &CStr, unbound: bool, highpri: bool, flags: u32) -> Result { + let flags = flags + | if unbound { bindings::wq_flags_WQ_UNBOUND } else { bindings::wq_flags_WQ_PERCPU } + | if highpri { bindings::wq_flags_WQ_HIGHPRI } else { 0 }; + // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. + let ptr = unsafe { bindings::alloc_workqueue_flags(name.as_char_ptr(), flags) }; + + // INVARIANT: `alloc_workqueue()` returns a valid workqueue with these flags, or NULL. + Ok(Self { + queue: NonNull::new(ptr).ok_or(ENOMEM)?, + ordered: false, + unbound, + highpri, + }) + } + + /// Restricts the queue's workers to the CPUs in `cpus`. + /// + /// An ordered queue stays ordered and a high-priority one keeps its priority. Allocates, so it + /// must be called from a context that may sleep. A per-CPU queue cannot be restricted + /// (`EINVAL`). + pub fn restrict(&self, cpus: &crate::cpumask::Cpumask) -> Result { + if !self.unbound { + return Err(EINVAL); + } + // SAFETY: The queue is a valid unbound workqueue by the type invariant and `cpus` is a + // valid mask; the helper copies it. + crate::error::to_result(unsafe { + bindings::workqueue_restrict(self.queue.as_ptr(), cpus.as_raw(), self.ordered, self.highpri) + }) } /// Waits until the workqueue is empty. @@ -446,7 +557,7 @@ impl OwnedQueue { /// stop every other producer before draining. Must be called from a context that may sleep. pub fn drain(&self) { // SAFETY: The pointer is valid by the type invariant. - unsafe { bindings::drain_workqueue(self.0.as_ptr()) }; + unsafe { bindings::drain_workqueue(self.queue.as_ptr()) }; } } @@ -456,7 +567,7 @@ impl Deref for OwnedQueue { fn deref(&self) -> &Queue { // SAFETY: The pointer is valid by the type invariant and stays valid for as long as `self` // does, which bounds the returned reference. - unsafe { Queue::from_raw(self.0.as_ptr()) } + unsafe { Queue::from_raw(self.queue.as_ptr()) } } } @@ -464,7 +575,7 @@ impl Drop for OwnedQueue { fn drop(&mut self) { // SAFETY: The pointer is valid by the type invariant. `destroy_workqueue()` drains the // queue before freeing it, so no work item can run on a freed queue. - unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + unsafe { bindings::destroy_workqueue(self.queue.as_ptr()) }; } } diff --git a/tools/asahi/apple-dart-fw-tests.py b/tools/asahi/apple-dart-fw-tests.py index d37ff84ba29ef..7abd6c161da50 100644 --- a/tools/asahi/apple-dart-fw-tests.py +++ b/tools/asahi/apple-dart-fw-tests.py @@ -4,8 +4,8 @@ from pathlib import Path import argparse, hashlib, json, re, subprocess ROOT=Path(__file__).resolve().parents[2] -NAMES=('apple_dart_fw_handoff_enabled','apple_dart_pte_to_paddr','apple_dart_fw_table_ram','apple_dart_fw_table_decode','apple_dart_fw_slot_owned','apple_dart_fw_leaf_decode','apple_dart_fw_mapping_matches','apple_dart_fw_page_matches','apple_dart_fw_slot_mirrors','apple_dart_publish_root','apple_dart_retire_root','apple_dart_free_fw_root','apple_dart_snapshot_fw_root','apple_dart_fw_slot_foreign','apple_dart_domain_uses_fw','apple_dart_check_fw_map','apple_dart_inherited_phys','apple_dart_fw_slot_range','apple_dart_get_fw_resv_regions','apple_dart_fw_invalidate','apple_dart_hw_unmap_locked_ttbr','apple_dart_retire_error','apple_dart_t8110_hw_invalidate_tlb','apple_dart_release_device','apple_dart_domain_free') -CASES=("legacy-map-match","legacy-map-mismatch","legacy-lookup","legacy-snapshot",'mirror','changed-leaf','wrong-permission','wrong-physical','malformed-leaf','vacant','changed-owner','retire','reserved-ram','unreserved','mmio','misalignment','address-width','root-encoding','leaf-as-table','snapshot','snapshot-malformed','snapshot-unreserved','snapshot-root-change','snapshot-map-failure','snapshot-allocation-failure','range-three','range-offset','range-width','range-overflow','range-underflow','range-four','range-four-last','map-firmware','map-hole','map-host','map-owned','map-unmatched','map-attached','map-wrong-physical','map-wrong-permission','map-changed-root','map-four','lookup-inherited','lookup-changed-leaf','lookup-changed-root','lookup-unmatched','reserve-firmware','reserve-teardown','reserve-unmatched','reserve-owned','invalidate-three','invalidate-four','invalidate-changed-root','invalidate-error','core-inherited','core-hole','core-changed-leaf','core-second-dart','snapshot-unmatched','snapshot-four','range-four-width','range-three-last','handoff-matched','handoff-missing','handoff-duplicate','handoff-zero','handoff-unlocked','handoff-other-soc','retire-free','retire-free-timeout','retire-release','retire-release-timeout','retire-rollback-timeout','retire-repeat-timeout') +NAMES=('apple_dart_fw_handoff_enabled','apple_dart_pte_to_paddr','apple_dart_fw_table_page','apple_dart_fw_table_ram','apple_dart_fw_table_pte','apple_dart_fw_table_decode','apple_dart_fw_tree_table','apple_dart_fw_slot_owned','apple_dart_fw_leaf_decode','apple_dart_fw_mapping_matches','apple_dart_fw_page_matches','apple_dart_fw_slot_mirrors','apple_dart_publish_root','apple_dart_retire_root','apple_dart_free_fw_root','apple_dart_snapshot_fw_root','apple_dart_free_fw_tree','apple_dart_fw_tree_phys','apple_dart_fw_slot_foreign','apple_dart_domain_uses_fw','apple_dart_check_fw_map','apple_dart_inherited_phys_prot','apple_dart_inherited_phys','apple_dart_fw_slot_range','apple_dart_get_fw_resv_regions','apple_dart_fw_invalidate','apple_dart_hw_unmap_locked_ttbr','apple_dart_retire_error','apple_dart_t8110_hw_invalidate_tlb','apple_dart_release_device','apple_dart_domain_free') +CASES=("legacy-map-match","legacy-map-mismatch","legacy-lookup","legacy-snapshot",'mirror','changed-leaf','wrong-permission','wrong-physical','malformed-leaf','vacant','changed-owner','retire','reserved-ram','unreserved','mmio','misalignment','address-width','root-encoding','leaf-as-table','snapshot','snapshot-malformed','snapshot-unreserved','snapshot-root-change','snapshot-map-failure','snapshot-allocation-failure','range-three','range-offset','range-width','range-overflow','range-underflow','range-four','range-four-last','map-firmware','map-hole','map-host','map-owned','map-unmatched','map-attached','map-wrong-physical','map-wrong-permission','map-changed-root','map-four','lookup-inherited','lookup-changed-leaf','lookup-changed-root','lookup-unmatched','reserve-firmware','reserve-teardown','reserve-unmatched','reserve-owned','invalidate-three','invalidate-four','invalidate-changed-root','invalidate-error','core-inherited','core-hole','core-changed-leaf','core-second-dart','snapshot-unmatched','snapshot-four','range-four-width','range-three-last','handoff-matched','handoff-missing','handoff-duplicate','handoff-zero','handoff-unlocked','handoff-other-soc','retire-free','retire-free-timeout','retire-release','retire-release-timeout','retire-rollback-timeout','retire-repeat-timeout','tree-lookup','tree-changed-root','tree-changed-mid','tree-unrecorded','tree-reserve','tree-mirror-only','subtree-free-partial','subtree-table-reserved','subtree-table-unclaimed','subtree-table-claimed','subtree-table-ram','subtree-table-misalignment') def function(s,name): match=re.search(r'^static[^;{}]*\b'+re.escape(name)+r'\([^;{}]*\)\s*\{',s,re.M) if not match:return None diff --git a/tools/asahi/apple-dart-fw/fixture.c b/tools/asahi/apple-dart-fw/fixture.c index 13e39fed942d5..6c141591b63d0 100644 --- a/tools/asahi/apple-dart-fw/fixture.c +++ b/tools/asahi/apple-dart-fw/fixture.c @@ -50,7 +50,7 @@ static unsigned lock_depth; static size_t find_next_bit(unsigned long*p,size_t n,size_t a){for(;alocked_ttbr[0][0]=NULL;retiring_dart->locked_owned[0][0]=NULL;free(retired_table);retired_table=NULL;}struct iommu_resv_region*r=®ions[region_count++];r->start=s;r->length=l;r->type=type;return r;} static void list_add_tail(struct list_head*l,struct list_head*h){h->count++;} struct apple_dart_fw_root {u64*entry;u64*accepted;u64**leaf;}; +struct apple_dart_fw_tree {u64*entry;u64**mid;u64**mid_entry;u64***leaf;}; #define DART_TCR(d,s) (s) static unsigned apple_dart_readl(struct apple_dart*d,unsigned reg){return d->tcr;} static u64 firmware_leaf[2048], ours_leaf[2048], root[2048]; @@ -80,6 +81,11 @@ static int ram_valid=1,reserved=1,map_failure,allocation_failure,change_root,map static int pfn_valid(unsigned long pfn){return ram_valid&&pfn>=4&&pfn<8;} static unsigned long pfn_to_page(unsigned long pfn){return pfn;} #define PageReserved(p) (reserved) +#define IORESOURCE_MEM 0x200 +#define IORES_DESC_NONE 0 +enum {REGION_DISJOINT,REGION_INTERSECTS,REGION_MIXED}; +static int claimed; +static int region_intersects(u64 s,size_t n,unsigned long f,unsigned long d){assert(n==SZ_16K&&f==IORESOURCE_MEM&&d==IORES_DESC_NONE);return claimed?REGION_INTERSECTS:REGION_DISJOINT;} static void*allocation(size_t n){if(allocation_failure)return NULL;return calloc(1,n);} #define kzalloc(n,g) allocation(n) #define kcalloc(n,z,g) allocation((n)*(z)) @@ -207,6 +213,22 @@ int main(int argc,char**argv){ else if(!strcmp(c,"lookup-unmatched")){d.fw_handoff=0;assert(!apple_dart_inherited_phys(&domain,0));} else assert(!strcmp(c,"lookup-inherited")); } + else if(!strncmp(c,"tree-",5)){ + /* fw_tree: firmware's recorded subtree of a four-level root (slot 0, mid 0) resolves; a changed root or mid + * entry, an unrecorded tree, and a fw_mirror DART without fw_tree resolve and reserve nothing. */ + static u64 mid_table[2048];u64*entry=calloc(2048,sizeof(u64)),**mid=calloc(2048,sizeof(u64*)),**mid_entry=calloc(2048,sizeof(u64*));u64***leaves3=calloc(2048,sizeof(u64**)); + assert(entry&&mid&&mid_entry&&leaves3);mid_entry[0]=calloc(2048,sizeof(u64));leaves3[0]=calloc(2048,sizeof(u64*));assert(mid_entry[0]&&leaves3[0]); + root[0]=0x1401;entry[0]=0x1401;mid[0]=mid_table;mid_table[0]=0x1801;mid_entry[0][0]=0x1801;leaves3[0][0]=firmware_leaf; + struct apple_dart_fw_tree tree={entry,mid,mid_entry,leaves3}; + d.fw_handoff=0;d.fw_mirror=1;d.fw_tree=strcmp(c,"tree-mirror-only")!=0;d.tcr=1;d.locked_fw[0][0]=NULL; + if(strcmp(c,"tree-unrecorded")&&strcmp(c,"tree-mirror-only"))d.locked_fw_tree[0]=&tree; + if(!strcmp(c,"tree-lookup"))assert(apple_dart_inherited_phys(&domain,0)==0x8000); + else if(!strcmp(c,"tree-changed-root")){root[0]=0x1001;assert(!apple_dart_inherited_phys(&domain,0));} + else if(!strcmp(c,"tree-changed-mid")){mid_table[0]=0x1c01;assert(!apple_dart_inherited_phys(&domain,0));} + else if(!strcmp(c,"tree-unrecorded"))assert(!apple_dart_inherited_phys(&domain,0)); + else if(!strcmp(c,"tree-reserve")){apple_dart_get_fw_resv_regions(&dev,&head);assert(head.count==1&®ions[0].start==0&®ions[0].length==(1ULL<<36)&®ions[0].type==IOMMU_RESV_RESERVED);} + else {assert(!strcmp(c,"tree-mirror-only"));assert(!apple_dart_inherited_phys(&domain,0));apple_dart_get_fw_resv_regions(&dev,&head);assert(!head.count);} + } else if(!strncmp(c,"reserve-",8)){ root[0]=0x1401;d.locked_window[0]=1ULL<<40; if(!strcmp(c,"reserve-teardown")){retired_table=calloc(2048,sizeof(u64));assert(retired_table);retired_table[0]=retired_table[5]=0x1401;d.locked_ttbr[0][0]=retired_table;release_on_alloc=1;retiring_dart=&d;} @@ -269,6 +291,17 @@ int main(int argc,char**argv){ else if(!strcmp(c,"vacant")){ours[0]=0;ours[1]=0x801;assert(!apple_dart_publish_root(&d,live,owned,ours,&fw,2));assert(live[0]==0x1401&&live[1]==0x801&&owned[1]==0x801);} else if(!strcmp(c,"changed-owner")){live[0]=0x1801;assert(apple_dart_publish_root(&d,live,owned,ours,&fw,2)==-EBUSY);assert(!owned[0]);} else if(!strcmp(c,"retire")){owned[1]=0x801;live[1]=0xc01;apple_dart_retire_root(live,owned,2);assert(live[0]==0x1401&&live[1]==0xc01&&!owned[1]);} + else if(!strcmp(c,"subtree-free-partial")){ + /* A failed allocation leaves any of the four arrays missing: freeing must not touch them. */ + struct apple_dart_fw_tree*t=calloc(1,sizeof(*t));assert(t);t->leaf=calloc(2048,sizeof(u64**));assert(t->leaf);t->leaf[0]=calloc(2048,sizeof(u64*));assert(t->leaf[0]);t->leaf[0][0]=firmware_leaf; + apple_dart_free_fw_tree(&d,t);assert(unmap_calls==1); + } + /* fw_tree tables: reserved RAM, or memory nothing claims; never Linux's RAM or a device's registers. */ + else if(!strcmp(c,"subtree-table-reserved")){claimed=1;assert(apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-unclaimed")){ram_valid=0;assert(apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-claimed")){ram_valid=0;claimed=1;assert(!apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-ram")){reserved=0;claimed=1;assert(!apple_dart_fw_tree_table(&d,0x14000));} + else if(!strcmp(c,"subtree-table-misalignment")){ram_valid=0;assert(!apple_dart_fw_tree_table(&d,0x14001));} else if(!strcmp(c,"reserved-ram")){assert(apple_dart_fw_table_ram(&d,0x14000));} else if(!strcmp(c,"unreserved")){reserved=0;assert(!apple_dart_fw_table_ram(&d,0x14000));} else if(!strcmp(c,"mmio")){ram_valid=0;assert(!apple_dart_fw_table_ram(&d,0x14000));} diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index e7617fba021c2..7ded29db0c3c7 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -78,6 +78,7 @@ def item(text, marker): static COMPLETED_REGISTRATION: Cell = const { Cell::new(0) }; static ACTIVATED: Cell = const { Cell::new(false) }; static BINDING_READY: Cell = const { Cell::new(false) }; + static QUEUE_SHARE_READY: Cell = const { Cell::new(false) }; static ATTACH_ERROR: Cell = const { Cell::new(false) }; static PROBE_ERROR: Cell = const { Cell::new(0) }; } @@ -146,6 +147,10 @@ def item(text, marker): mod binding { pub unsafe fn init() { crate::BINDING_READY.set(true); } } +// The queue-share registry must also be ready before an adapter can probe. +mod queue_share { + pub unsafe fn init() { crate::QUEUE_SHARE_READY.set(true); } +} mod cleanup { pub struct QueueOwner; impl QueueOwner { pub fn new() -> crate::Result { Ok(Self) } } @@ -157,6 +162,7 @@ def item(text, marker): impl Registration { pub fn new(name: &'static std::ffi::CStr, _module: &'static ThisModule) -> impl PinInit { assert!(BINDING_READY.get(), "adapter registered before binding registry init"); + assert!(QUEUE_SHARE_READY.get(), "adapter registered before queue-share registry init"); try_pin_init!(Self { stage: { let stage = match name.to_bytes() { @@ -208,6 +214,7 @@ def item(text, marker): fn reset(fail_alloc: bool, touch: bool, error: i32) { MODULE_READY.store(true, Ordering::Release); BINDING_READY.set(false); + QUEUE_SHARE_READY.set(false); FAIL_ALLOC.set(fail_alloc); TOUCH.set(touch); PROBE_ERROR.set(error); RETAINED.set(false); ALLOCS.set(0); PROBES.set(0); } diff --git a/tools/asahi/g17-admission-deadline-tests.py b/tools/asahi/g17-admission-deadline-tests.py index d6e63e08d1188..501c903b59f8f 100644 --- a/tools/asahi/g17-admission-deadline-tests.py +++ b/tools/asahi/g17-admission-deadline-tests.py @@ -37,15 +37,17 @@ def fixture(): mod g17 {pub fn now_ns()->u64 {crate::NOW.load(crate::Ordering::SeqCst)}} mod kernel {pub mod time {pub fn msecs_to_jiffies(ms:u32)->u64 {u64::from(ms)}}} mod hazard {pub const WAIT_MS:u32=1000;} +mod cfg {pub const RENDER_SLOTS:usize=2;} const QUARANTINE_WAIT_MS:u32=1000; #[derive(Clone,Copy,PartialEq)] enum Plan {Any,Conflict} #[derive(Clone,Copy)] struct Request<'a>(&'a ()); -struct State {owners:[u64;2],quarantined:u64,spared:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} +struct State {owners:[u64;cfg::RENDER_SLOTS],quarantined:u64,spared:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} impl State { fn exhausted(&self)->bool {false} +''' + item(source, 'fn owner_quarantine(') + r''' fn refusal(&self,_:u64)->Result {Ok(())} fn plan(&self,_:u64,_:Option>,_:Option)->Plan {if self.conflict {Plan::Conflict}else{Plan::Any}} - fn pick(&mut self,_:u64,_:Plan)->Option {if self.ready{Some(1)}else{None}} + fn pick(&mut self,_:u64,_:usize,_:Plan)->Option {if self.ready{Some(1)}else{None}} fn record(&mut self,_:u64,slot:Option,_:u64,_:Request<'_>)->Option {if slot.is_none(){self.waiters+=1;}Some(0)} fn forget_waiter(&mut self,t:Option){if t.is_some(){self.waiters-=1;}} } @@ -76,31 +78,31 @@ def fixture(): for mode in 0..2 { for conflict in [false,true] { reset(mode);let pool=new_pool(conflict,0); - let result=pool.acquire_render(1,Some(Request(&())),2_000_000_000,||Ok(())); + let result=pool.acquire_render(1,1,Some(Request(&())),2_000_000_000,||Ok(())); assert!(matches!(result,Err(EBUSY)),"busy wait must expire"); assert!(NOW.load(Ordering::SeqCst)<=2_010_000_000,"absolute deadline extended by wakes"); assert_eq!(pool.state.lock().waiters,0,"expired waiter must be removed"); reset(mode);let pool=new_pool(conflict,0); - let result=pool.acquire_render(1,Some(Request(&())),2_000_000_000,||if g17::now_ns()>=5_000_000 {Err(ECANCELED)}else{Ok(())}); + let result=pool.acquire_render(1,1,Some(Request(&())),2_000_000_000,||if g17::now_ns()>=5_000_000 {Err(ECANCELED)}else{Ok(())}); assert!(matches!(result,Err(ECANCELED)),"cancel must interrupt a busy wait"); assert!(NOW.load(Ordering::SeqCst)<=15_000_000,"cancel observation bound"); assert_eq!(pool.state.lock().waiters,0,"cancelled waiter must be removed"); } reset(mode);let pool=new_pool(false,2); - assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EIO))); + assert!(matches!(pool.acquire_render(1,1,None,2_000_000_000,||Ok(())),Err(EIO))); assert!(NOW.load(Ordering::SeqCst)<=1_010_000_000,"quarantine budget extended"); reset(mode);let pool=new_pool(false,2);pool.state.lock().spared=2; - assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EBUSY))); + assert!(matches!(pool.acquire_render(1,1,None,2_000_000_000,||Ok(())),Err(EBUSY))); assert!((2_000_000_000..=2_010_000_000).contains(&NOW.load(Ordering::SeqCst)),"a spared quarantine waits for the caller's deadline"); } reset(2);let pool=new_pool(true,0); - assert!(matches!(pool.acquire_render(1,Some(Request(&())),2_000_000_000,||Ok(())),Err(ERESTARTSYS))); + assert!(matches!(pool.acquire_render(1,1,Some(Request(&())),2_000_000_000,||Ok(())),Err(ERESTARTSYS))); assert_eq!(pool.state.lock().waiters,0); // A previous outer retry already spent the budget. Even available capacity // cannot restart it or install a lease after cancellation. reset(0);NOW.store(2_000_000_000,Ordering::SeqCst);let pool=new_pool(false,0);pool.state.lock().ready=true; - assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EBUSY))); - assert!(matches!(pool.acquire_render(1,None,u64::MAX,||Err(ECANCELED)),Err(ECANCELED))); + assert!(matches!(pool.acquire_render(1,1,None,2_000_000_000,||Ok(())),Err(EBUSY))); + assert!(matches!(pool.acquire_render(1,1,None,u64::MAX,||Err(ECANCELED)),Err(ECANCELED))); assert_eq!(WAITS.load(Ordering::SeqCst),0); println!("PASS production admission: all-pairs-busy/no-progress and 20,000 unrelated wakes expire at absolute deadline; no-event cancellation bounded; conflict/quarantine/signal cleanup preserved"); } diff --git a/tools/asahi/g17-admission-order-tests.py b/tools/asahi/g17-admission-order-tests.py index e247afd638318..849fcefcbe576 100644 --- a/tools/asahi/g17-admission-order-tests.py +++ b/tools/asahi/g17-admission-order-tests.py @@ -29,49 +29,86 @@ def main(): #![allow(dead_code)] use std::sync::atomic::{AtomicUsize,AtomicBool,Ordering}; type Result = std::result::Result; -const ECANCELED:i32=-125;const EIO:i32=-5;const EFAULT:i32=-14; -mod g17 {pub mod fw {pub mod compute_pool { - pub fn required_blocks(_:usize,cores:u32,pages:u16)->crate::Result { - if cores==0 {Err(-22)} else {Ok(21+usize::from(pages))} +const ECANCELED:i32=-125;const EIO:i32=-5;const EFAULT:i32=-14;const EINVAL:i32=-22; +mod g17 { +pub mod fw { + pub mod compute_pool { + pub fn required_blocks(_:usize,cores:u32,pages:u16)->crate::Result { + if cores==0 {Err(-22)} else {Ok(21+usize::from(pages))} + } } -}}} -struct Status(i32); -impl Status {fn get(&self)->i32{self.0}} -struct Context {status:Status} -impl Context {fn status(&self)->&Status{&self.status} fn is_current(&self)->bool{true}} -struct Shared {compute_blocks:usize,active_cores:u32,crashed:AtomicBool} -struct Backend {shared:Shared,physical:AtomicUsize,closed:AtomicBool,context:Context} -impl Backend { - fn ensure_compute(&self,_:u16)->Result {self.physical.fetch_add(1,Ordering::SeqCst);Ok(())} -''' + method('runtime.rs', 'fn admission_open(') + r''' + pub mod m5 { pub mod pool { #[derive(Clone,Copy,Debug,PartialEq)] pub struct PoolGeometry(pub u32); } } } -trait Api {fn ensure_compute(&self,scs_pages:u16)->Result;} -impl Api for Backend { +pub mod freelist { + #[derive(Debug,PartialEq)] + pub enum ComputeCapacity { Blocks(usize), M5(Option) } +} +pub mod render_resources { #[derive(Clone,Copy)] pub struct Plan { pub geometry: super::fw::m5::pool::PoolGeometry } } +pub mod command { + pub struct ScratchRequest(pub u16); + impl ScratchRequest { pub fn pages(&self)->u16{self.0} } + pub enum Validated { + Render, + Compute { plan: Option, scratch: ScratchRequest }, + } +} +pub mod runtime { + use super::*; + use crate::{Result,ECANCELED,EIO,EFAULT,EINVAL}; + use std::sync::atomic::{AtomicUsize,AtomicBool,Ordering}; + pub struct Status(pub i32); + impl Status {fn get(&self)->i32{self.0}} + pub struct Context {pub status:Status,pub m5:bool} + impl Context {fn status(&self)->&Status{&self.status} fn is_current(&self)->bool{true} fn is_m5(&self)->bool{self.m5}} + pub struct Shared {pub compute_blocks:usize,pub active_cores:u32,pub crashed:AtomicBool} + pub struct Admission; + pub struct Backend {pub shared:Shared,pub physical:AtomicUsize,pub closed:AtomicBool,pub context:Context} + impl Backend { + pub fn ensure_compute(&self,_:freelist::ComputeCapacity,_:&Admission)->Result {self.physical.fetch_add(1,Ordering::SeqCst);Ok(())} +''' + method('runtime.rs', 'pub(super) fn compute_capacity(').replace('pub(super) ', 'pub ', 1) + '\n' + method('runtime.rs', 'fn admission_open(').replace('fn ', 'pub fn ', 1) + r''' + } + pub trait Api {fn ensure_compute(&self,command:&command::Validated)->Result;} + impl Api for Backend { ''' + method('runtime/backend.rs', 'fn ensure_compute(') + r''' + } +} +} +use g17::{command::{ScratchRequest,Validated},freelist::ComputeCapacity,fw::m5::pool::PoolGeometry,render_resources::Plan,runtime::*}; +fn compute(pages:u16)->Validated {Validated::Compute{plan:None,scratch:ScratchRequest(pages)}} +fn backend(cores:u32,m5:bool)->Backend { + Backend{shared:Shared{compute_blocks:21,active_cores:cores,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0),m5}} } fn main(){ - let backend=Backend{shared:Shared{compute_blocks:21,active_cores:10,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0)}}; + let backend=backend(10,false); // Four clients each enqueue nine queues in reverse dependency order. The // final host signal cannot be submitted if ioctl admission blocks on channels. - for _client in 0..4 {for _queue in (0..9).rev() {::ensure_compute(&backend,0).unwrap();}} + for _client in 0..4 {for _queue in (0..9).rev() {::ensure_compute(&backend,&compute(0)).unwrap();}} assert_eq!(backend.physical.load(Ordering::SeqCst),0,"unsignalled work must not pin physical capacity"); - Backend::ensure_compute(&backend,0).unwrap(); + assert_eq!(backend.compute_capacity(&compute(3)),Ok(ComputeCapacity::Blocks(24))); + Backend::ensure_compute(&backend,backend.compute_capacity(&compute(0)).unwrap(),&Admission).unwrap(); assert_eq!(backend.physical.load(Ordering::SeqCst),1); - let invalid=Backend{shared:Shared{compute_blocks:21,active_cores:0,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0)}}; - assert_eq!(::ensure_compute(&invalid,0),Err(-22)); + let invalid=self::backend(0,false); + assert_eq!(::ensure_compute(&invalid,&compute(0)),Err(-22)); assert_eq!(invalid.physical.load(Ordering::SeqCst),0); + assert_eq!(::ensure_compute(&backend,&Validated::Render),Err(EINVAL)); + // M5 sizes a compute pool by its plan's geometry; that, too, admits nothing physical. + let m5=self::backend(0,true); + let planned=Validated::Compute{plan:Some(Plan{geometry:PoolGeometry(7)}),scratch:ScratchRequest(0)}; + for _ in 0..36 {::ensure_compute(&m5,&planned).unwrap();} + assert_eq!(m5.compute_capacity(&planned),Ok(ComputeCapacity::M5(Some(PoolGeometry(7))))); + assert_eq!(m5.physical.load(Ordering::SeqCst),0,"unsignalled work must not pin physical capacity"); for _ in 0..512 { backend.closed.store(true,Ordering::Release); - assert_eq!(backend.admission_open().and_then(|()| Backend::ensure_compute(&backend,0)),Err(ECANCELED),"closed constructor must not install"); + assert_eq!(backend.admission_open().and_then(|()| Backend::ensure_compute(&backend,ComputeCapacity::Blocks(21),&Admission)),Err(ECANCELED),"closed constructor must not install"); assert_eq!(backend.physical.load(Ordering::SeqCst),1); } - println!("PASS closed builders cannot install after logical close; production ioctl sizing validates 36 reverse-order submissions without physical admission; invalid sizing consumes no channel"); + println!("PASS closed builders cannot install after logical close; production ioctl sizing validates 36 reverse-order submissions without physical admission, on G17P and M5; invalid sizing consumes no channel"); } ''' - fixed = 'crate::g17::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages)\n .map(|_| ())' + fixed = 'self.compute_capacity(command).map(|_| ())' assert code.count(fixed) == 1 with tempfile.TemporaryDirectory(prefix='g17-admission-order-') as tmp: - for name, source in [('fixed', code), ('eager-physical-admission', code.replace(fixed, 'Backend::ensure_compute(self, scs_pages)')), ('ignore-logical-close', code.replace('self.closed.load(Ordering::Acquire)', 'false'))]: + for name, source in [('fixed', code), ('eager-physical-admission', code.replace(fixed, 'self.compute_capacity(command).and_then(|required| Backend::ensure_compute(self, required, &Admission))')), ('ignore-logical-close', code.replace('self.closed.load(Ordering::Acquire)', 'false'))]: path = Path(tmp) / (name + '.rs') binary = path.with_suffix('') path.write_text(source) diff --git a/tools/asahi/g17-capacity-fault-tests.py b/tools/asahi/g17-capacity-fault-tests.py index c03764ca28d60..bd79442ef05a9 100644 --- a/tools/asahi/g17-capacity-fault-tests.py +++ b/tools/asahi/g17-capacity-fault-tests.py @@ -33,9 +33,14 @@ def fixture(): 'pub(crate) fn recycle_quarantined_idle(', 'pub(crate) fn recycle_killed(', 'fn recycle_settled(', 'fn release_witnessed_inner(', 'pub(crate) fn share_idle(', 'pub(crate) fn shareable(', 'pub(crate) fn can_rearm(', 'pub(crate) fn recycled_pending(']) exit_source = (ROOT / 'queue/compute/exit.rs').read_text() - methods += '\n' + item(exit_source, 'pub(crate) fn settle_exit(').replace('super::Previous', 'Previous').replace('crate::g17::qos::Owner', 'qos::Owner') + methods += '\n' + (item(exit_source, 'pub(crate) fn settle_exit(').replace('super::Previous', 'Previous') + .replace('crate::g17::qos::Owner', 'qos::Owner').replace('super::retirement::', 'retirement::')) drain = item((ROOT / 'mod.rs').read_text(), 'fn drain_reclaims(') proof = item(retirement, 'pub(crate) const fn stamp_newer(') + '\n' + item(retirement, 'pub(crate) const fn stamp_covers(') + '\n' + item(retirement, 'pub(crate) fn quarantined_retired(') + # M5 judges retained commands by their completion records. + proof += '\n' + '#[derive(Clone, Copy, Debug, PartialEq, Eq)]\n' + item(retirement, 'pub(crate) struct Observation') + proof += '\n' + next(line for line in retirement.splitlines() if 'const CANCELLED_TICKS:' in line) + proof += '\n' + item(retirement, 'pub(crate) fn unexecuted(') return r''' #![allow(dead_code)] use std::sync::{Arc, atomic::{AtomicUsize, AtomicU32, Ordering}}; @@ -50,6 +55,7 @@ def fixture(): struct Context { id: u64, live: Arc, killed: AtomicU32 } impl Context { fn note_compute_binding(&self, _: bool) {} + fn retain_firmware_pages(&self) {} fn new(id: u64, live: &Arc) -> Arc { live.fetch_add(1, Ordering::SeqCst); Arc::new(Self { id, live: live.clone(), killed: AtomicU32::new(1) }) @@ -64,8 +70,9 @@ def fixture(): } #[derive(Clone, Copy)] struct Cursors { count: u32, producer: u32, consumer: u32 } ''' + proof + r''' -mod retirement { pub(crate) use super::{quarantined_retired, stamp_newer}; } -struct Graph { cursors: Cursors, stamp: u32, fail: bool, resets: usize, owner: u64 } +mod retirement { pub(crate) use super::{quarantined_retired, stamp_newer, unexecuted}; } +// These queues are G17P's (m5 false); the M5 branches compile but do not run. +struct Graph { cursors: Cursors, stamp: u32, fail: bool, resets: usize, owner: u64, m5: bool } impl Graph { fn set_owner(&mut self, context: &Context) -> Result { if self.fail { Err(EIO) } else { self.owner = context.id; Ok(()) } } fn reset_items(&mut self) -> Result { if self.fail { Err(EIO) } else { self.resets += 1; Ok(()) } } @@ -75,7 +82,7 @@ def fixture(): struct Kick; impl Kick { fn clear_parent_after_recovery(&mut self) {} } #[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { Published, Acknowledged, Unproven, Released } +enum Phase { Published, Acknowledged, Unproven, Witnessed, Released } struct Exit { phase: Phase } impl Exit { fn released(&self) -> bool { self.phase == Phase::Released } } struct State { killed: u32 } @@ -85,9 +92,10 @@ def fixture(): fn has_firmware_consumer(&self) -> bool { false } fn compute_words(&self) -> Result<[u32; 2]> { Ok([0, 0]) } fn replay_witness_words(&self) -> Option<[u32; 2]> { Some([0, 0]) } + fn signalled(&self) -> bool { false } } struct Packet { completion: Arc } -struct Active { packet: Arc, ticket: Option, qos_pending: bool } +struct Active { packet: Arc, ticket: Option, observed: Option, qos_pending: bool } struct Fifo { entries: Vec, len: usize } impl Fifo { fn new() -> Self { Self { entries: vec![], len: 0 } } @@ -110,6 +118,8 @@ def fixture(): retire_pending: bool, retirement_proved: bool, replay_retirement: bool, replay_error: Option, replay_prepared: bool, submitted: u32, kick: Kick, config_cache: Option<()>, source_hint: ReaderHint, source_readers: AtomicU32, claims: u32, fault_stamp: Option, + configuration: Option<()>, retained_owner: Option<()>, timed_out_at: u64, + m5_install: Option<()>, published_config: Option<()>, outer_started: u32, ordinal: u64, } impl Queue { fn valid_ticket(&self, _: &Active, _: &Ticket) -> Result { Ok(()) } @@ -121,7 +131,7 @@ def fixture(): Queue { owner: Some(2), installed: true, active: Fifo::new(), quarantined: false, previous: None, unpublished_binding: false, released: false, - graph: Graph { cursors: Cursors { count: 1280, producer: 4, consumer: 4 }, stamp: 7 << 8, fail: false, resets: 0, owner: 2 }, + graph: Graph { cursors: Cursors { count: 1280, producer: 4, consumer: 4 }, stamp: 7 << 8, fail: false, resets: 0, owner: 2, m5: false }, binding: Some(Binding { context: Context::new(2, live) }), retirement_ready: false, released_at: 0, growing: false, pool_phase: PoolPhase::Populated, needs_fresh_backing: false, exit: None, replays: Fifo::new(), @@ -130,14 +140,16 @@ def fixture(): retire_pending: false, retirement_proved: false, replay_retirement: false, replay_error: None, replay_prepared: false, submitted: 0, kick: Kick, fault_stamp: None, config_cache: None, source_hint: ReaderHint { possible: false }, source_readers: AtomicU32::new(0), claims: 0, + configuration: None, retained_owner: None, timed_out_at: 0, + m5_install: None, published_config: None, outer_started: 0, ordinal: 0, } } fn fault(live: &Arc) -> Queue { let mut queue = idle(live); let old = Context::new(1, live); - queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: Some((11, 3)) }); + queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: Some((11, 3)), moved: false }); queue.active.push(Active { packet: Arc::new(Packet { completion: Arc::new(Completion) }), - ticket: Some(Ticket { owner: 2, item_count: 1280, descriptor: 0x1000, item_producer: 4, kick: 7 }), qos_pending: true }).unwrap(); + ticket: Some(Ticket { owner: 2, item_count: 1280, descriptor: 0x1000, item_producer: 4, kick: 7 }), observed: None, qos_pending: true }).unwrap(); queue.quarantined = true; queue.awaiting_witness = true; queue.submitted = 1; @@ -259,7 +271,7 @@ def fixture(): } let mut queue = idle(&live); let old = Context::new(1, &live); - queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: None }); + queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: None, moved: false }); queue.release_owner().unwrap(); assert_eq!(queue.graph.owner, 1); assert!(queue.previous.is_none()); diff --git a/tools/asahi/g17-compute-constructor-tests.py b/tools/asahi/g17-compute-constructor-tests.py index 14b736f50436b..3c96caec2c19a 100644 --- a/tools/asahi/g17-compute-constructor-tests.py +++ b/tools/asahi/g17-compute-constructor-tests.py @@ -53,7 +53,7 @@ def fixture(): } struct Shared{preparations:Gate,state:State,changed:Changed,crashed:AtomicBool} impl Shared{fn drm(&self)->Result<()>{Ok(())} fn note_changed(&self){self.changed.notify_all();}} -struct Context;impl Context{fn vm(&self)->&(){&()}} +struct Context{m5:bool}impl Context{fn vm(&self)->&(){&()} fn is_m5(&self)->bool{self.m5}} struct Backend{shared:Shared,context:Context} struct Allocator<'a>{dev:&'a (),uat:&'a ()} struct ComputeShared;struct ComputeScheduler; @@ -76,7 +76,7 @@ def fixture(): fn main(){ for scheduler in [false,true]{for cancel in [false,true]{for building in [false,true]{ NOW.store(0,Ordering::SeqCst);CANCEL.store(cancel,Ordering::SeqCst);ALLOCATE.store(!building,Ordering::SeqCst); - let b=Backend{context:Context,shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:building,compute_scheduler_building:building}}))))}}; + let b=Backend{context:Context{m5:false},shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:building,compute_scheduler_building:building}}))))}}; let check=||{if CANCEL.load(Ordering::SeqCst)&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; let admission=Admission::new(20_000_000,&check); let alloc=Allocator{dev:&(),uat:&()}; @@ -87,7 +87,13 @@ def fixture(): assert!(!fw.init.shared&&!fw.init.scheduler,"expired constructor must not install"); if !building{assert!(!fw.queues.compute_building&&!fw.queues.compute_scheduler_building,"failed constructor must clear builder flag");} }}} - println!("PASS 8 production shared/scheduler constructor waits: expiry, cancellation, off-lock-build expiry, no installation and flag cleanup"); + // M5 compute queues need no shared aperture: nothing waits, builds or installs. + NOW.store(0,Ordering::SeqCst);CANCEL.store(false,Ordering::SeqCst);ALLOCATE.store(true,Ordering::SeqCst); + let b=Backend{context:Context{m5:true},shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:true,compute_scheduler_building:false}}))))}}; + let check=||Ok(()); + assert_eq!(b.shared_compute(&Admission::new(20_000_000,&check)),Ok(())); + assert_eq!(now_ns(),0);assert!(!b.shared.state.lock().as_ref().unwrap().init.shared); + println!("PASS 8 production shared/scheduler constructor waits: expiry, cancellation, off-lock-build expiry, no installation and flag cleanup; M5 skips the G17P shared aperture"); } ''' diff --git a/tools/asahi/g17-compute-deadline-tests.py b/tools/asahi/g17-compute-deadline-tests.py index e33f643fc61f0..45e95c0026ead 100644 --- a/tools/asahi/g17-compute-deadline-tests.py +++ b/tools/asahi/g17-compute-deadline-tests.py @@ -33,12 +33,15 @@ def fixture(): for old, new in [('super::job::Packet', 'Packet'), ('super::command::Validated', 'Validated'), ('super::recovery::Memory', 'Memory'), - ('super::fw::compute_pool::required_blocks', 'required_blocks')]: + ('super::fw::compute_pool::required_blocks', 'required_blocks'), + ('super::freelist::ComputeCapacity', 'ComputeCapacity'), + ('super::preparation::QUEUE_WAIT_NS', 'QUEUE_WAIT_NS')]: packet = packet.replace(old, new) ensure = ensure.replace(old, new) admission = (item(preparation, "pub(super) struct Admission<'a>") + '\n' + item(preparation, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()').replace('super::timeout::', '') gate = item(preparation, 'pub(super) fn enter_until(').replace('pub(super) ', '') + queue_wait = next(line for line in preparation.splitlines() if 'const QUEUE_WAIT_NS:' in line).replace('pub(super) ', '') return r''' #![allow(dead_code,unused_variables,irrefutable_let_patterns)] use std::cell::Cell; @@ -47,11 +50,14 @@ def fixture(): const EINVAL:i32=-22;const ECANCELED:i32=-125;const EIO:i32=-5;const ENODEV:i32=-19;const EBUSY:i32=-16;const EAGAIN:i32=-11; const RECOVERY_SETTLE_POLLS:usize=200;const WORK_RING_WAIT_NS:u64=2_000_000_000; const RECOVERY_WAIT_MS:u32=26_000;const RECOVERY_WAIT_NS:u64=RECOVERY_WAIT_MS as u64*1_000_000; +''' + queue_wait + r''' +// A compute publication waits up to QUEUE_WAIT_NS, or a recovery's wait if that is longer. +const PUBLICATION_WAIT_NS:u64=if QUEUE_WAIT_NS>RECOVERY_WAIT_NS{QUEUE_WAIT_NS}else{RECOVERY_WAIT_NS}; static NOW:AtomicU64=AtomicU64::new(0);static CALLS:AtomicUsize=AtomicUsize::new(0); static MODE:AtomicUsize=AtomicUsize::new(0);static CANCEL:AtomicU64=AtomicU64::new(u64::MAX); static STATUS:AtomicU64=AtomicU64::new(u64::MAX);static CLOSED:AtomicBool=AtomicBool::new(false); fn now_ns()->u64{NOW.load(Ordering::SeqCst)} -fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);assert!(now_ns()<=RECOVERY_WAIT_NS+4_000_000_000,"admission budget exceeded");} +fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);assert!(now_ns()<=PUBLICATION_WAIT_NS+4_000_000_000,"admission budget exceeded");} mod kernel{pub mod time{ pub fn msecs_to_jiffies(ms:u32)->u32{ms} pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms*1_000_000)}pub fn from_micros(us:i64)->Self{Self(us as u64*1_000)}} @@ -98,7 +104,13 @@ def fixture(): struct ComputeDemand;impl ComputeDemand{fn new(_:&Shared)->Self{Self}} struct Backend{shared:Shared,owner:u64,context:Context} fn required_blocks(_:usize,_:u32,_:u16)->Result{Ok(1)} +type ComputeCapacity=usize; impl Backend{ + fn compute_capacity(&self,command:&Validated)->Result{let Validated::Compute{scratch}=command;required_blocks(self.shared.compute_blocks,self.shared.active_cores,scratch.pages())} + // Off M5 there is no shared work ring: no outer domain, and a claim always holds. + fn outer_domain(&self)->Option>{None} + fn claim_outer(&self,_:&Firmware,engine:g17::job::Engine)->Result{assert_eq!(engine,g17::job::Engine::Compute,"compute admission takes a compute work state");Ok(true)} + fn publish_claimed(&self,firmware:&mut Firmware,packet:&Arc,dependencies:&[()],deferred:&mut DeferredBatch)->Result{firmware.publish_compute(self.owner,packet.clone(),dependencies,deferred)} fn compute_direct(&self,_:&Arc,_:usize,_:&mut DeferredBatch)->Option{None} fn sources_published(&self,_:&Arc)->Result{Ok(true)} fn remember_direct(&self,_:&Queues){} @@ -122,9 +134,9 @@ def fixture(): MODE.store(mode,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); let r=b.compute_packet(&p,&mut DeferredBatch); assert_eq!(r,Err(EBUSY),"one admission budget covers binding theft/recovery/publication"); - let budget=if mode==1{RECOVERY_WAIT_NS}else{WORK_RING_WAIT_NS}; + let budget=if mode==1{PUBLICATION_WAIT_NS}else{QUEUE_WAIT_NS}; assert!(now_ns()>=budget&&now_ns()<=budget+10_000_000,"admission budget exceeded"); - if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),4);assert_eq!(now_ns(),WORK_RING_WAIT_NS);} + if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),(QUEUE_WAIT_NS/500_000_000) as usize);assert_eq!(now_ns(),QUEUE_WAIT_NS);} println!("PASS mode={} result={:?} calls={} elapsed_ms={}",mode,r,CALLS.load(Ordering::SeqCst),now_ns()/1_000_000); } // A context without a free work state waits for one under the admission @@ -132,7 +144,7 @@ def fixture(): MODE.store(4,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); let r=b.compute_packet(&p,&mut DeferredBatch); assert_eq!(r,Err(EBUSY),"a busy work state ends at the admission deadline"); - assert!(now_ns()>=WORK_RING_WAIT_NS&&now_ns()<=WORK_RING_WAIT_NS+10_000_000,"a busy work state waits for the admission deadline"); + assert!(now_ns()>=QUEUE_WAIT_NS&&now_ns()<=QUEUE_WAIT_NS+10_000_000,"a busy work state waits for the admission deadline"); println!("PASS busy work state waits elapsed_ms={}",now_ns()/1_000_000); for status in [false,true] { MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); @@ -146,6 +158,11 @@ def fixture(): let admission=Admission::new(WORK_RING_WAIT_NS,&check); assert!(matches!(b.shared.preparations.enter_until(&admission),Err(EBUSY)),"spent budget before available gate"); assert_eq!(b.ensure_compute(0,&admission),Err(EBUSY),"spent budget before nested admission"); + // A pending drain witness extends a shorter admission to the recovery wait. + MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst); + let admission=Admission::new(WORK_RING_WAIT_NS,&check); + assert_eq!(b.ensure_compute(0,&admission),Err(EBUSY)); + assert!(now_ns()>=RECOVERY_WAIT_NS&&now_ns()<=RECOVERY_WAIT_NS+10_000_000,"a pending witness waits out the recovery extension"); for cancel in [false,true] { NOW.store(0,Ordering::SeqCst);CLOSED.store(true,Ordering::SeqCst); let check=||{if cancel&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; @@ -162,14 +179,14 @@ def main(): source = fixture() mutants = [ ('fixed', source, ''), - ('renew-on-binding-loss', source.replace('let admission = Admission::new(deadline, &check);\n let progress = &self.shared.progress;\n loop {', 'let progress = &self.shared.progress;\n loop {\n let admission = Admission::new(now_ns().saturating_add(WORK_RING_WAIT_NS), &check);'), 'admission budget exceeded'), + ('renew-on-binding-loss', source.replace('let admission = Admission::new(deadline, &check);\n let progress = &self.shared.progress;\n let outer = self.outer_domain();\n loop {', 'let progress = &self.shared.progress;\n let outer = self.outer_domain();\n loop {\n let admission = Admission::new(now_ns().saturating_add(QUEUE_WAIT_NS), &check);'), 'admission budget exceeded'), ('ignore-deadline', source.replace('if now_ns() >= self.deadline.get()', 'if false'), 'admission budget exceeded'), ('render-work-state', source.replace('acquire(crate::g17::job::Engine::Compute)', 'acquire(crate::g17::job::Engine::Render)'), 'compute admission takes a compute work state'), ('omit-gate-check', source.replace(' admission.check()?;\n match self.try_enter()', ' match self.try_enter()').replace('admission.slice_ms()?', '10'), 'gate wake storm'), - ('no-recovery-extension', source.replace('admission.extend_for_recovery();', ''), 'admission budget exceeded'), + ('no-recovery-extension', source.replace('admission.extend_for_recovery();', ''), 'recovery extension'), ] # Bind the old unbounded mutant with the review's eventual cancellation. - mutants[1] = (mutants[1][0], mutants[1][1].replace('for mode in 0..4 {', 'CANCEL.store(40_000_000_000,Ordering::SeqCst);\n for mode in 0..4 {'), mutants[1][2]) + mutants[1] = (mutants[1][0], mutants[1][1].replace('for mode in 0..4 {', 'CANCEL.store(PUBLICATION_WAIT_NS+10_000_000_000,Ordering::SeqCst);\n for mode in 0..4 {'), mutants[1][2]) with tempfile.TemporaryDirectory(prefix='g17-compute-deadline-') as tmp: for name, code, expected in mutants: if name != 'fixed': diff --git a/tools/asahi/g17-compute-growth-deadline-tests.py b/tools/asahi/g17-compute-growth-deadline-tests.py index f2c148e5ec46a..4cff04e4343e0 100644 --- a/tools/asahi/g17-compute-growth-deadline-tests.py +++ b/tools/asahi/g17-compute-growth-deadline-tests.py @@ -29,7 +29,8 @@ def fixture(): admission = (item(prep, "pub(super) struct Admission<'a>") + '\n' + item(prep, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()').replace('super::timeout::', '') methods = '\n'.join(item(runtime, sig) for sig in [' fn grow_compute(', ' fn finish_uninstalled_compute(']) - methods = methods.replace('super::object::Allocator', 'Allocator').replace('super::freelist::Pages', 'Pages') + methods = (methods.replace('super::object::Allocator', 'Allocator').replace('super::freelist::Pages', 'Pages') + .replace('super::freelist::ComputeCapacity', 'ComputeCapacity')) return r''' #![allow(dead_code)] use std::cell::Cell; @@ -55,13 +56,16 @@ def fixture(): struct Available;impl Available{fn observe(&self)->u64{now_ns()}fn notify(&self){}} struct Shared{preparations:Gate,state:State,crashed:AtomicBool,progress:Available,channels:Available} impl Shared{fn drm(&self)->Result<()>{Ok(())}fn queue_events(&self){}} -struct Context;impl Context{fn vm(&self)->&(){&()}} +struct Vm;impl Vm{fn memory_client(&self)->Option<&()>{None}} +struct Context;impl Context{fn vm(&self)->&Vm{&Vm}} +// A G17P compute pool is sized in blocks; the sizes only flow through here. +type ComputeCapacity=usize; struct Backend{shared:Shared,context:Context,owner:u64} struct Allocator<'a>{dev:&'a (),uat:&'a ()} struct Pages;struct Alias; impl Pages{ - fn compute_sized(_:&Allocator<'_>,_:usize)->Result{PAGES.store(true,Ordering::SeqCst);Ok(Self)} - fn map_client(&self,_:&())->Result{ALIAS.store(true,Ordering::SeqCst);Ok(Alias)} + fn for_compute(_:&Allocator<'_>,_:ComputeCapacity,_:Option<&()>)->Result{PAGES.store(true,Ordering::SeqCst);Ok(Self)} + fn map_client(&self,_:&Vm)->Result{ALIAS.store(true,Ordering::SeqCst);Ok(Alias)} } impl Drop for Pages{fn drop(&mut self){PAGES.store(false,Ordering::SeqCst);}} impl Drop for Alias{fn drop(&mut self){ALIAS.store(false,Ordering::SeqCst);}} @@ -71,7 +75,7 @@ def fixture(): fn pool_fits(&self,_:usize)->bool{false} fn begin_growth(&mut self,_:&(),_:usize)->Result{Ok(false)} fn growth_token(&self)->Option{Some(7)} - fn finish_growth(&mut self,_:&(),_:&mut Option,_:&mut Option,_:usize)->Result{Err(EAGAIN)} + fn finish_growth(&mut self,_:&(),_:&mut Option,_:&mut Option)->Result{Err(EAGAIN)} fn abort_growth(&mut self,t:u64)->Result>{assert_eq!(t,7);ABORTED.store(true,Ordering::SeqCst);Ok(Some(self.fresh))} } impl Drop for Queue{fn drop(&mut self){if self.fresh{assert!(RESERVED.load(Ordering::SeqCst));assert!(!ALIAS.load(Ordering::SeqCst)&&!PAGES.load(Ordering::SeqCst),"growth aliases must precede fresh QID return");}}} diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py index 2150f121eaa6f..3f583ea3e0302 100644 --- a/tools/asahi/g17-growth-tests.py +++ b/tools/asahi/g17-growth-tests.py @@ -23,24 +23,26 @@ def fixture(): const EIO:i32=-5;const EBUSY:i32=-16;const EINVAL:i32=-22; fn now_ns()->u64{123} struct Context {live:Arc} -impl Context {fn note_compute_binding(&self,_:bool){} fn new(live:&Arc)->Arc{live.fetch_add(1,Ordering::SeqCst);Arc::new(Self{live:live.clone()})}} +impl Context {fn note_compute_binding(&self,_:bool){} fn retain_firmware_pages(&self){} fn new(live:&Arc)->Arc{live.fetch_add(1,Ordering::SeqCst);Arc::new(Self{live:live.clone()})}} impl Drop for Context {fn drop(&mut self){self.live.fetch_sub(1,Ordering::SeqCst);}} struct Binding {context:Arc,pool_alias:Option,descriptors:Option} -struct Previous {context:Arc,binding:Option,publication:Option<(u64,u64)>} +struct Previous {context:Arc,binding:Option,publication:Option<(u64,u64)>,moved:bool} +// A G17P compute pool is sized in blocks; M5's geometry does not reach this queue. +#[derive(Clone,Copy,Debug,PartialEq)] enum ComputeCapacity{Blocks(usize)} #[derive(Clone,Copy,PartialEq)] enum PoolPhase{Populated,ReleaseSent(u32),Vacant} struct Fifo {len:usize} struct Graph; impl Graph {fn set_owner(&mut self,_:&Context)->Result{Ok(())}} struct ReaderHint; impl ReaderHint {fn has_readers(&self,count:&AtomicU32)->bool{count.load(Ordering::Acquire)!=0}} struct Queue { - owner:Option,installed:bool,growing:bool,growth_blocks:usize,free_list_generation:u64, + owner:Option,installed:bool,growing:bool,growth:Option,free_list_generation:u64,configuration:Option<()>,retained_owner:Option<()>, pool_phase:PoolPhase,release_sent_ns:Option,pool:Pool,binding:Option,previous:Option,graph:Graph, active:Fifo,replays:Fifo,quarantined:bool,released:bool,retirement_ready:bool, retired_by_teardown:bool,released_at:u64,exit:Option<()>,needs_fresh_backing:bool, source_hint:ReaderHint,source_readers:AtomicU32, } -fn idle(live:&Arc)->Queue{Queue{owner:Some(1),installed:true,growing:false,growth_blocks:0, - free_list_generation:0,pool_phase:PoolPhase::Populated,release_sent_ns:None,pool:Pool{pages:Some(Pages(10)),fail:false}, +fn idle(live:&Arc)->Queue{Queue{owner:Some(1),installed:true,growing:false,growth:None, + free_list_generation:0,configuration:None,retained_owner:None,pool_phase:PoolPhase::Populated,release_sent_ns:None,pool:Pool{pages:Some(Pages(10,21)),fail:false}, binding:Some(Binding{context:Context::new(live),pool_alias:Some(1),descriptors:Some(2)}),previous:None,graph:Graph, active:Fifo{len:0},replays:Fifo{len:0},quarantined:false,released:false,retirement_ready:true, retired_by_teardown:false,released_at:0,exit:None,needs_fresh_backing:false,source_hint:ReaderHint,source_readers:AtomicU32::new(0)}} @@ -53,8 +55,14 @@ def fixture(): const EAGAIN:i32=-11;const EFAULT:i32=-14; static CONSUMER:AtomicU32=AtomicU32::new(0); static PRODUCER:AtomicU32=AtomicU32::new(3); -struct Pages(u64); -impl Pages {fn page_list_va(&self)->u64{self.0}} +// A page list's address and its block count. +struct Pages(u64,usize); +struct PoolDescriptor; +impl Pages { + fn page_list_va(&self)->u64{self.0} + fn fits_capacity(&self,required:ComputeCapacity)->bool{let ComputeCapacity::Blocks(blocks)=required;self.1>=blocks} + fn descriptor(&self,_:u16)->Result{Ok(PoolDescriptor)} +} mod mmu {pub type KernelMapping=u64;} mod g17 { pub fn now_ns()->u64{crate::now_ns()} @@ -65,7 +73,7 @@ def fixture(): impl InitData { pub fn control_consumer(&self)->crate::Result<&crate::AtomicU32>{Ok(&crate::CONSUMER)} pub fn control_producer(&self)->crate::Result<&crate::AtomicU32>{Ok(&crate::PRODUCER)} - pub fn initialize_compute_pool_blocks(&self,_:u16,_:u64,_:usize)->crate::Result{if self.fail{Err(crate::EIO)}else{Ok(())}} + pub fn install_compute_pool(&self,_:crate::PoolDescriptor)->crate::Result{if self.fail{Err(crate::EIO)}else{Ok(())}} } } } @@ -103,19 +111,19 @@ def fixture(): let live=Arc::new(AtomicUsize::new(0));let init=initdata::InitData{fail:false}; for close_first in [false,true] {for consumed_first in [false,true] { let mut q=idle(&live);q.retirement_ready=true; - assert!(q.begin_growth(&init,40).unwrap());let token=q.growth_token().unwrap(); + assert!(q.begin_growth(&init,ComputeCapacity::Blocks(40)).unwrap());let token=q.growth_token().unwrap(); assert!(q.growing);assert!(q.pool_phase==PoolPhase::ReleaseSent(3)); assert_eq!(q.release_sent_ns,Some(now_ns()),"exact publication starts the release deadline"); assert_eq!(q.abort_growth(Growth{qid:8,phase:token.phase}).unwrap(),None,"wrong physical queue"); assert_eq!(q.abort_growth(Growth{qid:token.qid,phase:PoolPhase::ReleaseSent(4)}).unwrap(),None,"stale release transaction"); assert!(q.growing); - assert!(q.begin_growth(&init,40).is_err(),"another builder cannot steal growth"); + assert!(q.begin_growth(&init,ComputeCapacity::Blocks(40)).is_err(),"another builder cannot steal growth"); if close_first {q.release_owner().unwrap();} CONSUMER.store(if consumed_first{3}else{2},Ordering::SeqCst); assert!(q.observe_vacancy(CONSUMER.load(Ordering::SeqCst),3).unwrap().is_none(),"live builder owns growth"); // Timeout, epoch invalidation, notify/MMIO error all run this same abort. assert_eq!(q.abort_growth(token).unwrap(),Some(false)); - assert!(!q.growing);assert!(q.owner.is_none());assert_eq!(q.growth_blocks,0); + assert!(!q.growing);assert!(q.owner.is_none());assert_eq!(q.growth,None); assert_eq!(q.abort_growth(token).unwrap(),None); assert!(q.pool.pages.is_some(),"abort must retain backing"); assert_eq!(q.release_sent_ns,Some(now_ns()),"abort must not refresh or discard an owned deadline"); @@ -135,19 +143,23 @@ def fixture(): drop(registry);drop(deferred);assert_eq!(live.load(Ordering::SeqCst),0); }} for failure in 0..2 { - let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,40).unwrap(); + let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,ComputeCapacity::Blocks(40)).unwrap(); let token=q.growth_token().unwrap();CONSUMER.store(3,Ordering::SeqCst); - let mut pages=Some(Pages(20));let mut alias=Some(30); + let mut pages=Some(Pages(20,40));let mut alias=Some(30); q.pool.fail=failure==0; - assert!(q.finish_growth(&initdata::InitData{fail:failure==1},&mut pages,&mut alias,40).is_err()); + assert!(q.finish_growth(&initdata::InitData{fail:failure==1},&mut pages,&mut alias).is_err()); assert_eq!(pages.as_ref().unwrap().0,20,"failed rearm preserves tentative pages"); assert_eq!(q.pool.pages.as_ref().unwrap().0,10,"failed rearm preserves old pages"); assert_eq!(alias,Some(30));assert_eq!(q.binding.as_ref().unwrap().pool_alias,Some(1)); q.abort_growth(token).unwrap();assert!(q.observe_vacancy(3,3).unwrap().is_some()); } - let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,40).unwrap(); - let token=q.growth_token().unwrap();let mut pages=Some(Pages(20));let mut alias=Some(30); - let old=q.finish_growth(&init,&mut pages,&mut alias,40).unwrap(); + let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,ComputeCapacity::Blocks(40)).unwrap(); + let token=q.growth_token().unwrap();let mut pages=Some(Pages(20,39));let mut alias=Some(30); + // The fresh backing must hold what the growth began for. + assert_eq!(q.finish_growth(&init,&mut pages,&mut alias).err(),Some(EAGAIN),"undersized backing must not commit"); + assert!(q.growing);assert_eq!(q.pool.pages.as_ref().unwrap().0,10);assert_eq!(alias,Some(30)); + let mut pages=Some(Pages(20,40)); + let old=q.finish_growth(&init,&mut pages,&mut alias).unwrap(); assert_eq!(q.release_sent_ns,None,"committed replacement clears the witnessed release deadline"); assert_eq!(old._pages.unwrap().0,10);assert_eq!(old._alias,Some(1));assert!(pages.is_none());assert!(alias.is_none()); assert_eq!(q.pool.pages.as_ref().unwrap().0,20);assert_eq!(q.abort_growth(token).unwrap(),None,"committed growth cannot be aborted"); @@ -161,7 +173,8 @@ def main(): runtime = (ROOT / 'runtime.rs').read_text() assert 'growth = queue.growth_token();' in runtime and 'queue.abort_growth(token)' in runtime cases = [('fixed', source), - ('missing-abort-transfer', source.replace('self.growing = false;\n self.growth_blocks = 0;\n self.release_owner().map(Some)', 'self.release_owner().map(Some)')), + ('missing-abort-transfer', source.replace('self.growing = false;\n self.growth = None;\n self.release_owner().map(Some)', 'self.release_owner().map(Some)')), + ('ignore-growth-target', source.replace('if !pages.as_ref().ok_or(EINVAL)?.fits_capacity(target) {', 'if false {')), ('ignore-consumption', source.replace('if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? {', 'if false {')), ('ignore-collector-pressure', source.replace('if !deferred.vacated_room() { break; }', ''))] with tempfile.TemporaryDirectory(prefix='g17-growth-') as tmp: @@ -169,6 +182,8 @@ def main(): path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021','-Awarnings',str(path),'-o',str(binary)],check=True) result=subprocess.run([str(binary)],capture_output=True,text=True,timeout=15) + if name!='fixed': + assert code!=source,f'vacuous mutation: {name}' if name=='fixed': assert result.returncode==0,result.stderr print(result.stdout.strip()) diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py index 55b95af1b279d..517eda49370d4 100755 --- a/tools/asahi/g17-owner-release-order-tests.py +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -41,7 +41,7 @@ def check_guards(): prepare = (ROOT / 'runtime/render/prepare.rs').read_text() build = item(prepare, 'fn build_render_pair(') # The in-place rebind is refused while the pair's scheduler release is pending. - ordered(build, 'if !pair.reusable()?', 'firmware.queues.teardown.releasing(pair.context())', + ordered(build, 'if !pair.bindable(self.owner, &self.context)?', 'firmware.queues.teardown.releasing(pair.context())', 'return Err(EBUSY);', 'Build::Rebind {') rebind = item(prepare, 'fn rebind_pair(') # A live donor that closed while the binding was prepared is caught at commit. @@ -73,8 +73,8 @@ def check_guards(): 'return Err(EAGAIN);', 'queue.publish(') # A busy pair (for example one whose scheduler release is still being consumed) # kicks the event worker, which observes the release and parks the pair. - ordered(item(submit, 'fn wait_render_busy('), 'WORK_RING_WAIT_NS', - 'self.shared.queue_events();', 'self.wait_domain(') + ordered(item(submit, 'fn wait_render_busy('), 'self.wait_render_retirement(', 'WORK_RING_WAIT_NS') + ordered(item(submit, 'fn wait_render_retirement('), 'self.shared.queue_events();', 'self.wait_domain(') ordered(item(submit, 'fn wait_domain('), 'if remaining == 0', 'return Err(ETIMEDOUT);', 'domain.block(') borrowed = item((ROOT / 'runtime/render.rs').read_text(), 'fn take_pair(') ordered(borrowed, 'entry.borrowed_switch = pair.switch_in_flight();', @@ -102,8 +102,10 @@ def fixture(): 'pub(crate) fn owner_switch_from(', 'pub(crate) fn switch_in_flight(']).replace('pub(crate) ', '') record = render[render.index(' let scheduler = packet.context.scheduler_va();'): render.index(' self.executed = scheduler;') + len(' self.executed = scheduler;')] - idle = item((ROOT / 'runtime/render/teardown.rs').read_text(), - 'pub(in crate::g17::runtime) fn teardown_idle(').replace('pub(in crate::g17::runtime) ', '') + render_teardown = (ROOT / 'runtime/render/teardown.rs').read_text() + # teardown_idle() is idle_for() without the release-time kill exception. + idle = '\n'.join(item(render_teardown, name) for name in [ + 'pub(in crate::g17::runtime) fn teardown_idle(', 'fn idle_for(']).replace('pub(in crate::g17::runtime) ', '') compute_switch = item((ROOT / 'queue/compute.rs').read_text(), 'pub(crate) fn switching_from(').replace('pub(crate) ', '') compute_idle = item((ROOT / 'runtime/teardown.rs').read_text(), 'fn compute_idle(') @@ -116,6 +118,8 @@ def fixture(): impl Context { fn scheduler_va(&self) -> u64 { self.scheduler } fn is_published(&self) -> bool { self.published } + // Reached only for a release-time check, which this fixture does not make. + fn kill_published(&self) -> bool { unreachable!() } } #[derive(Clone, Copy, PartialEq, Eq)] enum Phase { Deferred, Draining, Released(u32) } diff --git a/tools/asahi/g17-ray-query-timeout-tests.sh b/tools/asahi/g17-ray-query-timeout-tests.sh index 9aaa3aa653c14..2ac2a7dc852a0 100755 --- a/tools/asahi/g17-ray-query-timeout-tests.sh +++ b/tools/asahi/g17-ray-query-timeout-tests.sh @@ -5,6 +5,18 @@ set -eu out=${1:?usage: g17-ray-query-timeout-tests.sh OUTPUT_DIRECTORY} root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) mkdir -p "$out" +# timeout.rs reads the driver's clock as crate::g17::now_ns(), so it builds +# as crate::g17::timeout beside a host clock, as it sits in the driver. +cat > "$out/g17-timeout-tests.rs" < u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now().duration_since(UNIX_EPOCH).map_or(1, |t| t.as_nanos() as u64) + } + #[path = "$root/drivers/gpu/drm/asahi/g17/timeout.rs"] + pub(crate) mod timeout; +} +RUST "${RUSTC:-rustc}" --edition=2021 --test \ - "$root/drivers/gpu/drm/asahi/g17/timeout.rs" -o "$out/g17-timeout-tests" + "$out/g17-timeout-tests.rs" -o "$out/g17-timeout-tests" "$out/g17-timeout-tests" diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index ae7e53ec6cf6f..f08622360d929 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -19,7 +19,11 @@ def item(start): while depth: depth+=(s[end]=='{')-(s[end]=='}');end+=1 return s[at:end] - constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_PER_OWNER','RENDER_DEPTH']) + constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_MAX','RENDER_DEPTH']) + pairs=(source/'g17/pairs.rs').read_text() + assert 'const G17D_PAIRS: u8 = RENDER_PAIRS_MAX as u8;' in pairs + assert 'PAIRS.store(if board.g17d_forms() { G17D_PAIRS } else { 1 }, Relaxed);' in pairs + at=pairs.index('const SIBLING_BACKLOG');backlog=pairs[at:pairs.index('\n}\n',pairs.index('fn sibling_backlog',at))+3] mmu=(source/'mmu.rs').read_text() limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) header='''#![allow(dead_code, unused_macros)] @@ -27,51 +31,67 @@ def item(start): mod g17 { pub mod trace { pub fn teardown(_:u8,_:u64,_:u64,_:u64) {} } pub mod capacity { pub enum Site { RenderSlots, LogicalQueues, ComputeReserve, PhysicalQids } pub fn refused(_: Site) -> i32 { -28 } + } pub mod pairs { use std::sync::atomic::{AtomicU8, Ordering::Relaxed}; + '''+backlog+''' }} const ENOSPC:i32=-28; const EIO:i32=-5; macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } type KVec = Vec; struct InFlight; mod cfg {\n'''+constants+'}\n'+limits+'\n' - methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + methods=['const fn new','fn exhausted','fn owner_quarantine','fn refusal','fn advance','fn take_idle','fn acquire','fn usable','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' + // A queue holds g17::pairs::per_owner() pairs (Backend::pairs): one, or + // RENDER_PAIRS_MAX on G17D. Both admissions run. fn main(){ - if cfg::RENDER_PAIRS_PER_OWNER == 1 { + one_pair(1); + two_pairs(cfg::RENDER_PAIRS_MAX); + } + fn one_pair(pairs:usize){ let mut state=State::new(); let mut slots=Vec::new(); for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - let slot=state.acquire(owner).expect("every logical owner must fit");slots.push(slot); + let slot=state.acquire(owner,pairs).expect("every logical owner must fit");slots.push(slot); assert_eq!(state.owners.iter().filter(|id|**id==owner).count(),1); } for slot in slots {state.release(slot);} - let mut state=State::new();let first=state.acquire(1).unwrap(); - for _ in 1..cfg::RENDER_DEPTH {assert_eq!(state.acquire(1),Some(first));} - assert_eq!(state.acquire(1),None);state.release(first);assert_eq!(state.acquire(1),Some(first)); + let mut state=State::new();let first=state.acquire(1,pairs).unwrap(); + for _ in 1..cfg::RENDER_DEPTH {assert_eq!(state.acquire(1,pairs),Some(first));} + assert_eq!(state.acquire(1,pairs),None);state.release(first);assert_eq!(state.acquire(1,pairs),Some(first)); println!("shared render admission: all58 owners fit; depth16 waits for exact retirement without consuming a spare"); - return; - } + } + fn two_pairs(pairs:usize){ + // 32 owners fill the 63 usable slots: 31 with a sibling, the last without. + assert_eq!(pairs,2); let mut state=State::new(); for owner in 1..=32 { assert!(state.refusal(owner).is_ok()); - let slot=state.acquire(owner).expect("existing owner fits"); + let slot=state.acquire(owner,pairs).expect("existing owner fits"); state.release(slot); } assert_eq!(state.active,0,"all submitted work retired"); let retained=state.owners.iter().filter(|o|**o!=0).count(); assert_eq!(retained,63); assert_eq!(state.refusal(33),Err(ENOSPC)); - assert!(state.acquire(33).is_none()); + assert!(state.acquire(33,pairs).is_none()); println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); // Yield one unused spare for each additional logical owner. The runtime // separately proves that these physical graphs have never been published. for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); - assert_eq!(state.acquire(owner),Some(slot as u8));state.release(slot as u8); + assert_eq!(state.acquire(owner,pairs),Some(slot as u8));state.release(slot as u8); } - for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner).unwrap();state.release(slot);} + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner,pairs).unwrap();state.release(slot);} assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); - let mut guarded=State::new();let first=guarded.acquire(1).unwrap();guarded.release(first); + let mut depth=State::new();let first=depth.acquire(1,pairs).unwrap(); + for _ in 1..pairs*usize::from(cfg::RENDER_DEPTH) {assert!(depth.acquire(1,pairs).is_some());} + assert_eq!(depth.acquire(1,pairs),None); + let held:Vec<_>=(0..cfg::RENDER_SLOTS).filter(|slot|depth.owners[*slot]==1).collect(); + assert_eq!(held.len(),pairs);assert!(held.iter().all(|slot|depth.refs[*slot]==cfg::RENDER_DEPTH)); + depth.release(first);assert_eq!(depth.acquire(1,pairs),Some(first)); + println!("two pairs: depth {} over both pairs, then waits for exact retirement",pairs*usize::from(cfg::RENDER_DEPTH)); + let mut guarded=State::new();let first=guarded.acquire(1,pairs).unwrap();guarded.release(first); let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; guarded.take_idle(spare,1); assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); @@ -86,23 +106,32 @@ def item(start): assert!(!guarded.reassign_unused_spare(64,None,Some(3))); guarded.released |= 1u64<bool { false } } impl ControlRecord { pub fn bytes(&self)->&[u8] { &[] } @@ -80,6 +81,11 @@ def item(source, signature): pub fn asserts_gpu_power(&self)->bool { false } } } + // M5's statistics tags and power rule, as the family methods below read them. + pub mod m5 { + M5_STATISTICS_TYPES + pub mod power { M5_ASSERTS_GPU_POWER } + } pub mod initdata { pub mod status { pub const TRACE_STATE:u32=0; pub const TRACE_RING:u32=0x100; @@ -94,9 +100,11 @@ def item(source, signature): pub struct InitData { pub cursors:[[AtomicU32;2];2], pub slots:RefCell<[[[u32;18];8];2]>, + pub m5:bool, } impl InitData { - pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]) } } + pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]), m5:false } } + INITDATA_FAMILY pub fn push(&self,role:Role,words:[u32;18]) { use std::sync::atomic::Ordering::Relaxed; let producer=self.cursors[role as usize][1].load(Relaxed); @@ -667,8 +675,20 @@ def main(): policy = '\n'.join(item(stats_rs, signature) for signature in ['pub(super) const fn max_age_ns', 'pub(super) const fn reports_pstate', 'pub(super) fn qualify(']) + # The rings take the family's statistics tags and control power rule from + # InitData; build those production methods into the fixture's InitData. + initdata = (DRIVER / 'g17/initdata.rs').read_text() + family = '\n'.join(item(initdata, signature).replace('pub(super) ', 'pub ', 1) for signature in + ['pub(super) fn statistics_types(', 'pub(super) fn control_asserts_gpu_power(']) + m5 = (DRIVER / 'g17/fw/m5/mod.rs').read_text() + m5_types = next(x for x in m5.splitlines() if 'const STATISTICS_TYPES:' in x).replace('pub(crate)', 'pub') + m5_power = item((DRIVER / 'g17/fw/m5/power.rs').read_text(), + 'pub(crate) const fn asserts_gpu_power(').replace('pub(crate)', 'pub', 1) + header = (RUST_HEADER.replace('M5_STATISTICS_TYPES', m5_types).replace('M5_ASSERTS_GPU_POWER', m5_power) + .replace('INITDATA_FAMILY', family)) + reader_rs = 'pub(crate) struct Reader(pub(super) Arc);\n' + item(stats_rs, 'impl Reader') - (work / 'rings.rs').write_text(RUST_HEADER + 'mod stats { use super::{fw::variant::InitGraph, telemetry::Readout, Shared};\n' + (work / 'rings.rs').write_text(header + 'mod stats { use super::{fw::variant::InitGraph, telemetry::Readout, Shared};\n' + 'use std::sync::{Arc,atomic::Ordering};\n' + reader_rs + '\n' + policy + '\n}\nmod telemetry;\nmod channel;\n' + RUST_TESTS) # Rust resolves child modules relative to the inline parent's directory. diff --git a/tools/asahi/g17-unpublished-tests.py b/tools/asahi/g17-unpublished-tests.py index f06eedf7f64f8..594621bebe661 100644 --- a/tools/asahi/g17-unpublished-tests.py +++ b/tools/asahi/g17-unpublished-tests.py @@ -46,7 +46,7 @@ def fixture(): let context=q.binding.as_ref().unwrap().context.clone(); if predecessor!=0 { let old=Context::new(100+owner,&live); - q.previous=Some(Previous{binding:if predecessor==1{Some(Binding{context:old.clone()})}else{None},context:old,publication:None}); + q.previous=Some(Previous{binding:if predecessor==1{Some(Binding{context:old.clone()})}else{None},context:old,publication:None,moved:false}); } channels.push(Backend{q:std::cell::RefCell::new(q),owner,context}); } @@ -64,12 +64,12 @@ def fixture(): match guard { 0=>q.unpublished_binding=false, 1=>q.exit=Some(Exit{phase:Phase::Acknowledged}), - 2=>q.active.push(Active{packet:Arc::new(Packet{completion:Arc::new(Completion)}),ticket:None,qos_pending:false}).unwrap(), + 2=>q.active.push(Active{packet:Arc::new(Packet{completion:Arc::new(Completion)}),ticket:None,observed:None,qos_pending:false}).unwrap(), 3=>q.replays.push(Arc::new(Packet{completion:Arc::new(Completion)})).unwrap(), 4=>q.growing=true, 5=>q.quarantined=true, 6=>q.awaiting_witness=true, - 7=>q.previous=Some(Previous{context:context.clone(),binding:None,publication:Some((1,1))}), + 7=>q.previous=Some(Previous{context:context.clone(),binding:None,publication:Some((1,1)),moved:false}), 8=>q.pool_phase=PoolPhase::ReleaseSent(3), 9=>owner=3, 10=>ctx=Context::new(4,&live), @@ -91,7 +91,9 @@ def fixture(): def main(): source=fixture() compute=(ROOT/'queue/compute.rs').read_text() - assert compute.count('visible = true;\n self.unpublished_binding = false;')==2 + # Every publication path (G17P shared, G17P, M5) clears the unpublished + # binding exactly when its record becomes firmware-visible. + assert compute.count('visible = true;\n self.unpublished_binding = false;')==3 backend=(ROOT/'runtime/backend.rs').read_text() assert 'if !lease.is_current() { drop(state); drop(lease); continue; }' in backend no_kill, count = re.subn(r'\|\|\s*self\.exit\.is_some\(\)\s*(?=\|\|\s*self\.active\.len\s*!=\s*0)', '', source) diff --git a/tools/asahi/neo-auxiliary-tests.py b/tools/asahi/neo-auxiliary-tests.py index 68236fe19d192..2fc8ada61d93a 100644 --- a/tools/asahi/neo-auxiliary-tests.py +++ b/tools/asahi/neo-auxiliary-tests.py @@ -79,15 +79,20 @@ def main(): args.output.mkdir(parents=True, exist_ok=True) source = (subprocess.check_output(['git','show','990a45b609dd:'+PATH.replace('/asahi/', '/asahi-neo/')], cwd=ROOT,text=True) if args.old else (ROOT/PATH).read_text()) - structure = item(source, 'pub(super) struct Generation').replace('pub(super)', '') - constructor = item(source[source.index('impl Generation'):], ' fn new(') - addresses = item(source, ' pub(super) fn addresses').replace('pub(super)', '') + # The G17P generation is NeoGeneration since the M5 one joined it behind + # the Generation enum; the old revision still calls it Generation. + generation = 'Generation' if args.old else 'NeoGeneration' + structure = item(source, 'pub(super) struct ' + generation + ' ').replace('pub(super)', '') + implementation = source[source.index('impl ' + generation + ' {'):] + constructor = item(implementation, ' fn new(') + addresses = item(implementation, ' pub(super) fn addresses').replace('pub(super)', '') storage=(ROOT/'drivers/gpu/drm/asahi/g17/fw/render/storage.rs').read_text() constants=[] for name in ['STATE_SIZE','DEFLAKE','TA_STATUS','AUXILIARY','AUXILIARY_SIZE']: line=next(x for x in storage.splitlines() if 'const '+name+':' in x) constants.append(line.replace('pub(crate)','pub')) - code=PREFIX.replace('CONSTANTS','\n'.join(constants))+structure+'\nimpl Generation{'+constructor+addresses+'}\n'+MAIN + code=(PREFIX.replace('CONSTANTS','\n'.join(constants))+structure+'\nimpl '+generation+'{'+constructor+addresses+'}\n' + +('' if generation=='Generation' else 'type Generation='+generation+';\n')+MAIN) if args.mutation: code=code.replace('self.state.gpu_va() + storage::AUXILIARY as u64','self.state.gpu_va()') path=args.output/'controls.rs';path.write_text(code) diff --git a/tools/asahi/pciec-policy-tests.py b/tools/asahi/pciec-policy-tests.py index a399c7ef6e023..a5a8fb96e143d 100644 --- a/tools/asahi/pciec-policy-tests.py +++ b/tools/asahi/pciec-policy-tests.py @@ -144,7 +144,7 @@ def main(): args = parser.parse_args() pci = (ROOT / 'drivers/pci/controller/pcie-apple.c').read_text() acio = (ROOT / 'drivers/thunderbolt/apple.c').read_text() - defines = '\n'.join(re.findall(r'^#define .*$', pci, re.M)) + defines = '\n'.join(re.findall(r'^#define (?:[^\n]*\\\n)*[^\n]*$', pci, re.M)) info = pci[pci.index('struct hw_info {'):pci.index('static const struct of_device_id apple_pcie_of_match[];')] matches = re.search(r'static const struct of_device_id apple_pcie_of_match\[\] = \{.*?\n\};', pci, re.S)[0] production = '\n'.join((defines, info, matches, diff --git a/tools/asahi/rtkit-crashlog-tests.py b/tools/asahi/rtkit-crashlog-tests.py new file mode 100755 index 0000000000000..382c8d39cd1a3 --- /dev/null +++ b/tools/asahi/rtkit-crashlog-tests.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Feed RTKit's crashlog parser a container log and malformed logs. + +drivers/soc/apple/rtkit-crashlog.c is compiled whole and unchanged, with ASan +and UBSan, against a stand-in for rtkit-internal.h that carries the real +header's crashlog definitions. Every log is allocated at its exact size, so a +read past it is a sanitizer error, and a walk that stops advancing runs into +the line limit. + +The container log has the layout of a T6050 DCP crash (a 'CLGH' container, +a nested one, and at 0x2000 the 'CLHE' log of task RTAL with eleven sections +and a footer), with neutral contents. The malformed logs cover a zero and an +oversized section size, a truncated section header, a short mailbox section, +a short and a truncated 'Crl4' section, unterminated strings, short buffers +and broken containers. --log FILE feeds a raw crashlog buffer read from a +device instead, and prints what the parser makes of it. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = 'drivers/soc/apple/rtkit-crashlog.c' +HEADER = 'drivers/soc/apple/rtkit-internal.h' + +# Each mutation removes one bound; a test case must catch it. +MUTATIONS = { + 'buffer-header': ('if (size < sizeof(header)) {', 'if (0) {'), + 'section-header': ('if (size - offset < 16) {', 'if (0) {'), + 'section-size': ('if (section_size < 16 || section_size > size - offset) {', 'if (0) {'), + 'footer-room': ('if (section_size > size - offset - 16)\n\t\t\tbreak;', ''), + 'mailbox-minimum': ('if (size < 28) {', 'if (0) {'), + 'version-string': ('"RTKit: Version: %.*s", (int)(size - 16), bfr + 16', '"RTKit: Version: %s", bfr + 16'), + 'message-string': ('%.*s", idx,\n\t\t\t\t (int)(end - ptr), ptr', '%s", idx,\n\t\t\t\t ptr'), +} + +STUB = r''' +#include +#include +#include +#include +#include +#include +#include +#include +/* As linux/build_bug.h has it: the message is optional. */ +#undef static_assert +#define __static_assert(expr, msg, ...) _Static_assert(expr, msg) +#define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr) +typedef uint8_t u8; +typedef uint32_t u32; +typedef unsigned long long u64; +#define __packed __attribute__((packed)) +#define BIT(n) (1UL << (n)) +#define EINVAL 22 +#define EAGAIN 11 +struct device { int id; }; +struct apple_rtkit { struct device *dev; }; +void dev_printk(const char *level, struct device *dev, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); +#define dev_warn(dev, ...) dev_printk("warn", dev, __VA_ARGS__) +#define dev_dbg(dev, ...) dev_printk("dbg", dev, __VA_ARGS__) +@DEFINITIONS@ +int apple_rtkit_crashlog_locate(const u8 *bfr, size_t size, size_t *log_offset, + u32 *log_id, unsigned int *depth); +void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size); +''' + +UNALIGNED = r''' +#include +#include +static inline uint32_t get_unaligned_le32(const void *p) { uint32_t v; memcpy(&v, p, 4); return v; } +static inline unsigned long long get_unaligned_le64(const void *p) +{ unsigned long long v; memcpy(&v, p, 8); return v; } +''' + +MAIN = r''' +#include "rtkit-internal.h" + +#define FOURCC(s) (((u32)(s)[0] << 24) | ((u32)(s)[1] << 16) | ((u32)(s)[2] << 8) | (u32)(s)[3]) +#define MAX_LINES 1000 + +static char lines[MAX_LINES][512]; +static unsigned int n_lines; +static struct device device; +static struct apple_rtkit rtk = { .dev = &device }; + +void dev_printk(const char *level, struct device *dev, const char *fmt, ...) +{ + char *line = lines[n_lines]; + va_list ap; + size_t n; + + assert(dev == &device); + /* A walk that does not advance prints without end. */ + assert(n_lines < MAX_LINES - 1); + n_lines++; + va_start(ap, fmt); + vsnprintf(line, sizeof(lines[0]), fmt, ap); + va_end(ap); + n = strlen(line); + while (n && line[n - 1] == '\n') + line[--n] = 0; + printf(" %s: %s\n", level, line); +} + +static void put32(u8 *b, size_t off, u32 v) { memcpy(b + off, &v, 4); } +static void put64(u8 *b, size_t off, u64 v) { memcpy(b + off, &v, 8); } + +/* A section header: its type, a fill word, flags and its size, header included. */ +static size_t section(u8 *b, size_t off, const char *type, u32 flags, u32 size) +{ + put32(b, off, FOURCC(type)); + put32(b, off + 4, 0xefefefef); + put32(b, off + 8, flags); + put32(b, off + 12, size); + return off + size; +} + +/* A 'CLHE' header (or footer) of a log of `size` bytes at `off`. */ +static void clhe(u8 *b, size_t off, u32 size, u32 flags) +{ + put32(b, off, FOURCC("CLHE")); + put32(b, off + 4, 0x303); + put32(b, off + 8, size); + put32(b, off + 12, flags); +} + +/* A 'CLGH' container at `off` with `count` (id, offset) records. */ +static void clgh(u8 *b, size_t off, u32 size, const char *current, unsigned int count, + const char *const *ids, const u32 *offsets) +{ + put32(b, off, FOURCC("CLGH")); + put32(b, off + 4, 0x202); + put32(b, off + 8, size); + put32(b, off + 12, FOURCC(current)); + put32(b, off + 16, count); + for (unsigned int i = 0; i < count; i++) { + put64(b, off + 0x14 + i * 0x18, FOURCC(ids[i])); + put32(b, off + 0x14 + i * 0x18 + 16, offsets[i]); + } +} + +/* Runs the dump on an exact-size copy of `log` and returns the number of lines. */ +static unsigned int dump(const char *name, const u8 *log, size_t size) +{ + u8 *copy = malloc(size ? size : 1); + + assert(copy); + memcpy(copy, log, size); + n_lines = 0; + printf("%s (%zu bytes)\n", name, size); + apple_rtkit_crashlog_dump(&rtk, copy, size); + free(copy); + return n_lines; +} + +static bool printed(const char *text) +{ + for (unsigned int i = 0; i < n_lines; i++) + if (strstr(lines[i], text)) + return true; + return false; +} + +static bool ended(void) +{ + return n_lines && !strcmp(lines[n_lines - 1], "RTKit: End of crashlog reached"); +} + +#define DCP_SIZE 0x17000 +#define DCP_LOG 0x2000 +#define DCP_LOG_SIZE 0x26e0 +#define MAILBOX_ENTRIES 128 + +/* The layout of a T6050 DCP crashlog: the RTAL task's log, two containers deep. */ +static void dcp_log(u8 *b) +{ + static const char *const top[] = { "CL4U", "CL4K" }, *const tasks[] = { "root", "RTAL" }; + static const u32 top_offsets[] = { 0x44, 0x12000 }, task_offsets[] = { 0x44, 0x1fbc }; + static const u32 pattern[8] = { 0x202, 0x242, 0x302, 0x342, 0x302, 0x342, 0x202, 0x242 }; + size_t off = DCP_LOG + 0x20, body; + + memset(b, 0, DCP_SIZE); + clgh(b, 0, DCP_SIZE, "CL4U", 2, top, top_offsets); + clgh(b, 0x44, 0x11fbc, "RTAL", 2, tasks, task_offsets); + clhe(b, DCP_LOG, DCP_LOG_SIZE, 0x820); + put32(b, DCP_LOG + 0x1c, FOURCC("RTAL")); + + body = off + 16; + off = section(b, off, "Cstr", 0x101, 0x60); + put32(b, body, 1); + strcpy((char *)b + body + 4, "crash message [0x87aa9a0]"); + off = section(b, off, "Cver", 0x300, 0xa0); + body = off + 16; + off = section(b, off, "Ctim", 0x100, 0x20); + put64(b, body, 87481704354ULL); + off = section(b, off, "Ccst", 0x400, 0x70); + off = section(b, off, "Crtk", 0x300, 0x17c0); + body = off + 16; + off = section(b, off, "Cmbx", 0x300, 0xc30); + for (unsigned int i = 0; i < MAILBOX_ENTRIES; i++) { + size_t e = body + 28 + i * 0x18; + + put64(b, e, 0x37); + put64(b, e + 8, pattern[i % 8]); + put32(b, e + 16, 0xda8656a9 + i * 0x1000); + } + off = section(b, off, "Cpct", 0x200, 0x50); + off = section(b, off, "Csta", 0x100, 0x30); + off = section(b, off, "Cced", 0x100, 0x50); + off = section(b, off, "Chex", 0x100, 0x20); + off = section(b, off, "Csta", 0x100, 0x30); + assert(off == DCP_LOG + DCP_LOG_SIZE - 0x20); + clhe(b, off, DCP_LOG_SIZE, 0x820); +} + +static void test_dcp_log(void) +{ + static const char *const expect[] = { + "RTKit: Crashlog container: log 'RTAL' at 0x2000 (depth 2)", + "RTKit: Message (id=1): crash message [0x87aa9a0]", + "RTKit: Version: ", + "RTKit: Crash time: 87481704354", + "RTKit: Unknown crashlog section: 43637374", + "RTKit: Unknown crashlog section: 4372746b", + "RTKit: Mailbox history (type = 0, index = 0)", + "RTKit: Unknown crashlog section: 43706374", + "RTKit: Unknown crashlog section: 43737461", + "RTKit: Unknown crashlog section: 43636564", + "RTKit: Unknown crashlog section: 43686578", + "RTKit: Unknown crashlog section: 43737461", + "RTKit: End of crashlog reached", + }; + static u8 b[DCP_SIZE]; + unsigned int n, i, j = 0, entries = 0; + size_t log_offset; + unsigned int depth; + u32 id; + + dcp_log(b); + assert(!apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth)); + assert(log_offset == DCP_LOG && id == FOURCC("RTAL") && depth == 2); + n = dump("T6050 DCP container log", b, DCP_SIZE); + for (i = 0; i < n; i++) { + if (!strncmp(lines[i], "RTKit: #", 9)) { + entries++; + continue; + } + assert(j < sizeof(expect) / sizeof(expect[0])); + assert(!strcmp(lines[i], expect[j])); + j++; + } + assert(j == sizeof(expect) / sizeof(expect[0])); + assert(entries == MAILBOX_ENTRIES); + assert(printed("RTKit: #000@da8656a9: 0000000000000037 0000000000000202")); + assert(printed("RTKit: #127@da8e46a9: 0000000000000037 0000000000000242")); + + /* Still being written: not a complete log. */ + put32(b, DCP_LOG + 12, 0x820 | 0x100); + assert(apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth) == -EAGAIN); + dump("log still being written", b, DCP_SIZE); + assert(printed("holds no complete log (-11)")); + put32(b, DCP_LOG + 12, 0x820); + + /* A record past the buffer, and a container that names itself. */ + put32(b, 0x44 + 0x14 + 0x18 + 16, DCP_SIZE); + assert(apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth) == -EINVAL); + dump("record past the buffer", b, DCP_SIZE); + assert(printed("holds no complete log (-22)")); + put32(b, 0x44 + 0x14 + 0x18 + 16, 0); + assert(apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth) == -EINVAL); + /* The log's section walk is bounded by the log, not the buffer. */ + dcp_log(b); + put32(b, DCP_LOG + 8, 0x20 + 0x60); + dump("log cut after its first section", b, DCP_SIZE); + assert(printed("End of crashlog reached but no footer present") && !printed("Message")); +} + +/* A bare log: a header, then sections built by the caller, then optionally a footer. */ +static u8 logbuf[0x1000]; + +static size_t bare(void) +{ + memset(logbuf, 0, sizeof(logbuf)); + return 0x20; +} + +static unsigned int finish(const char *name, size_t off, bool footer) +{ + size_t size = off + (footer ? 0x20 : 0); + + if (footer) + clhe(logbuf, off, size, 0); + clhe(logbuf, 0, size, 0); + return dump(name, logbuf, size); +} + +static void test_malformed(void) +{ + size_t off, body; + unsigned int i; + + off = section(logbuf, bare(), "Ctim", 0x100, 0x20); + section(logbuf, off, "Csta", 0x100, 0); + finish("zero section size", off + 0x30, true); + assert(printed("section 43737461 at 0x40 has a bad size 0x0") && n_lines == 2); + + off = section(logbuf, bare(), "Ctim", 0x100, 0x20); + section(logbuf, off, "Cmbx", 0x300, 0x10000); + finish("oversized section", off + 0x40, true); + assert(printed("section 436d6278 at 0x40 has a bad size 0x10000")); + + off = bare(); + put32(logbuf, off, FOURCC("Ctim")); + finish("truncated section header", off + 8, false); + assert(printed("section header at 0x20 is truncated") && n_lines == 1); + + off = section(logbuf, bare(), "Ctim", 0x100, 0x20); + finish("section, then the footer", off, true); + assert(printed("Crash time: 0") && ended() && n_lines == 2); + + /* Without a footer the last section may not be decoded: its decoder reads 16 bytes on. */ + body = bare(); + off = section(logbuf, body, "Cstr", 0x101, 0x40); + put32(logbuf, body + 16, 1); + memset(logbuf + body + 20, 'm', 0x40 - 20); + finish("section reaching the end of a log without a footer", off, false); + assert(printed("End of crashlog reached but no footer present") && n_lines == 1); + + for (u32 size = 16; size < 28; size++) { + off = section(logbuf, bare(), "Cmbx", 0x300, size); + finish("short mailbox section", off, true); + assert(printed("Mailbox section too small") && ended()); + } + off = section(logbuf, bare(), "Cmbx", 0x300, 28 + 2 * 0x18); + finish("two-entry mailbox section", off, true); + assert(printed("RTKit: #001@") && !printed("#002") && ended()); + + body = bare(); + off = section(logbuf, body, "Crl4", 0x100, 0x378); + put32(logbuf, body + 0x14, 0xd8); + put64(logbuf, body + 0x110, 0xfffffe0007001234ULL); + put64(logbuf, body + 0x360, 0xdead0000ULL); + for (i = 0; i < 28; i++) + put64(logbuf, body + 0x18 + i * 8, 0x1000 + i); + finish("whole Crl4 section", off, true); + assert(printed("Exception dump (Crl4):") && printed("PC = 0xfffffe0007001234") && + printed("FAR = 0xdead0000") && + printed("x24-x27 = 0000000000001018 0000000000001019 000000000000101a 000000000000101b") && + ended()); + + off = section(logbuf, bare(), "Crl4", 0x100, 0x378); + finish("truncated Crl4 section", 0x20 + 0x100, true); + assert(printed("section 43726c34 at 0x20 has a bad size 0x378")); + + off = section(logbuf, bare(), "Crl4", 0x100, 0x40); + finish("short Crl4 section", off, true); + assert(printed("Crl4 section too small: 0x40") && ended()); + + body = bare(); + off = section(logbuf, body, "Crl4", 0x100, 0x378); + put32(logbuf, body + 0x10, 1 << 16); + put32(logbuf, body + 0x14, 0xd8); + finish("undecodable Crl4 frame", off, true); + assert(printed("Crl4 frame not decodable") && ended()); + + /* Strings without a terminator, against the end of the log. */ + body = bare(); + off = section(logbuf, body, "Cver", 0x300, 0x40); + memset(logbuf + body + 32, 'v', 0x40 - 32); + put32(logbuf, off, FOURCC("CLHE")); + memset(logbuf + off + 4, 0x55, 12); + clhe(logbuf, 0, off + 16, 0); + dump("unterminated version", logbuf, off + 16); + assert(!strcmp(lines[0], "RTKit: Version: vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvEHLCUUUUUUUUUUUU")); + + body = bare(); + off = section(logbuf, body, "Cstr", 0x101, 0x40); + put32(logbuf, body + 16, 1); + memset(logbuf + body + 20, 'm', 0x40 - 20); + put32(logbuf, off, FOURCC("CLHE")); + memset(logbuf + off + 4, 0x55, 12); + clhe(logbuf, 0, off + 16, 0); + dump("unterminated message", logbuf, off + 16); + assert(!strcmp(lines[0], "RTKit: Message (id=1): mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmEHLCUUUUUUUUUUUU")); + + body = bare(); + off = section(logbuf, body, "Cstr", 0x101, 0x40); + put32(logbuf, body + 16, 2); + strcpy((char *)logbuf + body + 20, "first\nsecond\n"); + finish("message lines", off, true); + assert(printed("Message (id=2): first") && printed("Message (id=2): second") && ended()); + + /* Buffers too short for a header, and logs larger than their buffer. */ + for (size_t size = 0; size < 0x20; size += 4) { + memset(logbuf, 0, sizeof(logbuf)); + clhe(logbuf, 0, 0x20, 0); + dump("short buffer", logbuf, size); + assert(printed("Crashlog buffer too small")); + } + clhe(logbuf, 0, 0x40, 0); + dump("log larger than its buffer", logbuf, 0x20); + assert(printed("Crashlog size (40) is too large")); + memset(logbuf, 0, sizeof(logbuf)); + put32(logbuf, 0, FOURCC("CLGH")); + dump("container header only", logbuf, 0x20); + assert(printed("holds no complete log (-22)")); +} + +static int test_file(const char *path) +{ + FILE *f = fopen(path, "rb"); + size_t size; + u8 *b; + + assert(f); + fseek(f, 0, SEEK_END); + size = ftell(f); + rewind(f); + b = malloc(size); + assert(b && fread(b, 1, size, f) == size); + fclose(f); + dump(path, b, size); + free(b); + return 0; +} + +int main(int argc, char **argv) +{ + assert(FOURCC("CLHE") == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC); + assert(FOURCC("CLGH") == APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC); + if (argc > 1) + return test_file(argv[1]); + test_dcp_log(); + test_malformed(); + printf("PASS rtkit crashlog: container log decoded; malformed logs refused without a loop or an over-read\n"); + return 0; +} +''' + + +def definitions(header): + """The crashlog fourccs and header struct of rtkit-internal.h.""" + found = re.findall(r'^#define APPLE_RTKIT_CRASHLOG_\w+_FOURCC .*$', header, re.M) + found += re.findall(r'^struct apple_rtkit_crashlog_header \{.*?^\};\n(?:\nstatic_assert\(.*?\);)?', + header, re.M | re.S) + assert len(found) == 3, found + return '\n'.join(found) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--ref', help='Git revision of the parser to exercise') + parser.add_argument('--out', type=Path) + parser.add_argument('--mutation', choices=sorted(MUTATIONS)) + parser.add_argument('--log', type=Path, help='a raw crashlog buffer to dump') + args = parser.parse_args() + + def read(path): + if args.ref: + return subprocess.check_output(['git', 'show', f'{args.ref}:{path}'], cwd=ROOT, text=True) + return (ROOT / path).read_text() + source, header = read(SOURCE), read(HEADER) + if args.mutation: + old, new = MUTATIONS[args.mutation] + # A mutation that no longer matches the parser would test nothing. + assert source.count(old) == 1, 'stale mutation: ' + args.mutation + source = source.replace(old, new) + + def run(out): + out.mkdir(parents=True, exist_ok=True) + (out / 'linux').mkdir(exist_ok=True) + (out / 'rtkit-crashlog.c').write_text(source) + (out / 'rtkit-internal.h').write_text(STUB.replace('@DEFINITIONS@', definitions(header))) + (out / 'linux/unaligned.h').write_text(UNALIGNED) + (out / 'main.c').write_text(MAIN) + exe = out / 'crashlog-tests' + compiler = subprocess.run(['clang', '-std=gnu11', '-O1', '-g', '-Wall', '-Werror', '-Wno-unused-function', + '-fsanitize=address,undefined', '-fno-sanitize-recover=all', + '-I', str(out), str(out / 'rtkit-crashlog.c'), str(out / 'main.c'), + '-o', str(exe)], capture_output=True, text=True) + (out / 'compile.log').write_text(compiler.stdout + compiler.stderr) + if compiler.returncode: + print(compiler.stdout + compiler.stderr) + compiler.check_returncode() + env = dict(os.environ, ASAN_OPTIONS='abort_on_error=1:detect_leaks=0', UBSAN_OPTIONS='print_stacktrace=1') + result = subprocess.run([str(exe)] + ([str(args.log.resolve())] if args.log else []), + capture_output=True, text=True, env=env) + receipt = {'source_sha256': hashlib.sha256(source.encode()).hexdigest(), 'ref': args.ref, + 'mutation': args.mutation, 'log': str(args.log) if args.log else None, + 'exit': result.returncode} + (out / 'results.json').write_text(json.dumps(dict(receipt, stdout=result.stdout, + stderr=result.stderr), indent=2) + '\n') + print(result.stdout + result.stderr) + print(json.dumps(receipt)) + return result.returncode != 0 + + if args.out: + return run(args.out) + with tempfile.TemporaryDirectory(prefix='rtkit-crashlog-') as out: + return run(Path(out)) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index e472c9d74cc10..348141cf23df5 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -154,6 +154,9 @@ def mutations(flag, names, base, expect=KILL): 'offset-sign', 'event-bit', 'wrong-control', 'maverick-event', 'stowe-offset'), out()), 'qspi-nvram-tests.py': [(out('--output'), PASS)], + 'rtkit-crashlog-tests.py': [(out(), PASS)] + + mutations('--mutation', ('buffer-header', 'section-header', 'section-size', 'footer-room', + 'mailbox-minimum', 'version-string', 'message-string'), out()), 'rtkit-oslog-tests.py': [(out(), PASS)] + mutations('--mutate', ('admission', 'channel', 'counter', 'reset', 'buffer'), out()), 't6030-clock-tests.py': [(out(), PASS)], diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 0b0423fbb8e4e..f30cbb2ee12f2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -15,9 +15,11 @@ correlation, token preservation and command ID packing. The owned-copy test extracts the actual production function from afk.c, prints its SHA256, and compiles it with synthetic command slots and CPU buffers. It -checks active/inactive/done slots, high device addresses, boundaries and copies. -Locks and DMA barriers are mocked: this test does not qualify concurrency, -DMA coherency, firmware, a graphics device or any hardware display behavior. +checks active/inactive/done slots, high device addresses, boundaries and copies, +and that only firmware 27 hands an unowned buffer to its retained-DART path. +Locks, DMA barriers and that path (afk_v27_access_remote()) are mocked: this +test does not qualify concurrency, DMA coherency, firmware, a graphics device +or any hardware display behavior. All build products are temporary. No device node is opened and no GPU, firmware or target machine is contacted. The extractor fails if production function diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c index 8ff88ec1ec5b0..49d9340e1fd43 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c @@ -38,10 +38,40 @@ struct epic_cmd_info { size_t rxlen, txlen; }; +enum dcp_firmware_version { + DCP_FIRMWARE_UNKNOWN, + DCP_FIRMWARE_V_27_0, +}; + +struct apple_dcp { + enum dcp_firmware_version fw_compat; +}; + struct apple_dcp_afkep { bool wire_v2; + struct apple_dcp *dcp; }; +/* Firmware 27's retained-DART fallback: record the call, return the verdict. */ +static int remote_calls, remote_ret = -EACCES; +static u64 remote_dva; +static size_t remote_size; +static bool remote_to_remote, remote_copy; + +static int afk_v27_access_remote(struct apple_dcp_afkep *ep, u64 dva, + void *buffer, size_t size, bool to_remote, + bool copy) +{ + (void)ep; + (void)buffer; + remote_calls++; + remote_dva = dva; + remote_size = size; + remote_to_remote = to_remote; + remote_copy = copy; + return remote_ret; +} + struct apple_epic_service { int lock; struct apple_dcp_afkep *ep; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c index 770b16bc2fa94..f5cbfc7396088 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c @@ -3,7 +3,8 @@ int main(void) { u8 rx[16], tx[16], output[16]; - struct apple_dcp_afkep ep = { .wire_v2 = true }; + struct apple_dcp dcp = { .fw_compat = DCP_FIRMWARE_UNKNOWN }; + struct apple_dcp_afkep ep = { .wire_v2 = true, .dcp = &dcp }; struct apple_epic_service service = { .ep = &ep }; struct epic_cmd_info *cmd = &service.cmds[0]; @@ -51,6 +52,24 @@ int main(void) false) == 0); assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 16, true, false) == -EACCES); - puts("PASS owned-copy boundaries (actual extracted function; lock/barrier mocks)"); + /* Only firmware 27 falls back to its retained mappings, and only when unowned. */ + assert(remote_calls == 0); + dcp.fw_compat = DCP_FIRMWARE_V_27_0; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 12, output, 4, + false) == 0); + assert(remote_calls == 0); + assert(afk_v2_copy_owned(&service, 0x20000003000ULL, output, 4, true) == + -EACCES); + assert(remote_calls == 1 && remote_dva == 0x20000003000ULL && + remote_size == 4 && remote_to_remote && remote_copy); + remote_ret = 0; + assert(afk_v2_access_owned(&service, 0x20000003000ULL, NULL, 8, false, + false) == 0); + assert(remote_calls == 2 && remote_size == 8 && !remote_to_remote && + !remote_copy); + assert(afk_v2_copy_owned(&service, 0x20000003000ULL, NULL, 1, false) == + -EINVAL); + assert(remote_calls == 2); + puts("PASS owned-copy boundaries and firmware-27 fallback (actual extracted function; lock/barrier and fallback mocks)"); return 0; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index a7f88da81652e..d42cc06149b66 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -221,6 +221,7 @@ def brightness_takeover(root): #define WRITE_ONCE(x,v) ((x)=(v)) #define DCP_FIRMWARE_V_26_6_DESKTOP 5 #define DCP_FIRMWARE_H17P 1 +#define DCP_FIRMWARE_V_27_0 6 #define DCP_IOMFB_METHODS_H17G 7 #define MAX_BRIGHTNESS_PART2 510U #define BACKLIGHT_PLATFORM 1 @@ -397,7 +398,8 @@ def panel_identity(root, output=None): backend = (root / 'drivers/gpu/drm/apple/iomfb_v26_6_desktop.c').read_text() dictionary = block((root/'drivers/gpu/drm/apple/parser.c').read_text(), 'static int parse_panel_edid_value(') dictionary += block((root/'drivers/gpu/drm/apple/parser.c').read_text(), 'int parse_panel_edid_data(') - dictionary += block(backend, 'struct dcp_dict_passthrough_req_v26_6 {')+';\n' + # The dictionary record is shared with firmware 27's analytics event. + dictionary += block((root/'drivers/gpu/drm/apple/iomfb.h').read_text(), 'struct dcp_dict_passthrough_req {')+';\n' dictionary += block(backend, 'static bool trampoline_panel_dictionary_v26_6(') dictionary += block(backend, 'static bool iomfb_check_callback_exact_v26_6_desktop(') dictionary += '\n#define iomfb_check_callback_v26_6_desktop iomfb_check_callback_exact_v26_6_desktop\n' @@ -462,7 +464,7 @@ def panel_identity(root, output=None): }; for(unsigned form=0;form<3;form++) { desktop=(__typeof__(desktop)){0};notifications=0; - struct dcp_dict_passthrough_req_v26_6 wire={0};u32 result=0; + struct dcp_dict_passthrough_req wire={0};u32 result=0; strcpy((char *)wire.key,form?"ProductAttributes":"DisplayAttributes"); const u8 *data=form==2?minimal:form?product:attributes; size_t size=form==2?sizeof(minimal):form?sizeof(product):sizeof(attributes); @@ -595,6 +597,7 @@ def powerlog_framing(root): prefix += r''' #define DCP_FIRMWARE_H17P 1 #define DCP_FIRMWARE_V_26_6_DESKTOP 5 +#define DCP_FIRMWARE_V_27_0 6 #define DCP_IOMFB_METHODS_H17G 7 #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) @@ -663,10 +666,18 @@ def powerlog_framing(root): d.fw_compat=DCP_FIRMWARE_H17P; afk_recv_compact_message(&ep,3,wire,sizeof(wire)-8); assert(hints==3 && !service.users); + d.fw_compat=DCP_FIRMWARE_V_27_0; + afk_recv_compact_message(&ep,3,wire,sizeof(wire)-8); + assert(hints==3 && !service.users); d.fw_compat=DCP_FIRMWARE_V_26_6_DESKTOP; /* The first key without its value gives no reading. */ afk_recv_compact_message(&ep,3,wire,24+12); assert(hints==3 && !service.users); + /* Firmware 27 takes a framed report's hint as H17P does. */ + d.fw_compat=DCP_FIRMWARE_V_27_0;hint=0; + memcpy(wire+16,report,sizeof(report)); + afk_recv_compact_message(&ep,3,wire,16+sizeof(report)); + assert(hints==4 && hint==123000 && !service.users); return 0; } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 5caff361a38ca..50f6bd6f89166 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -53,6 +53,7 @@ def function(file,signature): /* The T6030 14.7 backend never matches these H17P scenarios. */ #define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) #define DCP_FIRMWARE_V_14_7_TEMPLATE (DCP_FIRMWARE_H17P + 2) +#define DCP_FIRMWARE_V_27_0 (DCP_FIRMWARE_H17P + 3) #define EPIC_TYPE_NOTIFY 0 #define EPIC_CAT_REPORT 0 #define max(a,b) ((a)>(b)?(a):(b)) @@ -81,7 +82,7 @@ def function(file,signature): static bool __attribute__((unused)) cancel_work_sync(struct work_struct *w) { bool queued=w->pending;assert(!queued);return queued; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; void afk_service_retry(struct apple_dcp_afkep *ep); -struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*init_props)(struct apple_epic_service *,s64,const void *,size_t);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length,reply_len;bool reply_len_valid;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; @@ -467,13 +468,21 @@ def function(file,signature): optional_admission_controls(); struct item h17p[]={{"compartment","dcpav-service-epic",0},{"name","AppleDCPDP2HDMI",0},{"interface-id",NULL,13}}; struct dcp_parse_ctx parse_ctx={.items=h17p,.count=3};const char *parsed_name,*parsed_class;s64 parsed_unit=-1; - assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true)); + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true,false)); assert(parsed_unit==13 && !strcmp(parsed_name,name) && !strcmp(parsed_class,class)); struct apple_epic_service *proof=prepare(13,parsed_unit,parsed_name,parsed_class);assert(proof);afk_service_discard(proof); free((void *)parsed_name);free((void *)parsed_class); struct item legacy[]={{"EPICName","dcpav-service-epic",0},{"EPICProviderClass","legacy-class",0},{"EPICUnit",NULL,0}}; parse_ctx=(struct dcp_parse_ctx){.items=legacy,.count=3}; - assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,false));assert(parsed_unit==0); + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,false,false));assert(parsed_unit==0); + free((void *)parsed_name);free((void *)parsed_class); + /* Firmware 27 carries both spellings; its EPICUnit is the unit in either order. */ + struct item both[]={{"compartment","dcpav-service-epic",0},{"name","AppleDCPDP2HDMI",0},{"EPICUnit",NULL,5},{"interface-id",NULL,13}}; + parse_ctx=(struct dcp_parse_ctx){.items=both,.count=4}; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true,true));assert(parsed_unit==5); + free((void *)parsed_name);free((void *)parsed_class); + parse_ctx=(struct dcp_parse_ctx){.items=both,.count=4}; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true,false));assert(parsed_unit==13); free((void *)parsed_name);free((void *)parsed_class); ep.wq=&ep;ep.command_tags=calloc(AFKV2_INTERFACES,1);assert(ep.command_tags); struct apple_epic_service *old=prepare(1,1,name,class);assert(old);publish(old,1); @@ -1025,6 +1034,7 @@ def startup(root): #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 #define DPAVSERV_ENDPOINT 0x28 #define DPAVSERV_ENDPOINT_V2 0x2a +#define DCP_FIRMWARE_V_27_0 7 #define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) #define PTR_ERR(p) ((int)(intptr_t)(p)) #define ERR_PTR(e) ((void *)(intptr_t)(e)) @@ -1041,6 +1051,7 @@ def startup(root): struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; bool ext_dead,external_native; struct { unsigned enable_completion; bool enabled; } dcpavserv; struct apple_dcp_afkep *dcpavservep; + int fw_compat; }; static const unsigned dpavservep_ops[] = {0}; static struct apple_dcp_afkep endpoint; @@ -1100,7 +1111,14 @@ def startup(root): init_error=0; } } - puts("PASS optional legacy discovery, AFKV2 early/absent service, endpoint failures"); + /* Firmware 27 speaks AFKV2 on the legacy endpoint and does not wait. */ + struct apple_dcp v27={.fw_compat=DCP_FIRMWARE_V_27_0}; + announce=false;waits=warnings=starts=shutdowns=detaches=debug=0; + assert(dpavservep_init(&v27)==0); + assert(starts==1&&!waits&&!warnings&&!shutdowns&&!detaches&&!debug); + assert(v27.dcpavservep==&endpoint&&endpoint.wire_v2&&!endpoint.inline_v2); + assert(endpoint.endpoint==DPAVSERV_ENDPOINT&&!v27.dcpavserv.enabled); + puts("PASS optional legacy discovery, AFKV2 early/absent service, firmware 27, endpoint failures"); return 0; } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index f42f8251fee56..38e35b36489f3 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1200,9 +1200,17 @@ def block(marker): struct device *dev; struct apple_dcp_hw_data hw; struct resource *disp_registers[10],disp_bw_scratch_res,disp_bw_doorbell_res; u32 disp_bw_scratch_index,disp_bw_scratch_offset,disp_bw_doorbell_index,nr_disp_registers; + u64 disp_firmware_base[MAX_DISP_REGISTERS]; }; +#define IS_ALIGNED(x, a) (((x) & ((a) - 1)) == 0) +#define U64_MAX UINT64_MAX static bool of_property_present(struct device_node *n,const char *name) { return !strcmp(name,"reg-names") && n->named; } +/* Firmware-physical windows follow the pmgr scratch lookup, which fails here. */ +static int of_property_count_u64_elems(struct device_node *n,const char *name) +{ (void)n;(void)name;assert(!"firmware-physical-regs outside the T8140 profiles");return -EINVAL; } +static int of_property_read_u64_array(struct device_node *n,const char *name,u64 *v,unsigned count) +{ (void)n;(void)name;(void)v;(void)count;assert(!"firmware-physical-regs outside the T8140 profiles");return -EINVAL; } static int of_property_count_strings(struct device_node *n,const char *name) { assert(!strcmp(name,"reg-names"));return n->count+1; } static int of_property_read_string_index(struct device_node *n,const char *name,int i,const char **v) @@ -1978,6 +1986,7 @@ def dispatcher(root): #define WRITE_ONCE(x,v) ((x)=(v)) #define DCP_FIRMWARE_H17P 1 #define DCP_FIRMWARE_V_26_6_DESKTOP 5 +#define DCP_FIRMWARE_V_27_0 6 #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 #define DCP_IOMFB_METHODS_H17G 3 #define DCP_MAX_CALL_DEPTH 8 @@ -2014,6 +2023,9 @@ def dispatcher(root): { (void)s; return n->external; } static bool iomfb_check_callback_h17p(struct apple_dcp *d, int t, u32 i, u32 o) { (void)d; (void)t; (void)i; (void)o; return true; } +/* Only handled callbacks are size-checked; this fixture registers none. */ +static bool iomfb_v27_callback_size_valid(unsigned t, u32 i, u32 o) +{ (void)t; (void)i; (void)o; assert(false); return false; } static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return false; } static bool test_and_set_bit(int t, unsigned long *b) { (void)t; (void)b; return false; } static void complete(int *p) { *p = 1; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py index 4d0e9c50a1082..2848ae1e2d212 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py @@ -514,6 +514,7 @@ def blank(root): #define DRM_FORMAT_XRGB8888 0x34325258U #define DRM_FORMAT_ARGB8888 0x34325241U #define DCP_FIRMWARE_H17P 1 +#define DCP_FIRMWARE_V_27_0 3 #define DCP_IOMFB_METHODS_H17G 2 #define READ_ONCE(x) (x) #define spin_lock_irqsave(p,f) do { (void)(p); (f)=0; } while(0) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index c4c92d244f377..f3bfbc3954b15 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -52,6 +52,11 @@ def wire(value, out): header = header[header.index('struct apple_dcp;'):header.index('int parse(')] refresh = (base / 'dcp-refresh.h').read_text().replace('#include ', '') header = refresh + '\n' + header + # parse_dptx_caps() fills the firmware-27 DPTX capabilities parser.c + # takes from dcp-peer-v27.h. + peer_v27 = base / 'dcp-peer-v27.h' + if peer_v27.exists(): + header += block(peer_v27.read_text(), 'struct dcp_dptx_caps {') + ';\n' # Header helpers stay header-like: unused here, as in any includer. header = header.replace('static inline ', 'static inline __attribute__((unused)) ') edid_header = (root / 'include/drm/drm_edid.h').read_text() @@ -1190,6 +1195,8 @@ def hotplug(root): #define trace_dptxport_call_set_link_rate(...) ((void)0) /* Not a T8132 dcpext: its DRM outputs never follow this link. */ static bool dcp_is_t8132_ext(const struct apple_dcp *d) { (void)d; return false; } +/* The J700 lane policy keys on the external H17P profile, not the AFK v2 codec. */ +static bool dcp_is_external_h17p(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } struct device; static void t8132_ext_drm_hotplug(struct device *dcpext) { (void)dcpext; assert(!"T8132 dcpext hotplug in an H17P scenario"); } static unsigned phy_capacity=2, phy_calls, hardware_rate, hardware_lanes; @@ -1229,10 +1236,11 @@ def hotplug(root): bool external_physically_absent,main_display,typec_crtc_off,external_cable,external_error,external_power_attempted,external_stopped; bool external_hotplug_pending,external_hotplug_connected,typec_poweron_hpd; u64 external_stop_generation,external_rx_generation,external_crtc_generation; - unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready;''') + unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready,typec_display_ready;''') stubs = r''' #define DCP_HOTPLUG_NOTIFY 1 -static bool dcp_is_external_h17p(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } +/* Firmware 27's resume readiness; this H17P display never runs it. */ +#define DCP_FIRMWARE_V_27_0 (DCP_FIRMWARE_H17P + 5) static bool dcp_is_typec_output(struct apple_dcp *d) { (void)d;return true; } static void complete_all(int *p) { (*p)++; } #define dcp_mode_invalidate(s) ((s)->valid=false) @@ -1285,6 +1293,7 @@ def hotplug(root): connected=0;d.typec_crtc_off=true;dcpep_cb_hotplug(&d,&connected);assert(d.rate_proof.sequence==sequence); d.typec_crtc_off=false;d.iomfb.presentation_paused=true;dcpep_cb_hotplug(&d,&connected); assert(d.rate_proof.sequence==sequence && !con.edid_invalidation_pending);d.iomfb.lock=false;probe(true); + assert(!d.typec_display_ready); dcpavserv_teardown(&av_services[0]);cleanup(); puts("PASS: actual RX HPD loss immediately denies getter/lookup before deferred work; 24 fresh-proof recovery orders need only one rate/lane callback; bootstrap/power/paused suppression preserved"); }""".replace('MAIN_PREFIX',n['main_prefix']) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index ac69a16514c3e..a39a47787c368 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -154,7 +154,9 @@ def main(): '#define iomfb_m4_get_mode(...) (-95)\n' '#define iomfb_m4_blank(...) (-95)\n') if 'dcp_fail_stop(' in unit: + # Firmware 27 fail-stops like H17P; no fixture here runs firmware 27. prefix += ('struct apple_dcp;\n' + '#define DCP_FIRMWARE_V_27_0 0x2700\n' 'static inline bool dcp_fail_stop(const struct apple_dcp *dcp);\n') unit += '\n' + block((apple / 'dcp-internal.h').read_text(), 'static inline bool dcp_fail_stop(') diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 2ae5be4b57e56..b0c3105235f7b 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -335,7 +335,7 @@ def main(): harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' - harness += f'#[path = "{G15 / "retain.rs"}"] mod retain;\n' + harness += f'#[path = "{G15.parent / "retain.rs"}"] mod retain;\n' harness += ("mod firmware_identity {\n" + f'#[path = "{G15 / "identity/matching.rs"}"] pub mod matching;\n' + f'#[path = "{G15 / "identity/lookup.rs"}"] pub mod lookup;\n' + "}\n") diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py index 930b97ac24326..b94595827a791 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Host-check production M4 Rust stream parsing with generated kernel UAPI types.""" from common import kernel_root, output_dir, rustc, flat_bindings +from abi import c_layout, differences, rust_layout import hashlib import json @@ -13,7 +14,41 @@ ROOT = kernel_root() OUT = output_dir() SOURCE = ROOT / 'drivers/gpu/drm/asahi/g16/command.rs' -BINDINGS = flat_bindings() +FLAT = flat_bindings() +# The M5 render payload and resource commands are newer than the frozen M4 +# bindings. Declare them beside the flat ones and check their layout and values +# against the kernel header, as the frozen ones are. +M5_UAPI = r''' +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct drm_asahi_render_stage_prefixes { + pub ta_cdm_barrier: __u16, + pub reserved0: __u16, + pub reserved1: __u32, + pub ta_prior_cdm: __u64, + pub ta_vdm_barrier: __u16, + pub reserved2: __u16, + pub reserved3: __u32, + pub ta_prior_vdm: __u64, +} +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct drm_asahi_cmd_render_m5 { + pub legacy: drm_asahi_cmd_render, + pub stage_prefixes: drm_asahi_render_stage_prefixes, +} +pub const drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES: drm_asahi_cmd_type = 5; +pub const drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES: drm_asahi_cmd_type = 6; +pub const drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2: drm_asahi_cmd_type = 7; +pub const drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES: drm_asahi_render_flags = 8388608; +''' +BINDINGS = OUT / 'm5-uapi.rs' +BINDINGS.write_text('include!(' + json.dumps(str(FLAT)) + ');\n' + M5_UAPI) +with tempfile.TemporaryDirectory(prefix='m4-m5-uapi-') as name: + errors = differences(c_layout(ROOT / 'include/uapi/drm/asahi_drm.h', BINDINGS, pathlib.Path(name), 'm5-layout'), + rust_layout(BINDINGS, pathlib.Path(name))) +if errors: + raise RuntimeError('Kernel UAPI ABI differs from the M5 bindings: ' + json.dumps(errors)) command = SOURCE.read_text() hardware = (SOURCE.parent / 'hw.rs').read_text() capacities = '\n'.join(re.findall(r'pub\(crate\) const (?:TPC_SIZE|TILEMAP_SIZE): u64 = .*?;', hardware)) @@ -24,7 +59,8 @@ constants = command[command.index('/// Largest command buffer.'):command.index('/// A validated render command.')] parser = command[command.index('/// One command of the stream.'):command.index('/// Checks that `[start, end)`')] g17 = (ROOT / 'drivers/gpu/drm/asahi/g17/command.rs').read_text() -g17_parser = g17[g17.index('// Commands stay inline'):g17.index('#[derive(Clone, Copy, PartialEq, Eq)]')] +# The stage-prefix offset and the render payload reader precede the parser. +g17_parser = g17[g17.index('/// Where a render command\'s stage-prefix fields start'):g17.index('#[derive(Clone, Copy, PartialEq, Eq)]')] g17_constants = '\n'.join(re.findall(r'const (?:RENDER_BASE_SIZE|COMPUTE_BASE_SIZE): usize = .*?;|const (?:RENDER_TIMESTAMP_COPIES|COMPUTE_TIMESTAMP_COPIES): Range = .*?;', g17, re.S)) reader = (ROOT / 'drivers/gpu/drm/asahi/util.rs').read_text() reader = reader[reader.index('pub(crate) unsafe trait AnyBitPattern:'):] @@ -58,6 +94,21 @@ mod g17 { use super::{uapi, AnyBitPattern, Reader, Result, EINVAL, size_of, offset_of, Range}; G17_CONSTANTS + const STAGE_PREFIXES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; + const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; + const GFP_KERNEL: u32 = 0; + pub struct Arc(std::rc::Rc); + impl Arc { fn new(value: T, _: u32) -> Result { Ok(Self(std::rc::Rc::new(value))) } } + impl Clone for Arc { fn clone(&self) -> Self { Self(self.0.clone()) } } + impl core::ops::Deref for Arc { type Target = T; fn deref(&self) -> &T { &self.0 } } + // The M5 program lists, likewise retained as their bytes. + pub struct Programs(pub Vec, pub u8); + impl Programs { + fn parse(bytes: &[u8]) -> Result { Ok(Self(bytes.to_vec(), 1)) } + fn parse_v2(bytes: &[u8]) -> Result { Ok(Self(bytes.to_vec(), 2)) } + } + pub struct ComputePrograms(pub Vec); + impl ComputePrograms { fn parse(bytes: &[u8]) -> Result { Ok(Self(bytes.to_vec())) } } // These stand-ins only retain extension bytes. This test checks the actual // parser's framing and offsets; backend resource validation has other tests. #[derive(Clone,Copy)] @@ -86,6 +137,9 @@ } G17_PARSER } +const SET_RENDER_RESOURCES: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES; +const SET_COMPUTE_RESOURCES: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES; +const SET_RENDER_RESOURCES_V2: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2; fn data(value: &T) -> Vec { unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size_of::()).to_vec() } } @@ -174,14 +228,16 @@ r.ray_query[0].scs_layout=0x1234; let wire=entry(CMD_RENDER,&data(&r),[NONE;2]); let parsed=g17::Parser::new(&wire).next().unwrap().unwrap(); - let g17::Payload::Render(cmd,_,ray)=parsed.payload else { panic!("wrong engine") }; + let g17::Payload::Render(cmd,prefixes,_,ray,resources)=parsed.payload else { panic!("wrong engine") }; + assert!(prefixes.is_none()&&resources.is_none()); assert_eq!([cmd.prior_vdm,cmd.prior_cdm],[17,29]); assert_eq!(u64::from_le_bytes(ray[0].0[..8].try_into().unwrap()),0x1234); let c=uapi::drm_asahi_cmd_compute {scs_layout:0x1234, prior_vdm:17,prior_cdm:29,cdm_auxiliary_allocation:8,intersection_pool:0x4000,..Default::default()}; let wire=entry(CMD_COMPUTE,&data(&c),[NONE;2]); let parsed=g17::Parser::new(&wire).next().unwrap().unwrap(); - let g17::Payload::Compute(cmd,_,intersection)=parsed.payload else { panic!("wrong engine") }; + let g17::Payload::Compute(cmd,_,intersection,resources)=parsed.payload else { panic!("wrong engine") }; + assert!(resources.is_none()); assert_eq!([cmd.prior_vdm,cmd.prior_cdm],[17,29]); assert_eq!(cmd.scs_layout,0x1234); assert_eq!(u32::from_le_bytes(intersection.0[..4].try_into().unwrap()),8); assert_eq!(u64::from_le_bytes(intersection.0[8..16].try_into().unwrap()),0x4000); @@ -198,6 +254,55 @@ } } #[test] +fn g17_stage_prefixes_follow_the_complete_render() { + let mut m5=uapi::drm_asahi_cmd_render_m5 {legacy:render(),..Default::default()}; + m5.legacy.flags=uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; m5.legacy.prior_vdm=17; + m5.stage_prefixes.ta_cdm_barrier=3; m5.stage_prefixes.ta_prior_vdm=0x1234; + let wire=entry(CMD_RENDER,&data(&m5),[NONE;2]); + let parsed=g17::Parser::new(&wire).next().unwrap().unwrap(); + let g17::Payload::Render(cmd,prefixes,..)=parsed.payload else { panic!("wrong engine") }; + let prefixes=prefixes.expect("stage prefixes"); + assert_eq!(cmd.prior_vdm,17); assert_eq!((prefixes.ta_cdm_barrier,prefixes.ta_prior_vdm),(3,0x1234)); + // The flag needs the complete struct, its reserved fields zero, and no fragment barriers. + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&m5)[..448],[NONE;2])).next().is_err()); + let mut bad=m5; bad.stage_prefixes.reserved1=1; + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&bad),[NONE;2])).next().is_err()); + let mut bad=m5; bad.legacy.flags|=FRAGMENT_BARRIERS; + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&bad),[NONE;2])).next().is_err()); + // Without the flag the prefix fields are absent or zero. + let mut plain=m5; plain.legacy.flags=0; + assert!(g17::Parser::new(&entry(CMD_RENDER,&data(&plain),[NONE;2])).next().is_err()); + plain.stage_prefixes=Default::default(); + let parsed=g17::Parser::new(&entry(CMD_RENDER,&data(&plain),[NONE;2])).next().unwrap().unwrap(); + let g17::Payload::Render(_,prefixes,..)=parsed.payload else { panic!("wrong engine") }; + assert!(prefixes.is_none()); +} +#[test] +fn g17_resource_lists_apply_to_later_commands_until_cleared() { + let mut stream=entry(SET_RENDER_RESOURCES,&[1,2,3,4],[NONE;2]); + stream.extend(entry(SET_COMPUTE_RESOURCES,&[5,6],[NONE;2])); + stream.extend(entry(CMD_RENDER,&data(&render()),[NONE;2])); + stream.extend(entry(CMD_COMPUTE,&data(&compute()),[NONE;2])); + stream.extend(entry(SET_RENDER_RESOURCES_V2,&[7],[NONE;2])); + stream.extend(entry(CMD_RENDER,&data(&render()),[NONE;2])); + stream.extend(entry(SET_RENDER_RESOURCES,&[],[NONE;2])); + stream.extend(entry(CMD_RENDER,&data(&render()),[NONE;2])); + let mut parser=g17::Parser::new(&stream); + let mut renders=Vec::new(); + while let Some(command)=parser.next().unwrap() { + match command.payload { + g17::Payload::Render(..,resources)=>renders.push(resources.map(|r|(r.0.clone(),r.1))), + g17::Payload::Compute(..,resources)=>assert_eq!(resources.map(|r|r.0.clone()),Some(vec![5,6])), + } + } + assert!(parser.finish().is_ok()); + assert_eq!(renders,[Some((vec![1,2,3,4],1)),Some((vec![7],2)),None]); + // Like the attachment commands, they carry no barriers. + for kind in [SET_RENDER_RESOURCES,SET_COMPUTE_RESOURCES,SET_RENDER_RESOURCES_V2] { + assert!(g17::Parser::new(&entry(kind,&[1],[0,NONE])).next().is_err()); + } +} +#[test] fn unsupported_flags_and_compute_fields() { for flag in [1u32,4,1<<19,FRAGMENT_BARRIERS] { let mut r=render(); r.flags=flag; @@ -310,6 +415,6 @@ 'g17_source':str(ROOT/'drivers/gpu/drm/asahi/g17/command.rs'), 'g17_sha256':hashlib.sha256(g17.encode()).hexdigest(), 'uapi_generated':str(BINDINGS),'uapi_sha256':hashlib.sha256(BINDINGS.read_bytes()).hexdigest(), - 'result':'pass','test_groups':14, + 'result':'pass','test_groups':16, 'limitations':'Links complete production encoder/firmware helper modules; compiles isolated production G16 structural parser/reader and G17 framing/prefix parser with generated UAPI structs. G17 extension validators are stand-ins; does not exercise kernel VM/scheduler or hardware.' },indent=2)+'\n') diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py index 3419bca8cd181..901b1264553ea 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-words.py @@ -81,7 +81,7 @@ def body(source,marker): use std::{cell::Cell, rc::Rc, ops::{Range,Deref}};use std::sync::atomic::{AtomicBool,Ordering}; macro_rules! dev_err { ($($arg:tt)*) => {}; } type Result = std::result::Result; -const EOVERFLOW:u32=75; +const EOVERFLOW:u32=75;const EBUSY:u32=16; #[derive(Clone,Copy)]struct Prot; struct KernelMappingInner{mapped_size:usize} mod mm { @@ -95,16 +95,18 @@ def body(source,marker): impl Uat{ fn lock_m4_upper(self:&Rc,upper:bool)->Result{assert!(upper);assert!(!self.held.replace(true),"recursive shared-UAT lock");Ok(Lease(self.clone()))} } -struct Table{uat:Rc,rolled_back:Cell} +struct Table{uat:Rc,rolled_back:Cell,busy:bool} impl Table{ fn prepare_map(&mut self,_r:Range)->Result{assert!(!self.uat.held.get(),"allocation under PPL");Ok(())} fn discard_partial_map(&mut self,_r:Range)->Result{assert!(self.uat.held.get(),"rollback outside PPL");self.rolled_back.set(true);Ok(())} + // Another mapping's live leaves in the range refuse the map before any store. + fn ensure_unmapped(&mut self,_r:Range)->Result{assert!(self.uat.held.get(),"live-leaf check outside PPL");if self.busy{Err(EBUSY)}else{Ok(())}} } -struct Owner{uat_inner:Rc,page_table:Table,is_kernel:bool,fail:bool,invalidations:Cell} +struct Owner{uat_inner:Rc,page_table:Table,is_kernel:bool,fail:bool,invalidations:Cell,stores:Cell} impl Owner{ fn mapping_mutation(&mut self){} fn map_node_prepared(&mut self,_n:&mm::Node<(),KernelMappingInner>,_p:Prot)->Result{ - assert!(self.uat_inner.held.get(),"live leaf stores outside PPL");if self.fail{Err(5)}else{Ok(())} + assert!(self.uat_inner.held.get(),"live leaf stores outside PPL");self.stores.set(self.stores.get()+1);if self.fail{Err(5)}else{Ok(())} } fn tlbi_contexts_naming_root(&self,_va:u64,_size:usize){assert!(!self.uat_inner.held.get(),"naming-root lookup recursively locks UatShared");self.invalidations.set(self.invalidations.get()+1);} fn finish_failed_map(&self,va:u64,size:usize){self.tlbi_contexts_naming_root(va,size);} @@ -113,12 +115,20 @@ def body(source,marker): #[test]fn success_and_rollback_release_ppl_before_root_lookup(){ for fail in [false,true]{ let uat=Rc::new(Uat{held:Cell::new(false),m4_adopted:true,failed:AtomicBool::new(false)}); - let mut owner=Owner{uat_inner:uat.clone(),page_table:Table{uat:uat.clone(),rolled_back:Cell::new(false)},is_kernel:true,fail,invalidations:Cell::new(0)}; + let mut owner=Owner{uat_inner:uat.clone(),page_table:Table{uat:uat.clone(),rolled_back:Cell::new(false),busy:false},is_kernel:true,fail,invalidations:Cell::new(0),stores:Cell::new(0)}; let node=mm::Node{start:0xfffffc2000000000,inner:KernelMappingInner{mapped_size:0x4000},marker:core::marker::PhantomData}; assert_eq!(owner.map_node(&node,Prot),if fail{Err(5)}else{Ok(())}); assert!(!uat.held.get());assert_eq!(owner.page_table.rolled_back.get(),fail);assert_eq!(owner.invalidations.get(),1); } } +#[test]fn a_live_range_is_refused_before_any_store_and_releases_ppl(){ + let uat=Rc::new(Uat{held:Cell::new(false),m4_adopted:true,failed:AtomicBool::new(false)}); + let mut owner=Owner{uat_inner:uat.clone(),page_table:Table{uat:uat.clone(),rolled_back:Cell::new(false),busy:true},is_kernel:true,fail:false,invalidations:Cell::new(0),stores:Cell::new(0)}; + let node=mm::Node{start:0xfffffc2000000000,inner:KernelMappingInner{mapped_size:0x4000},marker:core::marker::PhantomData}; + assert_eq!(owner.map_node(&node,Prot),Err(EBUSY)); + assert!(!uat.held.get());assert_eq!(owner.stores.get(),0);assert!(!owner.page_table.rolled_back.get()); + assert_eq!(owner.invalidations.get(),0);assert!(!uat.failed.load(Ordering::Acquire)); +} '''.replace('__MAP_TRANSACTION__',map_transaction) lock_unit=OUT/'map-node-lock.rs';lock_unit.write_text(lock_source) subprocess.run(rustc()+['--edition=2021','--test',str(lock_unit),'-o',str(OUT/'map-node-lock')],check=True)